%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: mpi-report.dvi %%CreationDate: Sun Aug 15 00:00:01 1993 %%Pages: 150 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips -o mpi-report.ps mpi-report %DVIPSSource: TeX output 1993.08.14:2358 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72 mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1} ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{ statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{ pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get} B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D }B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258437 52099154 1000 300 300 (/amd/iliamna/projects/dsrg/C/staff/otto/papers/Editor/MPI/draft5/mpi-report.dvi) @start /Fa 7 88 df<387CFEFEFE7C380000000000000000387CFEFEFE7C3807167C95 10>58 D<000070000000007000000000F800000000F800000000F800000001FC00000001 FC00000003FE00000003FE00000003FE00000006FF000000067F0000000E7F8000000C3F 8000000C3F800000183FC00000181FC00000381FE00000300FE00000300FE00000600FF0 00006007F00000E007F80000FFFFF80000FFFFF800018001FC00018001FC00038001FE00 030000FE00030000FE000600007F000600007F00FFE00FFFF8FFE00FFFF825227EA12A> 65 D<0003FE0040001FFFC0C0007F00F1C001F8003FC003F0000FC007C00007C00FC000 03C01F800003C03F000001C03F000001C07F000000C07E000000C07E000000C0FE000000 00FE00000000FE00000000FE00000000FE00000000FE00000000FE00000000FE000FFFFC 7E000FFFFC7F00001FC07F00001FC03F00001FC03F00001FC01F80001FC00FC0001FC007 E0001FC003F0001FC001FC003FC0007F80E7C0001FFFC3C00003FF00C026227DA12C>71 D73 D78 D82 D87 D E /Fb 1 1 df0 D E /Fc 1 106 df<040C0000000000705898983030606464683006127E910B>105 D E /Fd 5 118 df<03F80FFC1C1C380870006000E000E000E000E00060007000380E1C 1E0FFC03F00F107E8F14>99 D110 D<07C01FF03C78701C701CE00EE00EE00EE00EE00EE00E701C783C3C781FF007C00F107E 8F14>I<030007000700070007007FFCFFFC07000700070007000700070007000700070E 070E070E070C03FC00F00F157F9414>116 DI E /Fe 9 90 df<0000180000300000600000E00000C0000180000380000700000600 000E00000C00001C0000380000380000700000700000E00000E00001E00001C00001C000 0380000380000380000780000700000700000F00000E00000E00001E00001E00001E0000 1C00001C00003C00003C00003C00003C0000380000780000780000780000780000780000 780000780000780000700000F00000F00000F00000F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000F00000700000780000780000780000780000780000 7800007800007800003800003C00003C00003C00003C00001C00001C00001E00001E0000 1E00000E00000E00000F000007000007000007800003800003800003800001C00001C000 01E00000E00000E000007000007000003800003800001C00000C00000E00000600000700 0003800001800000C00000E0000060000030000018157C768121>32 DI<00001C00003C0000F80001E00003C000 0780000F00000E00001E00003C00003C00003C0000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000F00000F00000F00001 E00001E00003C0000380000700000E00001C0000780000E00000E000007800001C00000E 000007000003800003C00001E00001E00000F00000F00000F00000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 7800007800007800007800007800007800007800007800007800007800007800003C0000 3C00003C00001E00000E00000F000007800003C00001E00000F800003C00001C167C7B81 21>40 D<0018007800F001E003C007800F001F001E003E003C007C007C007800F800F800 F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800 F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C 007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C 00F800F800F800F001F001E003E003C0078007000E001C003800F000C000F00038001C00 0E000700078003C003E001E001F000F000F800F800F8007C007C007C007C007C007C007C 007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C0E4D 798025>60 D62 D80 D89 D E /Ff 1 1 df0 D E /Fg 2 111 df<3E0006000C000C000C000C001800187018B819383230340038003E00630063106310 6310C320C1C00D147E9312>107 D<30F8590C4E0C9C0C980C180C180C30183019303130 316032601C100D7F8C15>110 D E /Fh 2 50 df<0F0030C0606060604020C030C030C0 30C030C030C030C030C030C03040206060606030C00F000C137E9211>48 D<0C001C00EC000C000C000C000C000C000C000C000C000C000C000C000C000C000C000C 00FFC00A137D9211>I E /Fi 8 116 df<60F0F070101020204040040A7D830A>59 D<003E000C000C000C000C0018001800180018073018F0307060706060C060C060C06080 C080C480C4C1C446C838700F177E9612>100 D<03000380030000000000000000000000 00001C002400460046008C000C0018001800180031003100320032001C0009177F960C> 105 D<00180038001000000000000000000000000001C002200430043008600060006000 6000C000C000C000C001800180018001806300E300C60078000D1D80960E>I<383C1E00 44C6630047028100460301008E0703000C0603000C0603000C060600180C0600180C0620 180C0C20180C0C4030180440301807801B0E7F8D1F>109 D<383C0044C6004702004602 008E06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15 >I<38F04518463846308C000C000C000C001800180018001800300030000D0E7F8D10> 114 D<07C00C201870187038001E000FC003E000606060E060C0C0C1803F000C0E7E8D10 >I E /Fj 3 55 df0 D<020002000200C218F2783AE00F80 0F803AE0F278C2180200020002000D0E7E8E12>3 D<0003000300060006000C000C0018 0018003000300060006000C000C00180018003000300060006000C000C00180018003000 300060006000C0004000101E7B9600>54 D E /Fk 12 122 df<01F8071C0C1E181C3818 3000700070007000E000E000E000600060047008301018200FC00F127D9112>99 D<01F8070C0C061C073803300370037FFF7000E000E000E00060006002300430081C3007 C010127E9112>101 D<00C001C001C0018000000000000000000000000000001F800780 03800700070007000700070007000E000E000E000E000E000E001C001E00FF800A1D7F9C 0C>105 D<1F8F8007B0C003C0E00780E00780E00700E00700E00700E00700E00E01C00E 01C00E01C00E01C00E01C00E01C01C03801E03C0FF9FF014127F9117>110 D<00FC000307000E01801C01C03800C03000C07000E07000E07000E0E001C0E001C0E001 C0600180600380700700380E001C180007E00013127E9115>I<0FC78003D86001E03003 C03803803803801803801C03801C03801C0700380700380700380700700700700700E00F 01C00EC3000E3C000E00000E00000E00001C00001C00001C00001C0000FF8000161A8091 17>I<00F0400308C00E05C01C0380380380380380700380700380700380E00700E00700 E00700600700700700700F00301E00186E00078E00000E00000E00000E00001C00001C00 001C00001C0000FF80121A7D9116>I<1F9C07EE03CF078E078C07000700070007000E00 0E000E000E000E000E001C001E00FFC010127F9110>I<03F20C0E18061004300438043E 001FE00FF007F8003C401C400C400C6018E010D0608FC00F127F9110>I<020002000200 060006000C001C003C00FFE01C001C003800380038003800380038007000704070407040 70407080708031001E000B1A7C9910>II<0FF0FE03C03801C03001C02001C06001C04001E08000E08000E10000E100 00E200007200007400007C00007800007000003000002000002000004000004000708000 F10000F10000E60000780000171A809116>121 D E /Fl 5 120 df<3078F06005047C830D>46 D<01E007100C1018083810701070607F80E000E000E000 E000E000E0086010602030C01F000D127B9113>101 D<01800380010000000000000000 000000000000001C002600470047008E008E000E001C001C001C00380038007100710071 00720072003C00091C7C9B0D>105 D<3C3C002646004687004707008E07008E07000E07 000E07001C0E001C0E001C0E001C1C00381C40381C40383840383880701900300E001212 7C9117>110 D<1E01832703874703874703838707018707010E07010E07011C0E021C0E 021C0E021C0E04180C04181C04181C081C1C100C263007C3C018127C911C>119 D E /Fm 6 111 df<60F0F06004047C830C>58 D<60F0F0701010101020204080040C7C 830C>I<0000038000000F0000003C000000F0000003C000000F0000003C000000F00000 03C000000F0000003C000000F0000000F00000003C0000000F00000003C0000000F00000 003C0000000F00000003C0000000F00000003C0000000F000000038019187D9520>I62 D<3F00070007000E000E000E000E001C001C001C001C0039E03A303C1838187018701C70 1C701CE038E038E038E030E070E060E0C061C023001E000E1D7E9C12>98 D<381F004E61804681C04701C08F01C08E01C00E01C00E01C01C03801C03801C03801C07 00380710380710380E10380E2070064030038014127E9119>110 D E /Fn 43 122 df<387C7E7E3E0E1E1C78F060070B798416>44 D<70F8F8F8700505788416>46 D<70F8F8F870000000000000000070F8F8F87005127891 16>58 D<00E00001F00001F00001B00001B00003B80003B80003B800031800071C00071C 00071C00071C00071C000E0E000E0E000FFE000FFE001FFF001C07001C07001C07007F1F C0FF1FE07F1FC013197F9816>65 D<7FF800FFFE007FFF001C0F001C07801C03801C0380 1C03801C07801C07001FFF001FFE001FFE001C1F001C03801C03C01C01C01C01C01C01C0 1C01C01C03C01C07807FFF80FFFF007FFC0012197F9816>I<01F18007FB800FFF801F0F 803C0780380380700380700380F00000E00000E00000E00000E00000E00000E00000E000 00F000007003807003803803803C07001F0F000FFE0007FC0001F00011197E9816>I<7F F800FFFE007FFF001C0F001C07801C03C01C01C01C01C01C01E01C00E01C00E01C00E01C 00E01C00E01C00E01C00E01C00E01C01C01C01C01C03C01C07801C0F807FFF00FFFE007F F8001319809816>I<7FFFC0FFFFC07FFFC01C01C01C01C01C01C01C01C01C00001C0000 1C1C001C1C001FFC001FFC001FFC001C1C001C1C001C00001C00E01C00E01C00E01C00E0 1C00E07FFFE0FFFFE07FFFE013197F9816>II<03E30007FF000F FF001E1F003C0F00380700700700700700F00000E00000E00000E00000E00000E03F80E0 7FC0E03F80F00700700700700700380F003C0F001E1F000FFF0007F70003E70012197E98 16>I73 D76 DI<7E1FC0FF3FE07F1FC01D07001D87001D87001D87001DC7001D C7001CC7001CC7001CE7001CE7001CE7001C67001C67001C77001C77001C37001C37001C 37001C17007F1F00FF9F007F0F0013197F9816>I<1FFC003FFE007FFF00780F00F00780 E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380 E00380E00380F00780F00780780F007FFF003FFE001FFC0011197E9816>I<7FF800FFFE 007FFF001C0F801C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE 001FF8001C00001C00001C00001C00001C00001C00001C00007F0000FF80007F00001219 7F9816>I<7FE000FFF8007FFC001C1E001C0F001C07001C07001C07001C07001C0F001C 1E001FFC001FF8001FFC001C1C001C0E001C0E001C0E001C0E001C0E201C0E701C0E707F 07E0FF87E07F03C014197F9816>82 D<7FFFE0FFFFE0FFFFE0E0E0E0E0E0E0E0E0E0E0E0 E000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E00007FC000FFE0007FC0013197F9816>84 D<7F07F0FF8FF87F07F0 1C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C0 1C01C01C01C01C01C01C01C00E03800E038007070007FF0003FE0000F8001519809816> I<7F1F807F3F807F1F800E1E000E1C00073C0007380003B80003F00001F00001E00000E0 0001E00001F00003F00003B80007B800071C00071C000E0E000E0E001C07007F1FC0FF1F E07F1FC013197F9816>88 D91 D93 D<1FE0003FF0007FF800783C00300E00000E00000E0003FE00 1FFE003E0E00700E00E00E00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116> 97 D<7E0000FE00007E00000E00000E00000E00000E00000E3E000EFF000FFF800F83C0 0F00E00E00E00E00700E00700E00700E00700E00700E00700E00E00F01E00F83C00FFF80 0EFF00063C001419809816>I<03F80FFC1FFE3C1E780C7000E000E000E000E000E000F0 00700778073E0E1FFC0FF803F010127D9116>I<003F00007F00003F0000070000070000 070000070003C7000FF7001FFF003C1F00780F00700700E00700E00700E00700E00700E0 0700E00700700F00700F003C1F001FFFE00FE7F007C7E014197F9816>I<03E00FF81FFC 3C1E780E7007E007FFFFFFFFFFFFE000E000700778073C0F1FFE0FFC03F010127D9116> I<03E3C007F7E00FFFE01C1CC0380E00380E00380E00380E00380E001C1C000FF8001FF0 001BE0003800001800001FFC001FFF003FFF807803C0E000E0E000E0E000E0E000E07001 C07C07C03FFF800FFE0003F800131C7F9116>103 D<7E0000FE00007E00000E00000E00 000E00000E00000E3C000EFE000FFF000F87800F03800E03800E03800E03800E03800E03 800E03800E03800E03800E03800E03807FC7F0FFE7F87FC7F01519809816>I<018003C0 03C0018000000000000000007FC07FC07FC001C001C001C001C001C001C001C001C001C0 01C001C001C07FFFFFFF7FFF101A7D9916>I<003000780078003000000000000000001F F81FF81FF800380038003800380038003800380038003800380038003800380038003800 380038003800386070F0F0FFE07FC03F800D237E9916>I108 DI<7E3C00FEFE007FFF000F 87800F03800E03800E03800E03800E03800E03800E03800E03800E03800E03800E03807F C7F0FFE7F87FC7F01512809116>I<03E0000FF8001FFC003C1E00780F00700700E00380 E00380E00380E00380E00380F00780700700780F003C1E001FFC000FF80003E00011127E 9116>I<7E3E00FEFF007FFF800F83C00F00E00E00E00E00700E00700E00700E00700E00 700E00700E00E00F01E00F83C00FFF800EFF000E3C000E00000E00000E00000E00000E00 000E00007FC000FFE0007FC000141B809116>I114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007F8003C600EE0 0EF00EF81EFFFCFFF8C7E00F127D9116>I<0300000700000700000700000700007FFF00 FFFF00FFFF00070000070000070000070000070000070000070000070100070380070380 07038007870003FE0001FC0000F80011177F9616>I<7E1F80FE3F807E1F800E03800E03 800E03800E03800E03800E03800E03800E03800E03800E03800E03800E0F800FFFF007FB F803E3F01512809116>I<7F1FC0FF1FE07F1FC01C07001E0F000E0E000E0E000E0E0007 1C00071C00071C00071C0003B80003B80003B80001F00001F00000E00013127F9116>I< 7F1FC07F3FC07F1FC00F1C00073C0003B80003F00001F00000E00001E00001F00003B800 073C00071C000E0E007F1FC0FF3FE07F1FC013127F9116>120 D<7F1FC0FF9FE07F1FC0 1C07000E07000E0E000E0E00070E00071C00071C00039C00039C0003980001B80001B800 00F00000F00000F00000E00000E00000E00001C00079C0007BC0007F80003F00003C0000 131B7F9116>I E /Fo 74 123 df<007F07F001FF1FF003FF3FF007807800070070000F 00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000FFF8FF80FFF8FF80FF F8FF800F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F 00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0001C 20809F1B>11 D<0000F000F8F001F8F003F8F00780000700000F00000F00000F00000F00 000F00000F00000F0000FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00 F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01421 80A018>I<00F8F001F8F003F8F00780F00700F00F00F00F00F00F00F00F00F00F00F00F 00F00F00F0FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F 00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01420809F18>I< 783C783C783C783C180C3018301860306030E0700E0A7F9F17>34 D<787878781830306060E0050A7D9F0D>39 D<007000E001C00380078007000E001E001E 003C003C003C0078007800780078007000F000F000F000F000F000F000F000F000F000F0 00F000F000700078007800780078003C003C003C001E001E000E0007000780038001C000 E000700C2E7EA112>I I<018001C001800180C183E187F99F7DBE1FF807E007E01FF87DBEF99FE187C183018001 8001C0018010147DA117>I<000600000006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 FFFFFFF0FFFFFFF000060000000600000006000000060000000600000006000000060000 00060000000600000006000000060000000600000006000000060000000600001C207D9A 23>I<787878781830306060E0050A7D830D>III<000100030003000600060006000C000C000C0018001800180030003000 3000600060006000C000C000C00180018001800300030003000600060006000C000C000C 00180018001800300030003000600060006000C000C000C000102D7DA117>I<03F00007 F8001FFE001E1E003C0F00380700780780780780700380F003C0F003C0F003C0F003C0F0 03C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C078078078078078 07803807003C0F001E1E001FFE000FFC0003F00012207E9E17>I<00C001C00FC0FFC0FF C0F3C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C0FFFEFFFEFFFE0F1F7C9E17>I<07F0000FFC001FFE00383F00 700F00600780E00780E003C04003C04003C00003C00003C00003C0000780000780000F00 000E00001C00003C0000780000E00001C0000380000700000E00001C0000380000700000 FFFFC0FFFFC0FFFFC0121F7E9E17>I<03F0000FFC001FFE003C1F00780F003007802007 80000780000780000780000F00000F00003E0003FC0003F80003FC00001E00000F000007 800003800003C00003C00003C00003C08003C0C003C0C00780700F807C1F003FFE000FFC 0003F00012207E9E17>I<003E00003E00005E00005E0000DE0001DE00019E00039E0003 9E00079E00071E000F1E000E1E001E1E003C1E003C1E00781E00781E00F01E00FFFFF0FF FFF0FFFFF0001E00001E00001E00001E00001E00001E00001E00001E00141E7F9D17>I< F0F0F0F0000000000000000000000000F0F0F0F004147C930D>58 D61 D<001F0000001F0000003F8000003B80 00003B8000007BC0000073C0000071C00000F1E00000E1E00000E0E00001E0F00001E0F0 0001C0F00003C0780003C078000380780007803C0007803C0007003C000FFFFE000FFFFE 000FFFFE001E000F001E000F003C000F803C0007803C000780780007C0780003C0780003 C0F00003E01B207F9F1E>65 DI<001FC000FFF801FFFC03E03C07800C0F00001E00003E00003C00007C 0000780000780000780000F00000F00000F00000F00000F00000F00000F00000F0000078 00007800007800007C00003C00003E00001E00000F000207800E03E03E01FFFC00FFF000 1FC017227DA01D>IIII<001FE000FFF8 01FFFE03E03E07800E0F00001E00003E00003C00007C0000780000780000780000F00000 F00000F00000F00000F00000F00000F003FEF003FE7803FE78001E78001E7C001E3C001E 3E001E1E001E0F001E07801E03E03E01FFFE00FFF8001FC017227DA01E>III75 DIII<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C00 0F003C000F00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F000 03C0F00003C0F00003C0F00003C0F00003C0F80007C07800078078000780780007803C00 0F003C000F001E001E001F003E000F807C0007E1F80003FFF00000FFC000003F00001A22 7DA021>II< 003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F00 780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C07800078078000780781E07803C0F0F003C0F8F00 1E079E001F03FE000F83FC0007E1F80003FFF00000FFF800003F780000007C0000003E00 00001E0000001F0000000F801A277DA021>II<01FC0007FF800FFFC01F03C03C00C03C000078000078 00007800007800007800007C00003C00003F00001FE0000FFC0007FE0001FF00003F8000 07C00003C00003E00001E00001E00001E00001E00001E00001C0C003C0F007C0FC0F807F FF001FFE0003F80013227EA019>IIIII<780007807C000F003E001F001E001E000F00 3C000F807C000780780003C0F00003E1F00001F1E00000F3C000007FC000007F8000003F 0000001F0000001E0000003F0000007F8000007FC00000F3C00001F1E00001E0F00003C0 F80007C0780007803C000F003E001F001E001E000F003C000F807C0007C0780003C0F000 03E01B207F9F1E>II< FFFFF8FFFFF8FFFFF80000F00001F00001E00003C00007C0000780000F80000F00001E00 003E00003C00007C0000780000F00001F00001E00003E00003C0000780000F80000F0000 1F00001E00003C00007C0000780000FFFFFCFFFFFCFFFFFC16207D9F1C>II<381C3018301860306030C060F078F078F078F0780E0A799F17> II<03C007E00FF01E781C38381C700EE00710087D 9F17>I<07E03FF87FFC701E401F000F000F000F003F07FF1FFF7E0FF80FF00FF00FF00F F83F7FFF3FEF1F8F10147E9316>97 DI<03F00FFC1FFE3E0E3C0278007800F000F000F000F000F000F0 00780078003C013E0F1FFF0FFE03F010147E9314>I<0007800007800007800007800007 8000078000078000078000078000078000078000078007C7800FF7801FFF803E1F807C07 80780780F80780F00780F00780F00780F00780F00780F00780F00780780780780F803E1F 801FFF800FF78007C78011207E9F17>I<03F0000FFC001FFE003E1F003C070078070070 0380FFFF80FFFF80FFFF80F00000F00000F000007000007800003C01003E07001FFF0007 FE0001F80011147F9314>I<007E01FE03FE078007000F000F000F000F000F000F000F00 FFF0FFF0FFF00F000F000F000F000F000F000F000F000F000F000F000F000F000F000F00 0F000F000F20809F0E>I<03E0F00FFFF01FFFF03E3E003C1E00780F00780F00780F0078 0F00780F003C1E003E3E001FFC003FF80033E0003000003800003FFE003FFF801FFFC03F FFE07803F0F000F0F000F0F000F0F801F07E07E03FFFC00FFF0003FC00141E7F9317>I< F000F000F000F000F000F000F000F000F000F000F000F000F1F8F3FCF7FEFC1FF80FF80F F00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00FF00F10207D9F17>II<01E001E001E001E00000000000000000000000000000000001E001E001E001E001E0 01E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E0 01E001E0C3C0FFC0FF803F000B29839F0C>IIIII<01F80007FE001FFF803F0FC03C03C07801E07801E0F000F0F000F0F000 F0F000F0F000F0F000F07801E07801E03C03C03F0FC01FFF8007FE0001F80014147F9317 >II114 D<07F01FFC3FFC780C7800780078007C003FC01FF00FF803F8007C003C003CC03CF07CFF F87FF00FC00E147F9311>I<1E001E001E001E001E001E00FFF0FFF0FFF01E001E001E00 1E001E001E001E001E001E001E001E001E001E001E201FF00FF007C00C1A7F9910>IIII<7801E07C03C03E07801E0F000F0F0007 9E0003FC0003F80001F80000F00001F00001F80003FC00079E000F0F000E0F001E07803C 03C07801E0F801F01414809315>I I<7FFF7FFF7FFF003E003C007800F800F001E003E007C007800F001F001E003C007C00FF FFFFFFFFFF10147F9314>I E /Fp 88 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8 F870000000000070F8F8F870051C779B18>33 D<4010E038F078E038E038E038E038E038 E038E038E038E038E03860300D0E7B9C18>I<030600078F00078F00078F00078F00078F 00078F007FFFC0FFFFE0FFFFE07FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFF C0FFFFE0FFFFE07FFFC01E3C001E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18 >I<3803007C07807C0780EE0F80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C007C 3C00387C0000780000780000F80000F00001F00001E00001E00003E00003C00003C00007 C0000783800787C00F87C00F0EE00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07C018 038013247E9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C38001C73 F01C73F01CE3F00FE3800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00E0F8 00E0F800E07070E0787070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C7C7E3E0E0E 0E1C1C38F8F0C0070E789B18>I<007000F001E003C007800F001E001C00380038007000 700070007000E000E000E000E000E000E000E000E0007000700070007000380038001C00 1E000F00078003C001F000F000700C24799F18>I<6000F00078003C001E000F00078003 8001C001C000E000E000E000E00070007000700070007000700070007000E000E000E000 E001C001C0038007800F001E003C007800F00060000C247C9F18>I<01C00001C00001C0 0001C000C1C180F1C780F9CF807FFF001FFC0007F00007F0001FFC007FFF00F9CF80F1C7 80C1C18001C00001C00001C00001C00011147D9718>I<00600000F00000F00000F00000 F00000F00000F00000F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F00000F00000 F00000F00000F00000600013147E9718>I<1C3E7E7F3F1F070E1E7CF860080C788518>I< 7FFF00FFFF80FFFF807FFF0011047D8F18>I<3078FCFC78300606778518>I<0003000007 80000780000F80000F00001F00001E00001E00003E00003C00007C0000780000780000F8 0000F00001F00001E00003E00003C00003C00007C0000780000F80000F00000F00001F00 001E00003E00003C00003C00007C0000780000F80000F00000F0000060000011247D9F18 >I<01F00007FC000FFE001F1F001C07003803807803C07001C07001C0E000E0E000E0E0 00E0E000E0E000E0E000E0E000E0E000E0E000E0F001E07001C07001C07803C03803801C 07001F1F000FFE0007FC0001F000131C7E9B18>I<01800380038007800F803F80FF80FB 80438003800380038003800380038003800380038003800380038003800380038003807F FCFFFE7FFC0F1C7B9B18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0F000E0 6000E00000E00000E00001C00001C00003C0000780000F00001E00003C0000780000F000 01E00007C0000F80001E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8001FFE 003FFF007807807803C07801C03001C00001C00003C0000380000F0003FF0003FE0003FF 000007800003C00001C00000E00000E00000E0F000E0F000E0F001C0F003C07C07803FFF 001FFE0003F800131C7E9B18>I<001F00003F0000770000770000E70001E70001C70003 87000787000707000E07001E07003C0700380700780700F00700FFFFF8FFFFF8FFFFF800 0700000700000700000700000700000700007FF000FFF8007FF0151C7F9B18>I<1FFF80 3FFF803FFF803800003800003800003800003800003800003800003800003BF8003FFE00 3FFF003C07801803C00001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F80 3FFF001FFC0003F000131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03C03801 80380000700000700000E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000E0F000 E07000E07000E07000E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18>II<03F8000FFE001FFF003E0F80380380 7001C07001C07001C07001C03803803C07801FFF0007FC000FFE001F1F003C07807001C0 F001E0E000E0E000E0E000E0E000E07001C07803C03E0F801FFF000FFE0003F800131C7E 9B18>I<03F0000FFC001FFE003C0F00780780700380E001C0E001C0E001C0E001E0E001 E07001E07803E03C0FE01FFFE00FFEE003F0E00000E00001C00001C00001C03003807807 80780F00783E003FFC001FF00007C000131C7E9B18>I<3078FCFC783000000000000000 003078FCFC78300614779318>I<183C7E7E3C180000000000000000183C7E7E3E1E0E1C 3C78F060071A789318>I<000300000780001F80003F00007E0001FC0003F00007E0001F C0003F00007E0000FC0000FC00007E00003F00001FC00007E00003F00001FC00007E0000 3F00001F8000078000030011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE0000000000000 000000000000FFFFE0FFFFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00007E00 003F00001FC00007E00003F00001FC00007E00003F00001F80001F80003F00007E0001FC 0003F00007E0001FC0003F00007E0000FC0000F0000060000011187D9918>I<00700000 F80000F80000D80000D80001DC0001DC0001DC00018C00038E00038E00038E00038E0003 06000707000707000707000707000FFF800FFF800FFF800E03800E03801C01C01C01C07F 07F0FF8FF87F07F0151C7F9B18>65 DI<00 F8E003FEE007FFE00F07E01E03E03C01E03800E07000E07000E0700000E00000E00000E0 0000E00000E00000E00000E00000E000007000007000E07000E03800E03C00E01E01C00F 07C007FF8003FE0000F800131C7E9B18>I<7FF800FFFE007FFF001C0F801C03C01C03C0 1C01E01C00E01C00E01C00F01C00701C00701C00701C00701C00701C00701C00701C0070 1C00F01C00E01C00E01C01E01C01C01C03C01C0F807FFF00FFFE007FF800141C7F9B18> III<01F1C003FDC00FFFC01F0FC01C03C03803C03801C07001C07001C0700000E00000 E00000E00000E00000E00000E00FF0E01FF0E00FF07001C07001C07003C03803C03803C0 1C07C01F0FC00FFFC003FDC001F1C0141C7E9B18>I<7F07F0FF8FF87F07F01C01C01C01 C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01FFFC01FFFC01FFFC01C01C01C01 C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C07F07F0FF8FF87F07F0151C 7F9B18>I<7FFF00FFFF807FFF0001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C0007FFF00FFFF807FFF00111C7D9B18>I<7F07F0FF87F87F07F01C03C0 1C07801C07001C0E001C1E001C3C001C38001C70001CF0001DF0001DF0001FB8001FB800 1F1C001E1C001C0E001C0E001C07001C07001C03801C03801C01C07F03F0FF87F87F03F0 151C7F9B18>75 D<7FE000FFE0007FE0000E00000E00000E00000E00000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00700E 00700E00700E00700E00707FFFF0FFFFF07FFFF0141C7F9B18>II<7E07F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C01CC1 C01CE1C01CE1C01CE1C01C61C01C71C01C71C01C31C01C39C01C39C01C39C01C19C01C19 C01C1DC01C0DC01C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18>I<0FF8003FFE007F FF00780F00700700F00780E00380E00380E00380E00380E00380E00380E00380E00380E0 0380E00380E00380E00380E00380E00380E00380E00380F00780700700780F007FFF003F FE000FF800111C7D9B18>II<0FF8003FFE 007FFF00780F00700700F00780E00380E00380E00380E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380E00380E1E380E1E380F0E78070F700787F007FFF 003FFE000FFC00001C00001E00000E00000F0000070000070011227D9B18>I<7FF800FF FE007FFF001C0F801C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001F FE001FFE001C0F001C07001C03801C03801C03801C03801C03801C039C1C039C1C039C7F 01F8FF81F87F00F0161C7F9B18>I<03F3801FFF803FFF807C0F80700780E00380E00380 E00380E000007000007800003F00001FF00007FE0000FF00000F800003C00001C00000E0 0000E06000E0E000E0E001E0F001C0F80780FFFF80FFFE00E7F800131C7E9B18>I<7FFF F8FFFFF8FFFFF8E07038E07038E07038E070380070000070000070000070000070000070 000070000070000070000070000070000070000070000070000070000070000070000070 0007FF0007FF0007FF00151C7F9B18>II< FF07F8FF07F8FF07F81C01C01C01C01C01C01C01C00E03800E03800E03800E03800F0780 070700070700070700070700038E00038E00038E00038E00018C0001DC0001DC0001DC00 00D80000F80000F800007000151C7F9B18>II<7F8FE07F9FE07F8FE00E07000F0700070E00078E00039C0003DC0001F80001F80000 F00000F00000700000F00000F80001F80001DC00039E00038E00070F000707000E07800E 03801E03C07F07F0FF8FF87F07F0151C7F9B18>II<3FFFE07FFFE07FFFE07001C07003C0700780700700000F00001E00001C00003C 0000780000700000F00001E00001C00003C0000780000700000F00001E00E01C00E03C00 E07800E07000E0FFFFE0FFFFE0FFFFE0131C7E9B18>II<600000F00000 F00000F800007800007C00003C00003C00003E00001E00001F00000F00000F00000F8000 07800007C00003C00003C00003E00001E00001F00000F00000F800007800007800007C00 003C00003E00001E00001E00001F00000F00000F8000078000078000030011247D9F18> II<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C38070E789E18>I<1FE0003FF8007FFC00781E00300E 0000070000070000FF0007FF001FFF007F0700780700E00700E00700E00700F00F00781F 003FFFF01FFBF007E1F014147D9318>I<7E0000FE00007E00000E00000E00000E00000E 00000E00000E3E000EFF800FFFC00FC1E00F80E00F00700E00700E00380E00380E00380E 00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF80063E00151C809B18>I< 01FE0007FF001FFF803E0780380300700000700000E00000E00000E00000E00000E00000 E000007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80003F 80001F8000038000038000038000038000038003E3800FFB801FFF803C1F80380F807007 80700380E00380E00380E00380E00380E00380E00380700780700780380F803C1F801FFF F00FFBF803E3F0151C7E9B18>I<01F00007FC001FFE003E0F00380780700380700380E0 01C0E001C0FFFFC0FFFFC0FFFFC0E000007000007001C03801C03E03C01FFF8007FF0001 FC0012147D9318>I<001F80007FC000FFE000E1E001C0C001C00001C00001C0007FFFC0 FFFFC0FFFFC001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C0007FFF007FFF007FFF00131C7F9B18>I<01E1F007FFF80FFF F81E1E301C0E003807003807003807003807003807001C0E001E1E001FFC001FF80039E0 003800001C00001FFE001FFFC03FFFE07801F0700070E00038E00038E00038E000387800 F07E03F01FFFC00FFF8001FC00151F7F9318>I<7E0000FE00007E00000E00000E00000E 00000E00000E00000E3E000EFF800FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B 18>I<03800007C00007C00007C0000380000000000000000000000000007FC000FFC000 7FC00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C000FFFF00FFFF80FFFF00111D7C9C18>I<0038007C007C007C003800 000000000000000FFC1FFC0FFC001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C6038F078FFF07FE03F800E277E9C18 >II<7FE000FFE0007FE00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E0007FFFC0FFFFE07FFFC0131C7E 9B18>I<7CE0E000FFFBF8007FFFF8001F1F1C001E1E1C001E1E1C001C1C1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 7F1F1F00FFBFBF807F1F1F001914819318>I<7E3E00FEFF807FFFC00FC1C00F80E00F00 E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3 FCFFE7FE7FC3FC1714809318>I<01F0000FFE001FFF003E0F803803807001C07001C0E0 00E0E000E0E000E0E000E0E000E0F001E07001C07803C03C07803E0F801FFF000FFE0001 F00013147E9318>I<7E3E00FEFF807FFFC00FC1E00F80E00F00700E00700E00380E0038 0E00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF800E3E000E0000 0E00000E00000E00000E00000E00000E00007FC000FFE0007FC000151E809318>I<01E3 8007FB801FFF803E1F80380F80700780700780E00380E00380E00380E00380E00380E003 80700780700780380F803C1F801FFF800FFB8003E3800003800003800003800003800003 80000380000380003FF8003FF8003FF8151E7E9318>I<7F87E0FF9FF07FBFF803F87803 F03003E00003C00003C00003800003800003800003800003800003800003800003800003 80007FFE00FFFF007FFE0015147F9318>I<07F7003FFF007FFF00780F00E00700E00700 E007007C00007FE0001FFC0003FE00001F00600780E00380E00380F00380F80F00FFFF00 FFFC00E7F00011147D9318>I<0180000380000380000380000380007FFFC0FFFFC0FFFF C00380000380000380000380000380000380000380000380000380000380400380E00380 E00380E001C1C001FFC000FF80003E0013197F9818>I<7E07E0FE0FE07E07E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E01E00F 03E007FFFC03FFFE01FCFC1714809318>I<7F8FF0FF8FF87F8FF01E03C00E03800E0380 0E0380070700070700070700038E00038E00038E00038E0001DC0001DC0001DC0000F800 00F80000700015147F9318>II<7F8FF07F9FF07F8FF0070700078E00039E0001DC0001F80000F80000 700000F00000F80001DC00039E00038E000707000F07807F8FF0FF8FF87F8FF015147F93 18>I<7F8FF0FF8FF87F8FF00E01C00E03800E0380070380070700070700038700038600 038E0001CE0001CE0000CC0000CC0000DC00007800007800007800007000007000007000 00F00000E00079E0007BC0007F80003F00001E0000151E7F9318>I<3FFFF07FFFF07FFF F07001E07003C0700780000F00001E00003C0000F80001F00003C0000780000F00701E00 703C0070780070FFFFF0FFFFF0FFFFF014147F9318>I<0007E0001FE0007FE000780000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00001E0007F C000FF8000FF80007FC00001E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E000007800007FE0001FE00007E013247E9F18>I<60F0F0F0F0F0 F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0600424769F18> I<7C0000FF0000FFC00003C00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000F000007FC0003FE0003FE0007FC000F00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00003C000FFC000FF00007C00 0013247E9F18>I E /Fq 61 124 df<000078007C7800FC7801FC7803C0000380000780 00078000078000078000078000078000078000078000FFFC78FFFC78FFFC780780780780 780780780780780780780780780780780780780780780780780780780780780780780780 78078078078078078078078078078078152480A31A>12 D<0000C018000000C018000000 C01800000180300000018030000001803000000180300000030060000003006000000300 600000030060000003006000000600C000000600C000000600C000000600C000000C0180 00FFFFFFFFC0FFFFFFFFC000180300000018030000001803000000180300000030060000 003006000000300600000030060000FFFFFFFFC0FFFFFFFFC000600C000000C018000000 C018000000C018000000C018000001803000000180300000018030000001803000000300 6000000300600000030060000003006000000600C000000600C000000600C00000222D7D A229>35 D<007000E001C003C0078007000F000E001E001C003C003C0038007800780078 0078007000F000F000F000F000F000F000F000F000F000F000F000F000F000F000700078 0078007800780038003C003C001C001E000E000F000700078003C001C000E000700C327D A413>40 DI<787878781830306060E0050A7C830E>44 DII<01F00007FC000FFE001F1F003C07803C07807803C07803C07803C0 7001C0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0 F001E0F001E0F001E0F001E07803C07803C07803C07803C03C07803C07801F1F000FFE00 07FC0001F00013237EA118>48 D<00C001C007C0FFC0FFC0FBC003C003C003C003C003C0 03C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C0 03C003C0FFFFFFFFFFFF10227CA118>I<03F0000FFC001FFE003C1F003007807007C060 03C0E003E0C001E04001E04001E00001E00001E00001E00003C00003C000078000078000 0F00001E00003C0000780000F00001E00001C0000380000700000E00001C000038000070 0000FFFFE0FFFFE0FFFFE013227EA118>I<01F00007FC001FFF003E0F003807807003C0 2003C02003C00003C00003C00003C0000780000780000F00001E0003FC0003F80003FE00 000F000007800003C00003C00001E00001E00001E00001E00001E08001E0C003C0E003C0 7007803C0F801FFF000FFC0003F00013237EA118>I<001F00001F00002F00002F00006F 0000EF0000CF0001CF0001CF00038F00038F00078F00070F000F0F000E0F001E0F003C0F 003C0F00780F00780F00F00F00FFFFF8FFFFF8FFFFF8000F00000F00000F00000F00000F 00000F00000F00000F00000F0015217FA018>I<3FFF803FFF803FFF803C00003C00003C 00003C00003C00003C00003C00003C00003C00003CF8003FFE003FFF003F0F803E07803C 03C03803C00001E00001E00001E00001E00001E00001E00001E04003C04003C0E003C070 07807C1F003FFE000FFC0003F00013227EA018>I<007E0001FF0003FF0007C1000F0000 1E00001E00003C00003C000078000078000078F800F1FE00F7FF00FF0F80FC0780F803C0 F803C0F801C0F001E0F001E0F001E0F001E0F001E07001E07001E07801E07803C03803C0 3C03801C07801F0F000FFE0007FC0001F00013237EA118>II<01F00007FC000FFE001E0F003C 07803C07807803C07803C07803C07803C07803C03803803C07801E0F000F1E0007FC0003 F8000FFE001E0F003C07807803C07803C0F001E0F001E0F001E0F001E0F001E0F001E078 03C07803C03C07803E0F801FFF0007FC0001F00013237EA118>I<01F00007FC000FFE00 1E0F003C07003807807803807803C0F003C0F001C0F001E0F001E0F001E0F001E0F001E0 F001E07003E07803E07803E03C07E03E1FE01FFDE00FF1E003E3C00003C00003C0000380 000780000700000F00101E00183C003FF8001FF00007C00013237EA118>I<001F000000 1F0000003F8000003F8000003B8000007BC0000073C0000071C00000F1E00000F1E00000 E0E00001E0F00001E0F00001C0F00003C0780003C078000380780007803C0007803C0007 003C000F001E000F001E000FFFFE001FFFFF001FFFFF001C000F003C0007803C00078038 000780780003C0780003C0700003C0F00001E0F00001E0E00001E01B237EA220>65 DI<000FF000007FFC0000FFFF0001F01F0003C00700078000000F0000001E0000 003E0000003C0000003C000000780000007800000078000000F0000000F0000000F00000 00F0000000F0000000F0000000F0000000F0000000F00000007800000078000000780000 003C0000003C0000003E0000001E0000000F0000000780008003C0038001F00F8000FFFF 00007FFC00000FF00019257DA31F>IIII<000FF000003FFE0000 FFFF8001F80F8003E00380078000000F0000001E0000001E0000003C0000003C00000078 0000007800000078000000F0000000F0000000F0000000F0000000F0000000F0000000F0 00FFC0F000FFC0F000FFC0780003C0780003C0780003C03C0003C03C0003C01E0003C01E 0003C00F0003C0078003C003E003C001F807C000FFFFC0003FFF00000FF8001A257DA321 >III76 DII<001FC000007FF00001FFFC00 03F07E0007C01F000F800F801F0007C01E0003C03C0001E03C0001E0780000F0780000F0 780000F070000070F0000078F0000078F0000078F0000078F0000078F0000078F0000078 F0000078F0000078780000F0780000F0780000F07C0001F03C0001E03E0003E01E0003C0 1F0007C00F800F8007C01F0003F07E0001FFFC00007FF000001FC0001D257DA324>II82 D<00FE0003FFC007FFE00F81E01E00603C00003C000078000078000078 00007800007800007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007E000 01F00000F00000F8000078000078000078000078000078000078C000F0E000F0F801E07E 07C03FFF800FFF0001FC0015257EA31B>III<07E01FF83FFC381E 201E000F000F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF00FF83F7FFF3FEF1F8F 10167E9517>97 DI<01FC0007FF000FFF801F03803C0180780000780000700000 F00000F00000F00000F00000F00000F000007800007800007800003C00401F03C00FFFC0 07FF8001FC0012167E9516>I<0003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C003E3C00FFBC01FFFC03F0FC03C07C07803C07803 C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07803C07803C03C07C03E0F C01FFFC00FFBC003E3C012237EA219>I<03F00007FC001FFE003E0F003C078078038078 0380F001C0FFFFC0FFFFC0FFFFC0F00000F00000F000007000007800007800003C00801F 07800FFF8007FF0001F80012167E9516>I<003F00FF01FF03C003800780078007800780 0780078007800780FFF8FFF8FFF807800780078007800780078007800780078007800780 07800780078007800780078007800780102380A20F>I<01F07807FFF80FFFF81F1F001E 0F003C07803C07803C07803C07803C07801E0F001F1F000FFE001FFC0019F00038000038 00003C00001FFE001FFFC01FFFE03FFFF07801F07800F8F00078F00078F00078F0007878 00F03E03E01FFFC00FFF8001FC0015217F9518>III<00F000F0 00F000F000000000000000000000000000000000000000F000F000F000F000F000F000F0 00F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F0 00F000F080F0E3E0FFE07FC01F000C2D83A20D>IIIII<01FC00 07FF000FFF801F07C03C01E07800F07800F0700070F00078F00078F00078F00078F00078 F000787800F07800F07C01F03E03E01F07C00FFF8007FF0001FC0015167F9518>II<03E3C007FBC00F FFC01F0FC03C07C07C03C07803C0F803C0F003C0F003C0F003C0F003C0F003C0F003C0F8 03C07803C07803C03C07C03F0FC01FFFC00FFBC003E3C00003C00003C00003C00003C000 03C00003C00003C00003C00003C00003C012207E9519>II<07F01FFC3FFE3C0E7806780078007C003F003FF01FF80FFC01FE001F000F000F00 0FC00FF81EFFFE3FFC0FF010167F9513>I<0F000F000F000F000F000F00FFF8FFF8FFF8 0F000F000F000F000F000F000F000F000F000F000F000F000F000F000F080F1C07FC07F8 03E00E1C7F9B12>IIII<7801F07C01E03E03C01E07C00F0780078F0007DE0003FC00 01FC0000F80000700000F80001FC0003DC00039E00078F000F07801E07801E03C03C01E0 7800F0F800F81516809516>II123 D E /Fr 57 123 df<00FC1F8001FC3F8003FC7F800700E0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C000FFE1FC00FFE1FC000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C000191D809C18>11 D<01F1C003F1C007F1C00701C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFF1C0FFF1C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 121D809C16>13 D<00E001C0038007000E000E001C001C00380038003800700070007000 7000E000E000E000E000E000E000E000E000E000E000E000E00070007000700070003800 380038001C001C000E000E000700038001C000E00B2A7E9E10>40 DI<018001C0018001806186F99F7DBE1FF8 07E007E01FF87DBEF99F61860180018001C0018010127E9E15>I<000600000006000000 060000000600000006000000060000000600000006000000060000000600000006000000 06000000060000FFFFFFE0FFFFFFE0000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000000600001B1C7E9720 >III<000100030006000600 06000C000C000C0018001800180030003000300060006000C000C000C001800180018003 0003000300060006000C000C000C00180018001800300030003000600060006000C000C0 0010297E9E15>47 D<07C01FF03878701C601EE00EC00F400F400700070007000F000E00 1E001C003C007800F001E001C0038007000E001C0038007000FFFFFFFF101C7E9B15>50 D<003C00007C00005C0000DC0001DC00019C00039C00039C00071C00071C000E1C000E1C 001C1C003C1C00381C00781C00701C00F01C00FFFFC0FFFFC0001C00001C00001C00001C 00001C00001C00001C00121B7F9A15>52 D<001C0000003E0000003E0000002E00000067 00000067000000E7800000C7800000C3800001C3C0000183C0000181C0000381E0000381 E0000700F0000700F0000600F0000E0078000FFFF8000FFFF8001C003C001C003C001800 3C0038001E0038001E0070001F0070000F0070000F00E0000780191D7F9C1C>65 DI<003FC000FFF003C0F00780300F 00001E00003C00003C0000780000780000780000F00000F00000F00000F00000F00000F0 0000F00000F00000F000007800007800007800003C00003C00001E00000F000807801803 C07800FFF0003F80151F7D9D1B>I III<003F8001FFF003C0F80780380F00181E00003C00003C0000780000 780000780000F00000F00000F00000F00000F00000F00000F007F8F007F8F00038780038 7800387800383C00383C00381E00380F003807803803C0F801FFF0003F80151F7D9D1C> III75 DIII<003F000001FFE00003FFF00007C0F8000F807C001E001E003E001F003C000F00 780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0 F00003C0F00003C0F80007C078000780780007807C000F803C000F003E001F001F003E00 0F807C0007C0F80003FFF00001FFE000003F00001A1F7E9D1F>II82 D<03F8000FFE001C0F00380700700300600000E00000E00000E00000E00000F000007800 007F00003FE0001FFC0007FE0001FF00001F800007800003C00003C00001C00001C00001 C00001C0C00180E00380F007007C0E001FFC0007F000121F7E9D17>IIII<78000E007C001E003C003C001E0038000F0070000F00F000 0781E00003C1C00001C3C00001E7800000F70000007E0000003E0000003C0000003C0000 007E00000077000000E7800001E3800003C1C0000381E0000700F0000F00F8000E007800 1C003C003C003E0078001F0070000F00F0000F80191D7F9C1C>88 DI<0FC03FF07FF87038401C001C001C00FC0FFC3FFC 781CE01CE01CE01CF07C7FFC7FDC3F1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000E0 00E000E000700070043C1C1FFC0FF807E00E127E9112>I<000E000E000E000E000E000E 000E000E000E000E000E0F8E1FEE3FFE7C3E700E700EE00EE00EE00EE00EE00EE00EF00E 701E7C3E3FFE1FEE0F8E0F1D7E9C15>I<07C01FE03FF078787018601CFFFCFFFCFFFCE0 00E000E000700070043C1C3FFC1FF807E00E127E9112>I<00FC01FC03FC07000E000E00 0E000E000E000E000E00FFE0FFE00E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E1D809C0D>I<03C3C00FFFC01FFFC01C3800381C00381C0038 1C00381C00381C001C38001FF8001FF0003BC0003800003800001FFC001FFF003FFF8070 0780E001C0E001C0E001C0F003C07C0F803FFF001FFE0007F800121B7F9115>III107 DIII<03 F0000FFC001FFE003C0F00780780700380E001C0E001C0E001C0E001C0E001C0F003C070 03807807803C0F001FFE000FFC0003F00012127F9115>II114 D<1FC03FF07FF0F030E000E000F0007F00 3FC01FE000F0003800388038F078FFF07FE01FC00D127F9110>I<1C001C001C001C001C 001C00FFE0FFE01C001C001C001C001C001C001C001C001C001C001C001C001C201FF00F F007C00C187F970F>IIII<7003807807 003C0E001C1C000E1C0007380003F00001E00001C00001E00003F0000738000E18000E1C 001C0E00380700700380F003C01212809113>II<7FFC7FFC7FFC007800F000E001E003C0038007000F001E001C003C0078 00FFFCFFFCFFFC0E127F9112>I E /Fs 30 122 df<00FF000387000707000607000E07 000E07000E07000E07000E07000E0700FFFF000E07000E07000E07000E07000E07000E07 000E07000E07000E07000E07000E07000E07000E07000E07007F9FE0131A809915>13 D<60F0F07010101020204080040B7D830B>44 D<60F0F06004047D830B>46 D<0004000C00180018001800300030003000600060006000C000C000C001800180018003 00030003000600060006000C000C000C00180018001800300030003000600060006000C0 00C0000E257E9B13>I<078018603030303060186018E01CE01CE01CE01CE01CE01CE01C E01CE01CE01CE01CE01C6018601870383030186007800E187E9713>I<03000700FF0007 000700070007000700070007000700070007000700070007000700070007000700070007 000700FFF00C187D9713>I<0F8010E02070607870382038007800700070006000C00F80 00E000700038003C003CE03CE03CC03C4038407030E00F800E187E9713>51 D<07801860303070306018E018E018E01CE01CE01C601C603C303C185C0F9C001C001800 18003870307060604021801F000E187E9713>57 D<000C0000000C0000000C0000001E00 00001E0000003F000000270000002700000043800000438000004380000081C0000081C0 000081C0000100E0000100E00001FFE00002007000020070000600780004003800040038 0008001C0008001C001C001E00FF00FFC01A1A7F991D>65 D<1FFC00E000E000E000E000 E000E000E000E000E000E000E000E000E000E000E000E000E000E000E040E0E0E0E0E041 C061801E000E1A7D9914>74 D<7FFFFF00701C0700401C0100401C0100C01C0180801C00 80801C0080801C0080001C0000001C0000001C0000001C0000001C0000001C0000001C00 00001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C0000001C00 00001C000003FFE000191A7F991C>84 D86 D<3F8070C070E020700070007007F01C 7030707070E070E071E071E0F171FB1E3C10107E8F13>97 D<007E00000E00000E00000E 00000E00000E00000E00000E00000E00000E0003CE000C3E00380E00300E00700E00E00E 00E00E00E00E00E00E00E00E00E00E00600E00700E00381E001C2E0007CFC0121A7F9915 >100 D<07C01C3030187018600CE00CFFFCE000E000E000E0006000300438080C1807E0 0E107F8F11>I<01F0031807380E100E000E000E000E000E000E00FFC00E000E000E000E 000E000E000E000E000E000E000E000E000E000E007FE00D1A80990C>I<0FCE18733030 7038703870387038303018602FC02000600070003FF03FFC1FFE600FC003C003C003C003 6006381C07E010187F8F13>II<18003C003C0018000000 00000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C00FF80091A80990A>I107 DIII<07E01C38300C700E6006E007E007E007E007E007E0076006700E381C1C38 07E010107F8F13>II114 D<1F2060E04020C020C020F0007F003FC01FE0 00F080708030C030C020F0408F800C107F8F0F>I<0400040004000C000C001C003C00FF C01C001C001C001C001C001C001C001C001C201C201C201C201C200E4003800B177F960F >II121 D E /Ft 50 122 df<000100020004000800100020006000C0018001800300070006000E00 0C001C0018003800380030007000700060006000E000E000C000C000C000C000C000C000 C000C000C000C000C000C000C0004000600060002000100010000800102E79A113>40 D<0010000008000004000006000002000003000003000003000001000001800001800001 800001800001800001800001800003800003800003800003000003000003000007000007 00000600000600000E00000C00000C00001C0000180000380000300000700000600000E0 0000C0000180000100000300000600000C0000180000300000600000800000112E80A113 >I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I< 70F8F8F0E005057B840E>I<000F800030E000E07001C0700380300380380700380F0078 0F00780E00781E00781E00703C00F03C00F03C00F03C00F07801E07801E07801E07801C0 7003C0F003C0F00380F00780F00700700700700E00701C003038001870000FC000151F7C 9D17>48 D<000200020006000E003C00DC031C001C003800380038003800700070007000 7000E000E000E000E001C001C001C001C003800380038003800780FFF80F1E7B9D17>I< 001F000061800080E00100E00200700220700420700410700820F00820F00820F00840E0 0881E00703C0000380000700000C00001800006000008000030000040000080040100040 1000802001807E030047FF0041FE0080FC00807800141F7C9D17>I<001F0000718000C0 C00180C00380E00700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C01C03C01C03 C01C07C01C0F800C0F8006378003C700000F00000E00000E00001C00601C00F03800F070 00E0600080C0004380003E0000131F7B9D17>57 D<070F1F1F0E00000000000000000000 70F8F8F0E008147B930E>I<00000200000006000000060000000E0000001E0000001E00 00003F0000002F0000004F0000004F0000008F0000010F0000010F0000020F0000020F00 00040F00000C0F0000080F0000100F0000100F0000200F80003FFF800040078000C00780 0080078001000780010007800200078002000780060007801E000F80FF807FF81D207E9F 22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C003C003C003C 003C003C0078007800780078007800F0007801E000F0078000FFFE0000F00F8000F003C0 01E001C001E001E001E001E001E001E003C001E003C001E003C001E003C001C0078003C0 0780078007800F0007801E000F007800FFFFE0001E1F7D9E20>I<0000FE020007818600 1C004C0038003C0060003C00C0001C01C0001803800018070000180F0000181E0000101E 0000103C0000003C00000078000000780000007800000078000000F0000000F0000000F0 000000F0000000F00000807000008070000080700001003800010038000200180004000C 001800060020000381C00000FE00001F217A9F21>I<01FFFF80001E00E0001E0070001E 0038001E001C003C001C003C000E003C000E003C000E0078000E0078000E0078000E0078 000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01E0007803C0 007003C0007003C000E003C001C0078001C00780038007800E0007801C000F007000FFFF C0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C0004003C00 04003C0004003C00040078080800780800007808000078180000F0300000FFF00000F030 0000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C0004003C000 40078000800780018007800100078007000F001F00FFFFFE001F1F7D9E1F>I<01FFFFFC 001E0038001E0018001E0008001E0008003C0008003C0008003C0008003C000800780010 00780800007808000078080000F0100000F0300000FFF00000F0300001E0200001E02000 01E0200001E0200003C0000003C0000003C0000003C00000078000000780000007800000 078000000F800000FFF800001E1F7D9E1E>I<0000FC040007030C001C00980030007800 E0007801C000380380003003800030070000300E0000301E0000201E0000203C0000003C 00000078000000780000007800000078000000F0000000F000FFF0F0000780F0000780F0 000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E000E00CC0003 83040000FC00001E217A9F23>I<01FFF3FFE0001F003E00001E003C00001E003C00001E 003C00003C007800003C007800003C007800003C007800007800F000007800F000007800 F000007800F00000F001E00000FFFFE00000F001E00000F001E00001E003C00001E003C0 0001E003C00001E003C00003C007800003C007800003C007800003C007800007800F0000 07800F000007800F000007800F00000F801F0000FFF1FFE000231F7D9E22>I<01FFF000 1F00001E00001E00001E00003C00003C00003C00003C0000780000780000780000780000 F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C00003C00007 80000780000780000780000F8000FFF800141F7D9E12>I<01FE00007FC0001E0000FC00 001E0000F80000170001780000170001780000270002F00000270004F00000270004F000 00270008F00000470009E00000470011E00000470021E00000470021E00000870043C000 00838043C00000838083C00000838083C000010381078000010382078000010382078000 0103840780000203840F00000203880F00000203900F00000203900F00000401E01E0000 0401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0002A1F7D9E29> 77 D<01FFFF80001E00E0001E0070001E0038001E003C003C003C003C003C003C003C00 3C003C0078007800780078007800F0007800E000F003C000F00F0000FFFC0000F0000001 E0000001E0000001E0000001E0000003C0000003C0000003C0000003C000000780000007 80000007800000078000000F800000FFF000001E1F7D9E1F>80 D<0007E040001C18C000 3005800060038000C0038001C00180018001000380010003800100038001000380000003 C0000003C0000003F8000001FF800001FFE000007FF000001FF0000001F8000000780000 007800000038000000380020003800200038002000300060007000600060006000E00070 00C000E8038000C606000081F800001A217D9F1A>83 D<0FFFFFF01E0780E01807802010 07802020078020200F0020600F0020400F0020400F0020801E0040001E0000001E000000 1E0000003C0000003C0000003C0000003C00000078000000780000007800000078000000 F0000000F0000000F0000000F0000001E0000001E0000001E0000001E0000003E00000FF FF00001C1F789E21>I<7FFC1FF807C003C00780010007800100078001000F0002000F00 02000F0002000F0002001E0004001E0004001E0004001E0004003C0008003C0008003C00 08003C00080078001000780010007800100078001000F0002000F0002000F0002000F000 4000F0004000700080007001000030020000380400000C18000007E000001D20779E22> II<00F1800389C00707 800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E20F01C 40F01C40703C40705C40308C800F070013147C9317>97 D<07803F800700070007000700 0E000E000E000E001C001C001CF01D0C3A0E3C0E380F380F700F700F700F700FE01EE01E E01EE01CE03CE038607060E031C01F0010207B9F15>I<007E0001C1000300800E07801E 07801C07003C0200780000780000780000F00000F00000F00000F00000F0000070010070 020030040018380007C00011147C9315>I<0000780003F8000070000070000070000070 0000E00000E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C0380 380700780700780700780700F00E00F00E00F00E00F00E20F01C40F01C40703C40705C40 308C800F070015207C9F17>I<007C01C207010E011C013C013802780C7BF07C00F000F0 00F000F0007000700170023804183807C010147C9315>I<00007800019C00033C00033C 000718000700000700000E00000E00000E00000E00000E0001FFE0001C00001C00001C00 001C00003800003800003800003800003800007000007000007000007000007000007000 00E00000E00000E00000E00000C00001C00001C0000180003180007B0000F30000660000 3C00001629829F0E>I<003C6000E27001C1E00380E00700E00F00E00E01C01E01C01E01 C01E01C03C03803C03803C03803C03803C07003C07001C0F001C17000C2E0003CE00000E 00000E00001C00001C00301C00783800F0700060E0003F8000141D7E9315>I<01E0000F E00001C00001C00001C00001C000038000038000038000038000070000070000071E0007 63000E81800F01C00E01C00E01C01C03801C03801C03801C038038070038070038070038 0E10700E20700C20701C20700C40E00CC060070014207D9F17>I<00C001E001E001C000 000000000000000000000000000E003300230043804300470087000E000E000E001C001C 001C003840388030807080310033001C000B1F7C9E0E>I<0001800003C00003C0000380 000000000000000000000000000000000000000000003C00004600008700008700010700 010700020E00000E00000E00000E00001C00001C00001C00001C00003800003800003800 00380000700000700000700000700000E00000E00030E00079C000F180006300003C0000 1228829E0E>I<01E0000FE00001C00001C00001C00001C0000380000380000380000380 000700000700000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC0 0038E000387000387000383840707080707080707080703100E03100601E0013207D9F15 >I<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C00 1C0038003800380038007000700070007100E200E200E200E200640038000A207C9F0C> I<1C0F80F0002630C318004740640C004780680E004700700E004700700E008E00E01C00 0E00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C0708038 038071003803806100380380E10038038062007007006600300300380021147C9325>I< 1C0F802630C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C0 1C01C01C03843803883803083807083803107003303001C016147C931A>I<007C0001C3 000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F007 80F00700700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C04 781C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C03803807803 80700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C 0000FFC000171D809317>I<00F0400388C00705800E03801C03803C0380380700780700 780700780700F00E00F00E00F00E00F00E00F01C00F01C00703C00705C0030B8000F3800 00380000380000700000700000700000700000E00000E0000FFE00121D7C9315>I<1C1E 002661004783804787804707804703008E00000E00000E00000E00001C00001C00001C00 001C000038000038000038000038000070000030000011147C9313>I<00FC030206010C 030C070C060C000F800FF007F803FC003E000E700EF00CF00CE008401020601F8010147D 9313>I<018001C0038003800380038007000700FFF007000E000E000E000E001C001C00 1C001C003800380038003820704070407080708031001E000C1C7C9B0F>I<0E00C03300 E02301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C07 101C0E20180E20180E201C1E200C264007C38014147C9318>I<0E03803307802307C043 83C04301C04700C08700800E00800E00800E00801C01001C01001C01001C02001C02001C 04001C04001C08000E300003C00012147C9315>I<0E00C1C03300E3C02301C3E04381C1 E04301C0E04701C060870380400E0380400E0380400E0380401C0700801C0700801C0700 801C0701001C0701001C0602001C0F02000C0F04000E13080003E1F0001B147C931E>I< 0383800CC4401068E01071E02071E02070C040E00000E00000E00000E00001C00001C000 01C00001C040638080F38080F38100E5810084C60078780013147D9315>I<0E00C03300 E02301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C07 001C0E00180E00180E001C1E000C3C0007DC00001C00001C00003800F03800F07000E060 00C0C0004380003E0000131D7C9316>I E /Fu 77 124 df<007E1F0001C1B1800303E3 C00703C3C00E03C1800E01C0000E01C0000E01C0000E01C0000E01C0000E01C000FFFFFC 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0007F87FC001A1D80 9C18>11 D<007E0001C1800301800703C00E03C00E01800E00000E00000E00000E00000E 0000FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C07F87F8151D809C17>I<007FC001C1C00303C0 0703C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFFFC00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C07FCFF8151D809C17>I<003F07E00001C09C18000380F018000701F03C000E01E0 3C000E00E018000E00E000000E00E000000E00E000000E00E000000E00E00000FFFFFFFC 000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C00 0E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E 00E01C000E00E01C007FC7FCFF80211D809C23>I<6060F0F0F8F8686808080808080810 1010102020404080800D0C7F9C15>34 D<60F0F8680808081010204080050C7C9C0C>39 D<004000800100020006000C000C0018001800300030007000600060006000E000E000E0 00E000E000E000E000E000E000E000E000E000600060006000700030003000180018000C 000C00060002000100008000400A2A7D9E10>I<800040002000100018000C000C000600 060003000300038001800180018001C001C001C001C001C001C001C001C001C001C001C0 01C0018001800180038003000300060006000C000C00180010002000400080000A2A7E9E 10>I<01800180018001804182F18F399C0FF003C003C00FF0399CF18F41820180018001 80018010127E9E15>I<0006000000060000000600000006000000060000000600000006 0000000600000006000000060000000600000006000000060000FFFFFFE0FFFFFFE00006 000000060000000600000006000000060000000600000006000000060000000600000006 00000006000000060000000600001B1C7E9720>I<60F0F0701010101020204080040C7C 830C>II<60F0F06004047C830C>I<0001000300060006000600 0C000C000C0018001800180030003000300060006000C000C000C0018001800180030003 000300060006000C000C000C00180018001800300030003000600060006000C000C00010 297E9E15>I<03C00C301818300C300C700E60066006E007E007E007E007E007E007E007 E007E007E007E007E007E00760066006700E300C300C18180C3007E0101D7E9B15>I<03 0007003F00C7000700070007000700070007000700070007000700070007000700070007 0007000700070007000700070007000F80FFF80D1C7C9B15>I<07C01830201C400C400E F00FF80FF807F8077007000F000E000E001C001C00380070006000C00180030006010C01 180110023FFE7FFEFFFE101C7E9B15>I<07E01830201C201C781E780E781E381E001C00 1C00180030006007E00030001C001C000E000F000F700FF80FF80FF80FF00E401C201C18 3007E0101D7E9B15>I<000C00000C00001C00003C00003C00005C0000DC00009C00011C 00031C00021C00041C000C1C00081C00101C00301C00201C00401C00C01C00FFFFC0001C 00001C00001C00001C00001C00001C00001C0001FFC0121C7F9B15>I<00F0030C06040C 0E181E301E300C700070006000E3E0E430E818F00CF00EE006E007E007E007E007E00760 0760077006300E300C18180C3003E0101D7E9B15>54 D<4000007FFF807FFF007FFF0040 020080040080040080080000100000100000200000600000400000C00000C00001C00001 800001800003800003800003800003800007800007800007800007800007800007800003 0000111D7E9B15>I<03C00C301818300C700C600EE006E006E007E007E007E007E00760 07700F300F18170C2707C700060006000E300C780C78187010203030C00F80101D7E9B15 >57 D<60F0F0600000000000000000000060F0F06004127C910C>I<60F0F06000000000 00000000000060F0F0701010101020204080041A7C910C>I<7FFFFFC0FFFFFFE0000000 0000000000000000000000000000000000000000000000000000000000FFFFFFE07FFFFF C01B0C7E8F20>61 D<0FE03038401CE00EF00EF00EF00E000C001C0030006000C0008001 800100010001000100010001000000000000000000000003000780078003000F1D7E9C14 >63 D<000600000006000000060000000F0000000F0000000F0000001780000017800000 1780000023C0000023C0000023C0000041E0000041E0000041E0000080F0000080F00001 80F8000100780001FFF80003007C0002003C0002003C0006003E0004001E0004001E000C 001F001E001F00FF80FFF01C1D7F9C1F>65 DI<001F808000E0618001801980070007800E0003801C0003801C000180380001807800 00807800008070000080F0000000F0000000F0000000F0000000F0000000F0000000F000 0000F0000000700000807800008078000080380000801C0001001C0001000E0002000700 04000180080000E03000001FC000191E7E9C1E>IIII<001F 808000E0618001801980070007800E0003801C0003801C00018038000180780000807800 008070000080F0000000F0000000F0000000F0000000F0000000F0000000F000FFF0F000 0F80700007807800078078000780380007801C0007801C0007800E00078007000B800180 118000E06080001F80001C1E7E9C21>III<1FFF00F800780078007800780078007800 780078007800780078007800780078007800780078007800787078F878F878F878F0F040 E021C01F00101D7F9B15>IIIII<003F800000E0E0000380380007001C00 0E000E001C0007003C00078038000380780003C0780003C0700001C0F00001E0F00001E0 F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0700001C0780003C0780003C0 380003803C0007801C0007000E000E0007001C000380380000E0E000003F80001B1E7E9C 20>II 82 D<07E0801C1980300580700380600180E00180E00080E00080E00080F00000F80000 7C00007FC0003FF8001FFE0007FF0000FF80000F800007C00003C00001C08001C08001C0 8001C0C00180C00180E00300D00200CC0C0083F800121E7E9C17>I<7FFFFFC0700F01C0 600F00C0400F0040400F0040C00F0020800F0020800F0020800F0020000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000001F800003FFFC001B1C7F9B 1E>IIII89 D<08081010202040404040808080808080B0B0F8 F8787830300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC 000F1C00381C00701C00601C00E01C40E01C40E01C40603C40304E801F870012127E9115 >97 DI<07E00C3018783078703060 00E000E000E000E000E000E00060007004300418080C3007C00E127E9112>I<003F0000 070000070000070000070000070000070000070000070000070000070003E7000C170018 0F00300700700700600700E00700E00700E00700E00700E00700E0070060070070070030 0700180F000C370007C7E0131D7E9C17>I<03E00C301818300C700E6006E006FFFEE000 E000E000E00060007002300218040C1803E00F127F9112>I<00F8018C071E061E0E0C0E 000E000E000E000E000E00FFE00E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E007FE00F1D809C0D>I<00038003C4C00C38C01C3880181800381C00 381C00381C00381C001818001C38000C300013C0001000003000001800001FF8001FFF00 1FFF803003806001C0C000C0C000C0C000C06001803003001C0E0007F800121C7F9215> II<18003C003C0018000000000000 000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C001C001C 001C001C001C00FF80091D7F9C0C>I<00C001E001E000C0000000000000000000000000 00000FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180 E001C0E001C0E001C0E001C0E001C0E001C06001807003803003001806000E1C0003F000 12127F9115>II<03C1000C3300180B00300F0070070070 0700E00700E00700E00700E00700E00700E00700600700700700300F00180F000C370007 C700000700000700000700000700000700000700000700003FE0131A7E9116>II<1F9030704030C010C010E010F8007F803FE00FF000F880388018C018C018E010 D0608FC00D127F9110>I<04000400040004000C000C001C003C00FFE01C001C001C001C 001C001C001C001C001C001C101C101C101C101C100C100E2003C00C1A7F9910>IIII<7F8FF00F03800F030007020003840001C8 0001D80000F00000700000780000F800009C00010E00020E000607000403801E07C0FF0F F81512809116>II<7FFC70386038407040F040E041C003 C0038007000F040E041C043C0C380870087038FFF80E127F9112>I I E /Fv 36 122 df<78FCFCFEFE7A0202040408083040070E7D850D>44 D<78FCFCFCFC7806067D850D>46 D<78FCFCFCFC7800000000000078FCFCFCFC7806127D 910D>58 D<00038000000380000007C0000007C0000007C000000FE000000FE000001FF0 00001BF000001BF0000031F8000031F8000061FC000060FC0000E0FE0000C07E0000C07E 0001803F0001FFFF0003FFFF8003001F8003001F8006000FC006000FC00E000FE00C0007 E0FFC07FFEFFC07FFE1F1C7E9B24>65 D<001FE02000FFF8E003F80FE007C003E00F8001 E01F0000E03E0000E03E0000607E0000607C000060FC000000FC000000FC000000FC0000 00FC000000FC000000FC000000FC0000007C0000607E0000603E0000603E0000C01F0000 C00F80018007C0030003F80E0000FFFC00001FE0001B1C7D9B22>67 DIII73 D76 DII80 D<7FFFFFE07FFFFFE0781F81E0701F80E0601F8060E01F8070C01F8030C01F8030C01F80 30C01F8030001F8000001F8000001F8000001F8000001F8000001F8000001F8000001F80 00001F8000001F8000001F8000001F8000001F8000001F8000001F8000001F800007FFFE 0007FFFE001C1C7E9B21>84 D<0FF8001C1E003E0F803E07803E07C01C07C00007C0007F C007E7C01F07C03C07C07C07C0F807C0F807C0F807C0780BC03E13F80FE1F815127F9117 >97 DI<03FC000E0E001C1F003C1F 00781F00780E00F80000F80000F80000F80000F80000F800007800007801803C01801C03 000E0E0003F80011127E9115>I<000FF0000FF00001F00001F00001F00001F00001F000 01F00001F00001F00001F001F9F00F07F01C03F03C01F07801F07801F0F801F0F801F0F8 01F0F801F0F801F0F801F07801F07801F03C01F01C03F00F0FFE03F9FE171D7E9C1B>I< 01FC000F07001C03803C01C07801C07801E0F801E0F801E0FFFFE0F80000F80000F80000 7800007C00603C00601E00C00F038001FC0013127F9116>I<007F0001E38003C7C00787 C00F87C00F83800F80000F80000F80000F80000F8000FFF800FFF8000F80000F80000F80 000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80007FF8 007FF800121D809C0F>I<03F8F00E0F381E0F381C07303C07803C07803C07803C07801C 07001E0F000E0E001BF8001000001800001800001FFF001FFFC00FFFE01FFFF07801F8F0 0078F00078F000787000707800F01E03C007FF00151B7F9118>II<1E003F003F003F003F001E000000000000000000000000 00FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F00FFE0FF E00B1E7F9D0E>I 108 DII<01FC000F07801C01C03C01E07800F07800F0F8 00F8F800F8F800F8F800F8F800F8F800F87800F07800F03C01E01E03C00F078001FC0015 127F9118>II114 D<1FD830786018E018E018F000FF807FE07FF01FF807FC007CC01C C01CE01CE018F830CFC00E127E9113>I<0300030003000300070007000F000F003FFCFF FC1F001F001F001F001F001F001F001F001F001F0C1F0C1F0C1F0C0F08079803F00E1A7F 9913>IIIIII E /Fw 6 62 df<0102040C1818303070606060E0E0E0E0E0E0E0E0E0E060606070303018 180C04020108227D980E>40 D<8040203018180C0C0E0606060707070707070707070706 06060E0C0C18183020408008227E980E>I<07C018303018701C600C600CE00EE00EE00E E00EE00EE00EE00EE00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF000700070007000700070007000700070007000700070007000700070007 00070007007FF00C157E9412>I<0F8030E040708030C038E03840380038007000700060 00C00180030006000C08080810183FF07FF0FFF00D157E9412>I61 D E /Fx 52 124 df<00030004000800100020006000C0018001000300060006000C000C 001C0018001800380030003000700070006000600060006000E000E000E000E000E000E0 0060006000600060006000300030001000180008000C00040002000100102E7CA112>40 D<008000C000600020003000180018000C000C000C000E00060006000600060006000600 060006000600060006000E000E000E000C000C001C001800180038003000300060006000 C000C001800100030006000C0018003000600080000F2E7FA112>I<1C3E7E7E3A020204 0408081020C0070E7D840D>44 DI<3078F8787005057C840D>I< 007E0001C3000301800701C00E00C00E00E01C00E01C00E03C01E03801E07801E07801E0 7801E07801E07801E0F003C0F003C0F003C0F003C0F003C0F00380F00780E00780E00700 E00700E00E00600E00701C003038003870000FC000131F7C9D17>48 D<000C001C00FC0F380038003800380038003800700070007000700070007000E000E000 E000E000E000E001C001C001C001C001C001C0038003C0FFFE0F1E7C9D17>I<003F8000 C1E00100F00200780400780400780F007C0F807C0F807C0F00780600780000F80000F000 01E00001C0000380000700000E00001C0000380000600000C00001800003002006002008 00401000403FFFC07FFF80FFFF80161E7E9D17>I<007F000183C00201E00400F00700F0 0F00F00F01F00F01F00001E00001E00003C0000380000700000E0000F800000E00000700 0007800007C00003C00007C03007C07807C0F807C0F807C0F00780800F00400E00201C00 18780007E000141F7D9D17>I<0000600000600000E00001C00003C00005C0000DC00009 C00011C000238000438000C3800083800103800203800407000807001807001007002007 00400700FFFFF0000E00000E00000E00000E00000E00001C00001E0001FFE0141E7E9D17 >I<01803001FFE003FFC003FF0003FC0002000002000002000004000004000004000004 7C000587000603800C01800801C00001C00001E00001E00001E00001E07003C0F803C0F0 03C0E00380800780400700400E00201C0018700007C000141F7D9D17>I<000F80007040 00C0200180E00301E00701E00E00C01E00001C00003C000038000078F800790E007A0700 7C0300F80380F80380F003C0F003C0F003C0F003C0F00780E00780E00780E00700E00F00 600E00701C0030180018700007C000131F7C9D17>I<2000003FFFE07FFFC07FFF804001 00C002008002008004000008000010000020000040000040000080000180000300000300 000700000600000E00000E00001E00001C00001C00003C00003C00003C00007800007800 00780000300000131F799D17>I<003F0000C1C00100600200600400300C00300C00300C 00300C00600E00600F80C00FC18007F60003FC0001FC0001FF00063F800C0F801007C030 03C06001C06000C0C000C0C000C0C000C0C00080C0010060030030040018180007E00014 1F7D9D17>I<007E0001C3000301800601C00E01C01C00C03C00E03C00E03C01E07801E0 7801E07801E07801E07803E07803E03803C03807C01C0BC00C13C003E380000780000780 000700000E00600E00F01C00F01800E0300080600041C0003F0000131F7C9D17>I<07E0 1838201C401C701CF03CF03C603C0038007000E001C00180030002000600040004000800 0800080008000000000000000000000030007800F800780070000E20799F15>63 D<0000100000001800000038000000380000007800000078000000FC000001BC0000013C 0000033C0000023C0000063C0000043E0000081E0000081E0000101E0000101E0000201E 0000200F0000400F0000400F0000FFFF0000800F0001000F800100078002000780020007 8004000780040007800C0007C03E0007C0FF807FFC1E207E9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800F8007800 F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007C001E003 C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003C003C007 C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E061800380138 007000F801E0007803C0007007800030078000300F0000301F0000301E0000303E000020 3C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000F8000000 F80000007800004078000080780000803C0000803C0001001C0002000E00020006000C00 0300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F0003C007800 78003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00F0001F00 F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01E0007C03 C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FFFFE00020 1F7E9E23>I<07FFFFF8007C0078003C0038003C00180078001800780008007800080078 0008007800080078080800F0100000F0100000F0100000F0300000FFF00000F0700001E0 200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C0002003C0 002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFFF8007C00 78003C0038003C001800780018007800080078000800780008007800080078000800F010 0000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E0200001E020 0001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C00000078000 0007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C00E0003C 01C0003803800018078000180F0000181F0000181E0000183E0000103C0000007C000000 7C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E0780001E0 780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B800380118001E06080 003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C007800007800F000 007800F000007800F000007800F000007800F000007800F00000F001E00000F001E00000 F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003C00001E0 03C00001E003C00001E003C00003C007800003C007800003C007800003C007800003C007 800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C0000 3C00003C0000780000780000780000780000780000780000F00000F00000F00000F00000 F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C00003C00003 C00003C00007800007C000FFFC00131F7F9E10>I<00FFFC0007C00003C00003C0000780 000780000780000780000780000780000F00000F00000F00000F00000F00000F00001E00 001E00001E00001E00001E00001E00003C00303C00783C00F83C00F83800F07800807000 40E00021C0001F000016207D9E17>I<07FFE07FE0007C001F00003C000C00003C001800 00780010000078004000007800800000780100000078020000007804000000F008000000 F010000000F060000000F0F0000000F1F0000000F278000001E478000001E878000001F0 3C000001E03C000001E01E000001E01E000003C00F000003C00F000003C00F000003C007 800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F7E9E23>I<07FF F000007E0000003C0000003C000000780000007800000078000000780000007800000078 000000F0000000F0000000F0000000F0000000F0000000F0000001E0000001E0000001E0 000001E0000001E0008001E0010003C0010003C0010003C0030003C0020003C0060003C0 060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC0007C0000F800003C00 017800003C00017800004E0002F000004E0002F000004E0004F000004E0004F000004E00 08F000004E0008F00000870011E00000870011E00000870021E00000870021E000008700 41E00000838041E00001038083C00001038083C00001038103C00001038203C0000101C2 03C0000101C403C0000201C40780000201C80780000201C80780000201D00780000200F0 0780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A1F7E9E2A>I<07FC 01FFC0003E003E00003E001800003E001800004F001000004F0010000047801000004780 10000043C010000043C010000083C020000081E020000081E020000080F020000080F020 000080782000010078400001007C400001003C400001003C400001001E400001001E4000 02000F800002000F800002000F800002000780000200078000060003800006000300000F 00010000FFE0010000221F7E9E22>I<0003F800001E0E000038070000E0038001C001C0 03C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F83C0000F87C0000F8 7C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0F80003E0780003E0 780003C0780007C07C0007803C000F003C001E001E001C000E0038000700F00003C3C000 00FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F0007800F0007800F800 7800F8007800F8007800F8007800F000F001F000F001E000F003C000F0078000F00F0000 FFF80001E0000001E0000001E0000001E0000001E0000001E0000003C0000003C0000003 C0000003C0000003C0000003C000000780000007C00000FFFC00001D1F7E9E1F>I<0003 F800001E0E000038070000F0038001E001C003C001E0078001E00F0000F00F0000F01F00 00F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C0000F8F80001F0F800 01F0F80001F0F80001F0F80003E0780003E0780003C0780007C0781E07803C210F003C40 9E001E409C000E80F8000740F00003C1C04000FEC0400000E0400000E0800000E1800000 FF800000FF000000FF0000007E0000003C001D297B9F23>I<07FFFC00007C0700003C03 C0003C01E0007801E0007801F0007801F0007801F0007801F0007801E000F003E000F003 C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E01C0001E01C0001E01C 0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C03E0207803E0407C01F 04FFFC0F18000003E01F207E9E21>I<003F040060CC01803C03801C03001C0700180600 080E00080E00080E00080E00000F00000F80000FE00007FE0003FF8001FFC0007FE00007 E00001E00000E00000F00000F04000E04000E04000E04000E06000C0600180E00380F803 00C60C0081F80016217D9F19>I<3FFFFFF03C0780F03007803060078030400F0010400F 0010C00F0010800F0010800F0010800F0010001E0000001E0000001E0000001E0000001E 0000001E0000003C0000003C0000003C0000003C0000003C0000003C0000007800000078 00000078000000780000007800000078000000F0000001F800007FFFE0001C1F7A9E21> IIII<03FFC0FFC0007F007E00003E003800001E003000001E002000000F004000000F0080 00000F81000000078200000007C600000003C400000003E800000001F000000001F00000 0000F000000000F800000000F8000000017C000000023C000000043C0000000C1E000000 081E000000101F000000200F000000400F800000C0078000008007C000010003C0000700 03E0001F8007E000FFE01FFE00221F7F9E22>II<03FFFFC003E007800380078007000F0006001E0004003C000C003C0008 0078000800F0000801F0000001E0000003C0000007800000078000000F0000001E000000 3C0000003C00000078000000F0010001E0010001E0020003C00200078002000F8006000F 0004001E000C003C001C007C003C007800F800FFFFF8001A1F7D9E1C>I<01FC07060E0F 1C0F380E78007000F000F000F000F000E000E000E000E000F0027004300818300FC01014 7C9314>99 D<0000700003F00000F00000700000700000E00000E00000E00000E00000E0 0000E00001C000F9C00305C00E03C01C03C03801C0780380700380F00380F00380F00380 F00380E00700E00700E00700E00700E00700700F00301E00186F000F8FE014207C9F19> I<00F800070E000E07001C0700380380780380700380F00380F00380FFFF80F00000E000 00E00000E00000E00000F001007002003004001C180007E00011147D9314>I<01C003E0 03E003C0018000000000000000000000000003801F800780038003800700070007000700 070007000E000E000E000E000E000E001C001E00FF800B1F7F9E0C>105 D<038F801F90E007A0E003C0600380600780E00700E00700E00700E00700E00700E00E01 C00E01C00E01C00E01C00E01C00E01C01C03801E03C0FFCFF815147E9319>110 D<00FC000387000E01801C00C03800E03800E07000F0F000F0F000F0F000F0F000F0E001 E0E001E0E001C0E003C0F00380700700380E001C1C0007E00014147D9317>I<00800100 01000100030007000F001E00FFF80E000E000E000E001C001C001C001C001C001C003800 38203820382038203840384018800F000D1C7C9B12>116 D<1C0380FC1F803C07801C03 801C0380380700380700380700380700380700380700700E00700E00700E00700E00701E 00701E00703C00305E001F9FC012147B9319>I<1FF0FF03C07801C06001C04000E08000 E180007300007600003C00003C00001C00002E00004E000087000107000203800603800C 01C03E03E0FF07FC18147F9318>120 D123 D E /Fy 64 124 df<00000F80003F8F80007F8F8000FF8F8001FF8F8003E0000003C000 0007C0000007C0000007C0000007C0000007C0000007C0000007C0000007C0000007C000 00FFFF8F80FFFF8F80FFFF8F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F 8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F 8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F 80192B7FAA20>12 D<001E003C007800F001F001E003C007C007800F800F001F001F001E 003E003E003E007C007C007C007C007C007800F800F800F800F800F800F800F800F800F8 00F800F800F800F800F800F80078007C007C007C007C007C003E003E003E001E001F001F 000F000F80078007C003C001E001F000F00078003C001E0F3D7CAC17>40 DI44 DII<007F000001FF C00007FFF0000FFFF8000FC1F8001F007C003F007E003E003E003C001E007C001F007C00 1F007C001F0078000F00F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F800 0F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F807800 0F007C001F007C001F007C001F003E003E003E003E003F007E001F80FC000FC1F8000FFF F80007FFF00001FFC000007F000019297EA71E>48 D<00180000380000F80007F800FFF8 00FFF800F8F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F8007FFFF07FFFF07FFFF01428 7BA71E>I<00FE0003FFC007FFE00FFFF01F03F83C00FC38007E78003E70003EF0001FF0 001F60001F20001F00001F00001F00001F00003E00003E00007C00007C0000F80001F000 01E00003C0000780000F00001E00003C0000780000F00001E00003C0000780000F00001E 00003C00007FFFFF7FFFFF7FFFFF7FFFFF18287EA71E>I<007F000001FFC00007FFF000 0FFFF8001FC1F8003E007C003C003E0078003E0038003E0010003E0000003E0000003E00 00003C0000007C000000FC000001F8000007F00000FFE00000FFC00000FFE00000FFF000 0001FC0000007C0000003E0000001F0000001F0000000F8000000F8000000F8000000F80 00000F8040000F8060001F00F0001F00F8003F007E007E003F81FC001FFFF8000FFFF000 03FFE000007F000019297EA71E>I<0003F0000007F0000005F000000DF000000DF00000 1DF0000039F0000039F0000079F0000079F00000F1F00000F1F00001E1F00003E1F00003 E1F00007C1F00007C1F0000F81F0000F81F0001F01F0001F01F0003E01F0007C01F0007C 01F000F801F000FFFFFF80FFFFFF80FFFFFF80FFFFFF800001F0000001F0000001F00000 01F0000001F0000001F0000001F0000001F0000001F0000001F00019277EA61E>I<3FFF FC3FFFFC3FFFFC3FFFFC3E00003E00003E00003E00003E00003E00003E00003E00003E00 003E00003E3F003EFFC03FFFE03FFFF03FE1F83F807C3F003E3E003E00003E00001F0000 1F00001F00001F00001F00001F00001F20001F60003E70003EF8007C7C00FC3F03F81FFF F00FFFE007FF8000FE0018287EA61E>I<000FF000003FFC0000FFFC0001FFFC0003F80C 0007E000000FC000000F8000001F0000001E0000003E0000003C0000007C0000007C0000 007C3FE000F8FFF000F9FFF800FBFFFC00FF807E00FF003E00FE003F00FC001F00FC001F 00FC000F80F8000F80F8000F80F8000F80F8000F8078000F807C000F807C000F807C000F 003E001F003E001F001F003E001F807C000FC1FC0007FFF80003FFF00001FFC000007F00 0019297EA71E>II<007F000001FFC00007 FFF0000FFFF8001FC1FC003F007E003E003E007E003F007C001F007C001F007C001F007C 001F007C001F003E003E003E003E001F007C000FC1F80007FFF00003FFE00003FFE0000F FFF8001FC1FC003F007E003E003E007C001F007C001F00F8000F80F8000F80F8000F80F8 000F80F8000F80F8000F807C001F007C001F007E003F003F007E001FC1FC000FFFF80007 FFF00003FFE000007F000019297EA71E>I<007F000001FFC00003FFE0000FFFF0000FC1 F8001F007C003E007C007C003E007C001E007C001F00F8001F00F8001F00F8000F00F800 0F80F8000F80F8000F80F8000F80F8001F807C001F807C001F807E003F803E007F803F00 FF801FFFEF800FFFCF8007FF8F8003FE1F0000001F0000001F0000001E0000003E000000 3E0000007C0000007C000000F8001801F0001E07E0003FFFC0001FFF80000FFE000003F8 000019297EA71E>I<03FC001FFF807FFFC0FFFFE0FC07F07001F04000F80000F80000F8 0000F80000F80001F80001F00007F0000FE0001FC0001F80003E00003E00007C00007800 00F80000F00000F00000F00000F00000F00001F00001F00001F000000000000000000000 00000000000000000000000001F00001F00001F00001F00001F000152A7DA91C>63 D<0001F000000003F800000003F800000007FC00000007BC00000007BC0000000F3E0000 000F1E0000000F1E0000001F1F0000001E1F0000001E0F0000003E0F8000003C0F800000 3C078000007C07C000007807C00000F803E00000F803E00000F003E00001F001F00001F0 01F00001E001F00003E000F80003E000F80003C000F80007FFFFFC0007FFFFFC000FFFFF FE000F80003E000F80003E001F00003F001F00001F001E00001F003E00000F803E00000F 803C00000F807C000007C07C000007C078000007C0F8000003E0F8000003E0232A7EA928 >65 DI<0001 FF00000FFFE0003FFFF8007FFFF800FE01F801F8003003F0001007C000000F8000001F80 00001F0000003E0000003E0000007E0000007C0000007C0000007C000000F8000000F800 0000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F80000007C00 00007C0000007C0000007E0000003E0000003E0000001F0000001F8000000F80000007C0 000003F0000401F8001C00FE00FC007FFFFC003FFFF8000FFFE00001FF001E2C7CAA26> IIII<0001FF00000FFFE0 003FFFFC007FFFFE00FF01FE01F8003E03F0000C07C000000FC000001F8000001F000000 3F0000003E0000007E0000007C0000007C0000007C000000F8000000F8000000F8000000 F8000000F8000000F8000000F8000000F8001FFEF8001FFEF8001FFE7C001FFE7C00003E 7C00003E7E00003E3E00003E3F00003E1F00003E1F80003E0FC0003E07C0003E03F0003E 01F8003E00FF00FE007FFFFE003FFFFC000FFFE00001FF001F2C7CAA28>III76 DII<0001FC0000000FFF8000003FFFE000007FFFF00001FE03FC0003F800FE0007E000 3F0007C0001F000F80000F801F000007C01F000007C03E000003E03E000003E07C000001 F07C000001F07C000001F078000000F0F8000000F8F8000000F8F8000000F8F8000000F8 F8000000F8F8000000F8F8000000F8F8000000F8F8000000F8F8000000F87C000001F07C 000001F07C000001F07E000003F03E000003E03F000007E01F000007C01F80000FC00FC0 001F8007E0003F0007F0007F0003F800FE0001FE03FC0000FFFFF800003FFFE000000FFF 80000001FC0000252C7DAA2C>II82 D<007FC00001FFF80007FFFE000FFFFF001FC07F003F000F007E0006007C0000 007C000000F8000000F8000000F8000000F8000000F8000000FC0000007E0000007F0000 003F8000001FF800000FFF800007FFE00003FFF80000FFFC00000FFE000000FF0000003F 0000001F8000000F8000000FC0000007C0000007C0000007C0000007C0000007C0000007 C000000F8060000F80F0001F00FC003F00FF80FE007FFFFC001FFFF80007FFE00000FF80 001A2C7DAA21>IIIII<01FE000FFF803FFFC03FFFE03C03 F03001F00001F80000F80000F80000F80000F80000F8007FF807FFF81FFFF83FE0F87F00 F8FC00F8F800F8F800F8F800F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8151B7E9A1D >97 DI<007FC001FFF007FF FC0FFFFC1FC07C1F00083E00007C00007C00007C0000F80000F80000F80000F80000F800 00F80000F800007C00007C00007E00003E00001F000C1FC07C0FFFFC07FFFC01FFF0007F 80161B7E9A1B>I<00003E00003E00003E00003E00003E00003E00003E00003E00003E00 003E00003E00003E00003E00003E00003E00FC3E03FF3E07FFFE0FFFFE1FC1FE3F007E3E 003E7C003E7C003EFC003EF8003EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C 003E7C003E3E007E3F00FE1FC1FE0FFFFE07FFBE03FF3E00FC3E172A7EA91F>I<007E00 03FF8007FFC00FFFE01F83F03F00F03E00787C00787C003878003CFFFFFCFFFFFCFFFFFC FFFFFCF80000F80000F800007800007C00007C00003E00003F000C1FC07C0FFFFC07FFFC 01FFF0007F80161B7E9A1B>I<001FC0007FC000FFC001FFC003F00003E00007C00007C0 0007C00007C00007C00007C00007C00007C00007C000FFFE00FFFE00FFFE0007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000122A7FA912 >I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0003E03E0003E03E0007C01F0007C01 F0007C01F0007C01F0007C01F0007C01F0003E03E0003E03E0001F07C0001FFFC0003FFF 80003BFE000038F8000078000000780000003C0000003FFFC0003FFFF8001FFFFC001FFF FE003FFFFF007C007F00F8001F80F8000F80F8000F80F8000F80FC001F807E003F003F80 FE003FFFFE000FFFF80007FFF00000FF80001A287E9A1E>III<007C007C007C007C00 7C0000000000000000000000000000000000000000007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C007C40FCF1F8FFF8FFF07FE00F800E3683A910>III< F83F003F00F8FFC0FFC0FBFFE3FFE0FFFFF7FFF0FF83F783F0FE01FE01F8FC00FC00F8FC 00FC00F8FC00FC00F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800 F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8F800F8 00F8F800F800F8F800F800F8F800F800F8F800F800F8F800F800F8251B7B9A30>II<007F000001FFC00007FFF0000FFFF8001FC1FC003F00 7E003E003E007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F80F800 0F80F8000F80F8000F807C001F007C001F007E003F003E003E003F007E001FC1FC000FFF F80007FFF00001FFC000007F0000191B7E9A1E>I I<00FC3E03FF3E07FFBE0FFFFE1FC1FE3F80FE3E007E7E003E7C003E7C003EF8003EF800 3EF8003EF8003EF8003EF8003EF8003EFC003E7C003E7E003E3E007E3F00FE1FC1FE1FFF FE0FFFBE03FF3E01F83E00003E00003E00003E00003E00003E00003E00003E00003E0000 3E00003E00003E00003E17277E9A1F>II<03FC001FFF803FFFC07FFFC07C07C0F80080F80000F80000F80000FC0000 7F80007FF8003FFE001FFF0007FF8000FFC0000FE00007E00003E00003E04003E0E007E0 FC0FC0FFFFC07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C00007C00007C0 0007C00007C000FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 4007E1C003FFE003FFE001FF8000FC0013227FA116>IIII<7C000FC03E001F803F001F001F803E000F807C0007C0FC0003E0F800 01F1F00001FBE00000FFC000007FC000003F8000001F0000001F0000003F8000007FC000 00FBC00000F3E00001F1F00003E0F80007C07C000F807C000F803E001F001F003E000F80 7E000FC0FC0007E01B1B809A1C>IIII E /Fz 17 117 df<0001FF8000000FFFF000003FFFFC0000FF81FF0001FE007F8003FC00 3FC007F8001FE00FF8001FF00FF0000FF01FF0000FF81FF0000FF83FF0000FFC3FE00007 FC3FE00007FC7FE00007FE7FE00007FE7FE00007FE7FE00007FE7FE00007FEFFE00007FF FFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFF E00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE0 0007FFFFE00007FFFFE00007FFFFE00007FF7FE00007FE7FE00007FE7FE00007FE7FE000 07FE7FE00007FE3FE00007FC3FF0000FFC3FF0000FFC1FF0000FF81FF0000FF80FF0000F F00FF8001FF007F8001FE003FC003FC001FE007F8000FF81FF00003FFFFC00000FFFF000 0001FF800028397CB731>48 D<00001E000000003E00000000FE00000003FE0000003FFE 0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE00007FFFFFFFC07FFFFFFFC07FFFFFFFC0 7FFFFFFFC0223879B731>I<0007FE0000007FFFE00001FFFFF80003FFFFFE0007F01FFF 000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC0007FF0FFE0007FF8FFF0003FF8 FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE0003FFC3FC0003FFC1F80003FFC00 00003FFC0000003FF80000007FF80000007FF00000007FF0000000FFE0000000FFC00000 01FF80000001FF00000003FE00000007FC00000007F80000000FF00000001FC00000003F 800000007F00000000FC00000001F800000001F0003C0003E0003C0007C0003C000F0000 78001E000078003C00007800780000F800F00000F801FFFFFFF803FFFFFFF007FFFFFFF0 0FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFFFFF0FFFFFFFFE0FFFFFFFFE0FF FFFFFFE026387BB731>I<0003FF8000001FFFF000007FFFFE0000FE03FF0001F000FF80 03C000FFC00780007FE00FF0007FF00FF8007FF01FFC007FF81FFE007FF81FFE007FF81F FE007FF81FFE007FF81FFE007FF80FFC007FF007F8007FF003F0007FF0000000FFE00000 00FFC0000001FF80000001FF00000003FE00000007FC0000001FF000000FFFC000000FFF 8000000FFFF800000003FE00000000FF800000007FE00000003FF00000003FF80000003F FC0000001FFC0000001FFE0000001FFE0200001FFF1FC0001FFF3FE0001FFF7FF0001FFF 7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF8001FFEFFF0001FFE7FF0003FFC7F E0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FFC003FFFFFF0001FFFFFE00003F FFF0000007FF800028397CB731>I<00000007C0000000000FC0000000000FC000000000 1FC0000000003FC0000000007FC000000000FFC000000000FFC000000001FFC000000003 FFC000000007FFC00000000FFFC00000000FFFC00000001EFFC00000003CFFC00000007C FFC0000000F8FFC0000000F0FFC0000001E0FFC0000003C0FFC0000007C0FFC000000F80 FFC000000F00FFC000001E00FFC000003C00FFC000007C00FFC00000F800FFC00000F000 FFC00001E000FFC00003C000FFC00007C000FFC0000F8000FFC0000F0000FFC0001E0000 FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFFFFFFFFC0FFFFFFFFFFC0FFFFFF FFFFC0FFFFFFFFFFC0000001FFC000000001FFC000000001FFC000000001FFC000000001 FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000007FF FFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A377DB631>I<04000000C00F8000 07C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000FFFFFFC000FFFFFF8000FFFFFF0 000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F800000000F800000000F80000000 0F800000000F800000000F800000000F800000000F800000000F81FF00000F8FFFE0000F BFFFF8000FFE03FE000FF000FF000FC000FF800F80007FC00F00007FE00700007FF00000 003FF00000003FF80000003FF80000003FF80000003FFC0000003FFC0600003FFC1F8000 3FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFFF0003FFCFFF0003FF8FFE0003F F8FFE0003FF87FC0007FF07F00007FF03C00007FE03E0000FFC01F0000FF800FC003FF00 07F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC000026397BB731>I<00000FF8 0000007FFF000003FFFF80000FFC07C0001FE001E0003FC001F0007F0007F000FF000FF0 01FE001FF803FC003FF807FC003FF80FFC003FF80FF8003FF81FF8001FF01FF8000FE03F F80007C03FF00000003FF00000007FF00000007FF00000007FF00000007FF07FF000FFF0 FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0FFFC001FF0FFFC000FF8FFF800 0FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFFF0000FFFFFF0000FFFFFF0000F FF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF3FF0000FFF3FF0000FFE 3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF8001FF807FC001FF003FC001FF001 FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF8000000FFC00028397CB731>I< 1E00000000001F00000000001FF0000000001FFFFFFFFFC01FFFFFFFFFC01FFFFFFFFFC0 3FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE003FFFFFFFFC003FFFFFFFF800 3FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC0007800001F80007800001F0000 7800003E0000F000007E0000F00000FC0000F00001F80000000003F00000000003E00000 000007E0000000000FC0000000000F80000000001F80000000003F80000000003F000000 00007F00000000007F0000000000FF0000000000FE0000000001FE0000000001FE000000 0003FE0000000003FE0000000003FE0000000007FC0000000007FC0000000007FC000000 000FFC000000000FFC000000000FFC000000000FFC000000000FFC000000001FFC000000 001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000 001FFC000000001FFC000000000FF80000000007F00000000003E00000002A3B7BB931> I<0001FF8000000FFFF800003FFFFE00007F00FF0000F8003F8001F0000FC003E0000FE0 07C00007F007C00007F00FC00003F80FC00003F81FC00003F81FC00003F81FE00003F81F F00003F81FF80003F81FFC0007F01FFF0007F01FFFC007E00FFFF00FE00FFFFC1FC007FF FE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE00007FFFFF00003FFFFF80003FFF FFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1FC007FFFC3F8001FFFE3F8000FF FE7F00003FFF7F00000FFFFE000003FFFE000001FFFE000000FFFE000000FFFE0000007F FE0000007FFE0000007EFE0000007E7F0000007E7F000000FC3F800000FC3FC00001F81F E00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE00003FFFF8000003FF80002839 7CB731>I<0001FF8000001FFFF000007FFFFC0000FF81FE0001FE007F0007FC003F800F F8003FC00FF8001FE01FF8001FF03FF0001FF03FF0000FF87FF0000FF87FF0000FFCFFF0 000FFCFFF0000FFCFFF0000FFEFFF0000FFEFFF0000FFEFFF0000FFEFFF0000FFEFFF000 0FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0001FFF7FF0001FFF3FF0001FFF3FF0001F FF1FF0003FFF0FF8003FFF07F8007FFF03FC006FFF01FE01CFFF00FFEF8FFF007FFF0FFF 000FFE0FFE0000000FFE0000000FFE0000000FFE0000000FFC0000000FFC03E0001FFC07 F0001FF80FF8001FF81FFC001FF01FFC003FF01FFC003FE01FFC003FC01FF8007FC00FF0 00FF800FE001FF0007C003FE0007F01FFC0003FFFFF00001FFFFC000007FFF0000000FF8 000028397CB731>I<0003FF800180001FFFF00380007FFFFC078001FFFFFF0F8003FE00 FF9F8007F0000FFF800FE00003FF801FC00001FF803F8000007F803F8000007F807F0000 003F807F0000001F807F0000001F80FF0000000F80FF0000000F80FF0000000F80FF8000 000780FF8000000780FFC000000780FFE000000780FFF8000000007FFE000000007FFFF0 0000007FFFFF0000003FFFFFF800003FFFFFFF00001FFFFFFFC0000FFFFFFFF00007FFFF FFF80003FFFFFFFC0001FFFFFFFE00007FFFFFFF00003FFFFFFF800007FFFFFF8000007F FFFFC0000007FFFFC00000003FFFE000000003FFE000000000FFF0000000007FF0000000 003FF0700000001FF0F00000001FF0F00000001FF0F00000000FF0F00000000FF0F80000 000FF0F80000000FE0F80000000FE0FC0000000FE0FC0000001FC0FE0000001FC0FF0000 001F80FFC000003F80FFF000007F00FFFC0001FE00FCFFC007FC00F87FFFFFF800F01FFF FFE000E003FFFF8000C0003FFC00002C3D7BBB37>83 D<0000FFF000000FFFFF00003FFF FF8000FFC01FC001FF003FE003FC007FF007FC007FF00FF8007FF01FF0007FF01FF0003F E03FF0003FE03FF0001FC07FE00007007FE00000007FE0000000FFE0000000FFE0000000 FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00000007FE00000007F E00000007FF00000003FF00000003FF00000001FF00000781FF80000780FF80000F007FC 0000F003FE0001E001FF8007C000FFE01F80003FFFFF00000FFFFC000000FFC00025267D A52C>99 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007FC001FC00F F8000FE00FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE00003FC7FE0 0003FC7FE00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE0000000FFE000 0000FFE0000000FFE00000007FE00000007FE00000007FE00000003FE00000003FF00000 3C1FF000003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E000FFC03FC0 003FFFFF000007FFFC000000FFE00026267DA52D>101 D<01F00007FC000FFE000FFE00 1FFF001FFF001FFF001FFF001FFF000FFE000FFE0007FC0001F000000000000000000000 00000000000000000000000000000000000000000000FF00FFFF00FFFF00FFFF00FFFF00 07FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF00 03FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF00 03FF0003FF0003FF0003FF0003FF00FFFFF8FFFFF8FFFFF8FFFFF8153D7DBC1B>105 D<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FFFE1C01FF0007FE3001FF 8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003FF8000FFC003FF8000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFF FFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<0000FFC00000000FFFFC0000003F FFFF000000FFC0FFC00001FE001FE00007FC000FF80007F80007F8000FF00003FC001FF0 0003FE003FF00003FF003FE00001FF007FE00001FF807FE00001FF807FE00001FF807FE0 0001FF80FFE00001FFC0FFE00001FFC0FFE00001FFC0FFE00001FFC0FFE00001FFC0FFE0 0001FFC0FFE00001FFC0FFE00001FFC0FFE00001FFC07FE00001FF807FE00001FF807FE0 0001FF803FF00003FF003FF00003FF001FF00003FE000FF80007FC000FF80007FC0007FC 000FF80003FE001FF00000FFC0FFC000003FFFFF0000000FFFFC00000001FFE000002A26 7DA531>I<0007800000078000000780000007800000078000000F8000000F8000000F80 00000F8000001F8000001F8000003F8000003F8000007F800000FF800001FF800007FF80 001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF800001FF800001FF800001FF800001FF80 0001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF80 0001FF800001FF800001FF800001FF800001FF800001FF803C01FF803C01FF803C01FF80 3C01FF803C01FF803C01FF803C01FF803C00FF807800FFC078007FC070003FE0E0001FFF C00007FF800001FF001E377EB626>116 D E /FA 11 104 df0 D<70F8F8F87005057C8D0D>I<01800180018001800180C183F18F399C0FF003 C003C00FF0399CF18FC1830180018001800180018010147D9417>3 D<03C00FF01C38300C60066006C003C003C003C003C003C00360066006300C1C380FF003 C010127D9317>14 D<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF7FFE7F FE3FFC1FF80FF003C010127D9317>I<000000C0000003C000000F0000003C000000F000 0003C00000070000001C00000078000001E00000078000001E00000078000000E0000000 780000001E0000000780000001E0000000780000001C0000000700000003C0000000F000 00003C0000000F00000003C0000000C00000000000000000000000000000000000000000 00000000000000007FFFFF80FFFFFFC01A247C9C23>20 D<0001FE00000FFF8000181F80 00600780008007000300070007000E000E000C000E0018001C0030001C00000038000000 38000000780000007000000070000200F0000E00F0001C00F0001C00F0003C00F0003C00 F0007800F8007800F800F8007C01F0007E0370003F06F0001FF8E0000FE0E0000001C000 0001C0000003800000030000380600007E0C00007FF000001FC0000019257E9F1B>71 D<00FC00002003FE0000E0043E0001C0181E0001C0301E000380701E000380F01E000700 C01C000700001C000E00003C000E00003C001E00003C001C000038001C00003BFFFC0000 7FFFF80000780038000070007800007000700000F000700000E000F00000E000F00001E0 00E00001E000E00001C001E00001C001E00003C001E000038001E000038001E000078001 E040070001E1C0060001F3000C0001FE00000000F80023217F9E26>I<400002C00006C0 0006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C0 0006C00006C00006C00006C00006C00006C0000660000C60000C3000181C00700F01E003 FF8000FE00171C7D9A1E>91 D<000F0038006000E001C001C001C001C001C001C001C001 C001C001C001C001C001C001C001C0038007001E00F8001E000700038001C001C001C001 C001C001C001C001C001C001C001C001C001C001C001C000E000600038000F102D7DA117 >102 DI E /FB 1 98 df<001800001800001800003C00003C00004E00004E00004E000087000087 000187800103800103800201C00201C003FFC00400E00400E00800700800701800703C00 78FE01FF18177F961C>97 D E /FC 10 58 df<1F00318060C04040C060C060C060C060 C060C060C060C060404060C031801F000B107F8F0F>48 D<0C003C00CC000C000C000C00 0C000C000C000C000C000C000C000C000C00FF8009107E8F0F>I<1F00618040C08060C0 600060006000C00180030006000C00102020207FC0FFC00B107F8F0F>I<1F00218060C0 60C000C0008001800F00008000400060C060C060804060801F000B107F8F0F>I<030003 0007000F000B001300330023004300C300FFE003000300030003001FE00B107F8F0F>I< 20803F002C002000200020002F0030802040006000600060C06080C061801F000B107F8F 0F>I<0780184030C060C06000C000CF00F080E040C060C060C060406060C030801F000B 107F8F0F>I<40007FE07FC08080808001000200040004000C0008000800180018001800 180018000B117E900F>I<1F00318060C060C060C071803F000F00338061C0C060C060C0 60404060801F000B107F8F0F>I<1F00318060C0C040C060C060C06040E021E01E600060 004060C0608043003E000B107F8F0F>I E /FD 29 123 df<70F8F8F87005057C840D> 58 D<70F8FCFC74040404080810102040060E7C840D>I<000001C00000078000001E0000 0078000001E00000078000000E00000038000000F0000003C000000F0000003C000000F0 000000F00000003C0000000F00000003C0000000F0000000380000000E00000007800000 01E0000000780000001E0000000780000001C01A1A7C9723>I62 D<000002000000060000000E0000000E0000001E0000001F0000002F0000002F0000004F 0000008F0000008F0000010F0000010F0000020F0000040F0000040F0000080F80000807 80001007800020078000200780007FFF8000400780008007800180078001000780020007 80020007C0040003C00C0003C01E0007C0FF807FFC1E207E9F22>65 D<00FFFFE000000F007800000F001C00000F000E00000F000700001E000700001E000380 001E000380001E000380003C000380003C000380003C000380003C000380007800078000 780007800078000780007800078000F0000F0000F0000F0000F0000E0000F0001E0001E0 001C0001E0003C0001E000380001E000700003C000E00003C001C00003C003800003C007 000007803C0000FFFFF00000211F7E9E26>68 D<0FFFFFFC1E03C0381803C0181003C008 2003C00820078008600780084007800840078008800F0010000F0000000F0000000F0000 001E0000001E0000001E0000001E0000003C0000003C0000003C0000003C000000780000 00780000007800000078000000F0000000F0000000F0000000F0000001F000007FFFC000 1E1F7F9E1B>84 D<00F1800389C00707800E03801C03803C038038070078070078070078 0700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F078014147E93 18>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E 380F380F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207E9F 14>I<0000780003F80000700000700000700000700000E00000E00000E00000E00001C0 0001C000F1C00389C00707800E03801C03803C0380380700780700780700780700F00E00 F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F078015207E9F18>100 D<007C01C207010E011C013C013802780C7BF07C00F000F000F000F00070007001700230 04183807C010147E9315>I<00007C0000CE00019E00039E00030C000700000700000700 000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C00001C00001C00 00380000380000380000380000380000700000700000700000700000700000E00000E000 00E00000E00000C00001C000318000798000F300006200003C000017297E9F16>I<001E 3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E01C01E01C01E01 C01E01C01E03801E03800E07800E0B8006170001E700000700000700000E00000E00300E 00781C00F038006070003FC000151D809316>I<00E001E001E000C00000000000000000 0000000000000E00130023804380438043808700070007000E000E001C001C001C203840 38403840388019000E000B1F7E9E10>105 D<0000C00001E00001E00001C00000000000 00000000000000000000000000000000001E000063000043800083800103800103800207 00000700000700000700000E00000E00000E00000E00001C00001C00001C00001C000038 0000380000380000380000700000700030700078E000F1C0006380003E00001328819E13 >I<01E0000FE00001C00001C00001C00001C00003800003800003800003800007000007 00000701E00706100E08700E10F00E20F00E40601C80001D00001E00001FC00038700038 3800383800381C20703840703840703840701880E01880600F0014207E9F18>I<03C01F C0038003800380038007000700070007000E000E000E000E001C001C001C001C00380038 00380038007000700070007100E200E200E200E200640038000A207E9F0E>I<1E07C07C 00231861860023A032030043C03403004380380380438038038087007007000700700700 070070070007007007000E00E00E000E00E00E000E00E00E000E00E01C101C01C01C201C 01C038201C01C038401C01C0184038038018801801800F0024147E9328>I<1E07802318 C023A06043C0704380704380708700E00700E00700E00700E00E01C00E01C00E01C00E03 821C03841C07041C07081C03083803101801E017147E931B>I<007C0001C3000301800E 01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780F0070070 0F00700E0030180018700007C00013147E9316>I<03C1E004621804741C08781C08701E 08701E10E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380700380E0 03C1C0072380071E000700000700000E00000E00000E00000E00001C00001C0000FFC000 171D819317>I<007C018203010603060706060E00078007F803FC01FE001F00077007F0 06F006E004400820301FC010147E9315>115 D<00C000E001C001C001C001C003800380 FFF8038007000700070007000E000E000E000E001C001C001C001C103820382038203840 18800F000D1C7F9B10>I<0F00601180702180E021C0E041C0E04380E08381C00701C007 01C00701C00E03800E03800E03800E03840E07080C07080C07080E0F1006131003E1E016 147E931A>I<0F01801183C02183E021C1E041C0E0438060838040070040070040070040 0E00800E00800E00800E01000E01000C02000E04000E040006180001E00013147E9316> I<0F006060118070F02180E0F821C0E07841C0E0384380E0188381C0100701C0100701C0 100701C0100E0380200E0380200E0380200E0380400E0380400E0380800E078080060781 000709860001F078001D147E9321>I<03C1C00C62201034701038F02038F02038604070 0000700000700000700000E00000E00000E00000E02061C040F1C040F1C080E2C0804463 00383C0014147E931A>I<0F00601180702180E021C0E041C0E04380E08381C00701C007 01C00701C00E03800E03800E03800E03800E07000C07000C07000E0F00061E0003EE0000 0E00000E00001C0078180078380070700060600021C0001F0000141D7E9316>I<01E020 03F04007F8C00C1F8008010000020000040000080000100000600000C000010000020000 0400800801001003003F060061FC0040F80080700013147E9315>I E /FE 91 128 df<001F83E000F06E3001C078780380F8780300F0300700700007007000 0700700007007000070070000700700007007000FFFFFF80070070000700700007007000 070070000700700007007000070070000700700007007000070070000700700007007000 0700700007007000070070000700700007007000070070007FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700000700000700 00FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000 E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F04 0001C07C06000380F80F000300F00F000700F00F00070070000007007000000700700000 070070000007007000000700700000FFFFFFFF0007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 07007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C07C0F000380F80F000300 F00700070070070007007007000700700700070070070007007007000700700700070070 0700FFFFFFFF000700700700070070070007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 070070070007007007000700700700070070070007007007007FE3FE3FF02420809F26> I<7038F87CFC7EFC7E743A0402040204020804080410081008201040200F0E7E9F17>34 D<000300C0000300C0000300C0000300C000060180000601800006018000060180000601 80000C0300000C0300000C0300000C0300000C0300001806007FFFFFFCFFFFFFFE00300C 0000300C0000300C0000300C0000601800006018000060180000601800FFFFFFFE7FFFFF FC00C0300001806000018060000180600001806000018060000300C0000300C0000300C0 000300C0000300C0000601800006018000060180001F297D9F26>I<0078000000840000 018400000302000007020000070200000702000007020000070400000704000007080000 070800000310000003A00FFC03C003E0038001C001C0008001C0010003E0010004E00200 08F00200187004003078080070380800701C1000F01E1000F00E2000F0074000F003C008 7003C0087801C010380670301C18386007E00F801E227EA023>38 D<70F8FCFC74040404080810102040060E7C9F0D>I<0020004000800100020006000C00 0C00180018003000300030007000600060006000E000E000E000E000E000E000E000E000 E000E000E000E0006000600060007000300030003000180018000C000C00060002000100 0080004000200B2E7DA112>I<800040002000100008000C000600060003000300018001 80018001C000C000C000C000E000E000E000E000E000E000E000E000E000E000E000E000 C000C000C001C001800180018003000300060006000C00080010002000400080000B2E7D A112>I<01800180018001800180C183F18F399C0FF003C003C00FF0399CF18FC1830180 018001800180018010147DA117>I<000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000000600000006000000 060000FFFFFFF0FFFFFFF000060000000600000006000000060000000600000006000000 06000000060000000600000006000000060000000600000006000000060000000600001C 207D9A23>I<70F8FCFC74040404080810102040060E7C840D>I I<70F8F8F87005057C840D>I<000100030003000600060006000C000C000C0018001800 1800300030003000600060006000C000C000C00180018001800300030003000600060006 000C000C000C00180018001800300030003000600060006000C000C000C000102D7DA117 >I<03F0000E1C001C0E00180600380700700380700380700380700380F003C0F003C0F0 03C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C070038070 03807003807807803807001806001C0E000E1C0003F000121F7E9D17>I<018003800F80 F38003800380038003800380038003800380038003800380038003800380038003800380 038003800380038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C00100E002007 00400780800780F007C0F803C0F803C0F803C02007C00007C0000780000780000F00000E 00001C0000380000700000600000C0000180000300000600400C00401800401000803FFF 807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00780F8078078078078038 0F80000F80000F00000F00000E00001C0000380003F000003C00000E00000F0000078000 07800007C02007C0F807C0F807C0F807C0F00780400780400F00200E001C3C0003F00012 1F7E9D17>I<000600000600000E00000E00001E00002E00002E00004E00008E00008E00 010E00020E00020E00040E00080E00080E00100E00200E00200E00400E00C00E00FFFFF0 000E00000E00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I<1803001FFE 001FFC001FF8001FE00010000010000010000010000010000010000011F000161C00180E 001007001007800003800003800003C00003C00003C07003C0F003C0F003C0E003804003 80400700200600100E000C380003E000121F7E9D17>I<007C000182000701000E03800C 07801C0780380300380000780000700000700000F1F000F21C00F40600F80700F80380F8 0380F003C0F003C0F003C0F003C0F003C07003C07003C07003803803803807001807000C 0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF80400100800200800200 80040000080000080000100000200000200000400000400000C00000C00001C000018000 038000038000038000038000078000078000078000078000078000078000078000030000 121F7D9D17>I<03F0000C0C001006003003002001806001806001806001807001807803 003E03003F06001FC8000FF00003F80007FC000C7E00103F00300F806003804001C0C001 C0C000C0C000C0C000C0C000806001802001001002000C0C0003F000121F7E9D17>I<03 F0000E18001C0C00380600380700700700700380F00380F00380F003C0F003C0F003C0F0 03C0F003C07007C07007C03807C0180BC00E13C003E3C000038000038000038000070030 0700780600780E00700C002018001070000FC000121F7E9D17>I<70F8F8F87000000000 00000000000070F8F8F87005147C930D>I<70F8F8F8700000000000000000000070F0F8 F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF000000000000000000000 00000000000000000000000000000000000000000000FFFFFFF07FFFFFE01C0C7D9023> 61 D<07000F800F800F8007000000000000000000000002000200020002000200020006 00060004000C001C00380078007000F018F03CF03CF01CF008701838200FC00E207D9615 >I<0FC0307040384038E03CF03CF03C603C0038007000E000C001800180010003000200 020002000200020002000000000000000000000007000F800F800F8007000E207D9F15> I<000100000003800000038000000380000007C0000007C0000007C0000009E0000009E0 000009E0000010F0000010F0000010F00000207800002078000020780000403C0000403C 0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F0002000780020007 8002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F204F00 07A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000001FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F00 001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004078004 0F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C>II<080410082010201040204020804080408040B85CFC7E FC7E7C3E381C0F0E7B9F17>II<08102020404080 8080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00000E00 000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10386720 0F83C014147E9317>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00780E00 780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E001520 7F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0007000700038 0138011C020E0C03F010147E9314>I<000380003F800003800003800003800003800003 8000038000038000038000038000038003E380061B801C07803803803803807003807003 80F00380F00380F00380F00380F00380F003807003807003803803803807801C07800E1B 8003E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F00380F0 0380FFFF80F00000F00000F000007000007000003800801800800C010007060001F80011 147F9314>I<007C00C6018F038F07060700070007000700070007000700FFF007000700 07000700070007000700070007000700070007000700070007000700070007007FF01020 809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F00780F0038 0E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC03001E060 0070C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I<0E0000 FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E3E00 0E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C001E003E001E00 1C000000000000000000000000000E007E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001F001F000E000 0000000000000000000000007007F000F000700070007000700070007000700070007000 70007000700070007000700070007000700070007000706070F060F0C061803F000C2882 9E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70000EF8000F38 000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF815207F9F18>I<0E 00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B20809F0C>I<0E1F 01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I<0E3E00 FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E001C03 803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000E07000 E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F01C00F00E00E 00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E00F00E00E00F 01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E00000E0000FF E000151D7F9319>I<03E0800619801C05803C0780380380780380700380F00380F00380 F00380F00380F00380F003807003807803803803803807801C0B800E138003E380000380 000380000380000380000380000380000380000380003FF8151D7E9318>I<0E78FE8C0F 1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00F 147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070803880188018 C018C018E030D0608F800D147E9312>I<020002000200060006000E000E003E00FFF80E 000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610031001 E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16147F93 19>III<7FC3FC0F01E00701C007018003810001C20000E40000EC 00007800003800003C00007C00004E000087000107000303800201C00601E01E01E0FF07 FE1714809318>II<3FFF380E200E 201C40384078407000E001E001C00380078007010E011E011C0338027006700EFFFE1014 7F9314>III<30307878F87C787830 300E057C9E17>127 D E /FF 63 123 df<000FF07F00007FF9FF8000F83FC7C001E07F 8FC003E07F0FC007C07F0FC007C03F078007C01F000007C01F000007C01F000007C01F00 0007C01F0000FFFFFFF800FFFFFFF80007C01F000007C01F000007C01F000007C01F0000 07C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007 C01F000007C01F000007C01F000007C01F000007C01F00003FF8FFF0003FF8FFF0002220 809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C0007C07C0007C07C0007C0 380007C0000007C0000007C0000007C1FE00FFFFFE00FFFFFE0007C03E0007C03E0007C0 3E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C0 3E0007C03E0007C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>I<380700 7C0F80FE1FC0FF1FE0FF1FE07F0FE03B07600300600300600600C00600C00E01C00C0180 180300700E0020040013107E9F1B>34 D<387CFEFFFF7F3B030306060E0C18702008107C 860F>44 DI<01FC0007FF001F07C01E03C03E03E07C 01F07C01F07C01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC 01F8FC01F8FC01F8FC01F87C01F07C01F07C01F03E03E01E03C01F8FC007FF0001FC0015 1D7E9C1A>48 D<00E00001E0000FE000FFE000F3E00003E00003E00003E00003E00003E0 0003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E0 0003E00003E00003E00003E00003E000FFFF80FFFF80111D7C9C1A>I<07F0001FFE0038 3F007C1F80FE0FC0FE0FC0FE0FE0FE07E07C07E03807E0000FE0000FC0000FC0001F8000 1F00003E0000780000F00000E00001C0000380600700600E00601C00E01FFFC03FFFC07F FFC0FFFFC0FFFFC0131D7D9C1A>I<01FC0007FF000E0F801E0FC03F07E03F07E03F07E0 3F07E01E0FC0000FC0000F80001F0001FC0001FC00000F800007C00003E00003F00003F8 3803F87C03F8FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF8003FC00151D7E9C1A> I<0001C00003C00007C00007C0000FC0001FC0003BC00073C00063C000C3C00183C00383 C00703C00E03C00C03C01803C03803C07003C0E003C0FFFFFEFFFFFE0007C00007C00007 C00007C00007C00007C000FFFE00FFFE171D7F9C1A>I<3803803FFF803FFF003FFE003F FC003FF0003F800030000030000030000030000033F80037FE003C1F00380F801007C000 07C00007E00007E07807E0FC07E0FC07E0FC07E0FC07C0780FC0600F80381F001FFC0007 F000131D7D9C1A>I<003F0001FFC007E0E00F81E01F03F01E03F03E03F07C03F07C01E0 7C0000FC1000FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8FC01F8FC01F87C01F8 7C01F87C01F83C01F03E01F01E03E00F07C007FF8001FE00151D7E9C1A>I<6000007FFF F87FFFF87FFFF07FFFE07FFFC0E00180C00300C00300C00600000C000018000038000038 0000780000700000F00000F00001F00001F00001F00001F00003F00003F00003F00003F0 0003F00003F00001E00000C000151E7D9D1A>I<01FC0007FF000F07801E03C01C01E03C 01E03C01E03E01E03F01E03FC3C01FE3801FFF000FFE0007FF8007FFC01FFFE03C3FF078 0FF07803F8F001F8F000F8F00078F00078F000707800707C00E03E03C00FFF8003FC0015 1D7E9C1A>I<01FC000FFF001F07803E03C07C03E07C01E0FC01F0FC01F0FC01F0FC01F8 FC01F8FC01F8FC01F87C03F87C03F83E05F81FFDF807F9F80041F80001F03C01F07E01F0 7E03E07E03E07E07C03C0780381F001FFC0007F000151D7E9C1A>I<387CFEFEFE7C3800 0000000000387CFEFEFE7C3807147C930F>I<0000E000000000E000000001F000000001 F000000001F000000003F800000003F800000006FC00000006FC0000000EFE0000000C7E 0000000C7E000000183F000000183F000000303F800000301F800000701FC00000600FC0 0000600FC00000C007E00000FFFFE00001FFFFF000018003F000018003F000030001F800 030001F800060001FC00060000FC000E0000FE00FFE00FFFE0FFE00FFFE0231F7E9E28> 65 DI<0007FC02003FFF0E00FE 03DE03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E0000067E00 0006FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E0000007E00 00067F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000FE01C0003F FF800007FC001F1F7D9E26>IIII<0007FC0200003FFF0E0000FE03DE0003F000FE0007E0003E000FC0001E001F 80001E003F00000E003F00000E007F000006007E000006007E00000600FE00000000FE00 000000FE00000000FE00000000FE00000000FE003FFFE0FE003FFFE07E00007E007E0000 7E007F00007E003F00007E003F00007E001F80007E000FC0007E0007E0007E0003F000FE 0000FE01FE00003FFF8E000007FC0600231F7D9E29>III76 DII<001FF80000FFFF0001F81F8007E007 E00FC003F01F8001F81F0000F83F0000FC7F0000FE7E00007E7E00007EFE00007FFE0000 7FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007F7E00007E7F0000 FE7F0000FE3F0000FC3F8001FC1F8001F80FC003F007E007E001F81F8000FFFF00001FF8 00201F7D9E27>II<001FF80000 FFFF0001F81F8007E007E00FC003F01F8001F81F8001F83F0000FC7F0000FE7F0000FE7E 00007EFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE 00007F7E00007E7E00007E7F0000FE3F0000FC3F87C1FC1F8FE1F80FD833F007F83FE001 F81F8000FFFF00001FFE0300001E0300001F0700000FFF00000FFF000007FE000007FE00 0003FC000001F8000000F020287D9E27>II< 03FC080FFF381E03F83800F8700078700038F00038F00018F00018F80000FC00007FC000 7FFE003FFF801FFFE00FFFF007FFF000FFF80007F80000FC00007C00003CC0003CC0003C C0003CE00038E00078F80070FE01E0E7FFC081FF00161F7D9E1D>I<7FFFFFFC7FFFFFFC 7C07E07C7007E01C6007E00C6007E00CE007E00EC007E006C007E006C007E006C007E006 0007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E000 0007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00003FFFFC0 03FFFFC01F1E7E9D24>III<0400800E01C0180300300600700E00600C00600C00C01800C0 1800DC1B80FE1FC0FF1FE0FF1FE07F0FE03E07C01C038013107B9F1B>92 D<07FC001FFF003F0F803F07C03F03E03F03E00C03E00003E0007FE007FBE01F03E03C03 E07C03E0F803E0F803E0F803E0FC05E07E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007 FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC0000FC0000FC0000FC0000FC00007C 00007E00003E00603F00C01F81C007FF0001FC0013147E9317>I<0007F80007F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F801F8F80FFEF81F83F8 3E01F87E00F87C00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87C00F8 7E00F83E01F81F07F80FFEFF03F8FF18207E9F1D>I<01FE0007FF800F83C01E01E03E00 F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C00003E00181E00 180F807007FFE000FF8015147F9318>I<001F8000FFC001F3E003E7E003C7E007C7E007 C3C007C00007C00007C00007C00007C000FFFC00FFFC0007C00007C00007C00007C00007 C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0003F FC003FFC0013207F9F10>I<01FC3C07FFFE0F079E1E03DE3E03E03E03E03E03E03E03E0 3E03E01E03C00F07800FFF0009FC001800001800001C00001FFF800FFFF007FFF81FFFFC 3C007C70003EF0001EF0001EF0001E78003C78003C3F01F80FFFE001FF00171E7F931A> II<1C003E00 7F007F007F003E001C00000000000000000000000000FF00FF001F001F001F001F001F00 1F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B217EA00E>I<003800 7C00FE00FE00FE007C003800000000000000000000000001FE01FE003E003E003E003E00 3E003E003E003E003E003E003E003E003E003E003E003E003E003E003E003E303E783EFC 3CFC7C78783FF01FC00F2A83A010>IIIII<01FF0007FFC01F83F03E00F83E00F87C007C7C007CFC007EFC007EFC007EFC00 7EFC007EFC007E7C007C7C007C3E00F83E00F81F83F007FFC001FF0017147F931A>II<01F81807FE381F87783F01F83E01F8 7E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87E00F87E00F83F01F8 1F87F80FFEF803F8F80000F80000F80000F80000F80000F80000F80000F80007FF0007FF 181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07FF83FFC1FFE03FE001FC007C007E007F006 F81EFFFCC7F010147E9315>I<01800180018003800380038007800F803F80FFFCFFFC0F 800F800F800F800F800F800F800F800F800F800F860F860F860F860F8607CC03F801F00F 1D7F9C14>I IIIII<3FFFE03FFFE03C07C0380F80701F80603F00603E00607C0000F800 01F80003F00003E06007C0600F80601F80E03F00C03E01C07C03C0FFFFC0FFFFC013147F 9317>I E /FG 37 122 df<00000007FF800000000001FFFFF0000000000FFFFFFC0000 00003FFFFFFE00000000FFFC00FF00000001FFC0003F80000007FF00007FC000000FFE00 01FFC000001FFC0001FFE000001FF80003FFE000003FF00003FFE000003FF00003FFE000 007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE000 01FFC000007FE00000FF8000007FE000003E0000007FE00000000000007FE00000000000 007FE00000000000007FE00000000000007FE00000000000007FE00000000000007FE000 3FFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000 FFFFFFFFFFFFF000007FF00000FFF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF0003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE0 3FFFFFE03FFFFFE03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC007FFE00FF FE00FFFF00FFFF00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E78000078000 0780000F00000F00000F00000F00001E00001E00003C00003C0000780000F80000F00001 F00003E00007C0000F80001F00001E00000C00001124788F21>44 D<0000000003E000000000000000000007F000000000000000000007F000000000000000 00000FF80000000000000000000FF80000000000000000000FF80000000000000000001F FC0000000000000000001FFC0000000000000000003FFE0000000000000000003FFE0000 000000000000007FFF0000000000000000007FFF0000000000000000007FFF0000000000 00000000FFFF800000000000000000FFFF800000000000000001FFFFC000000000000000 01FFFFC00000000000000001FFFFC00000000000000003FFFFE00000000000000003EFFF E00000000000000007EFFFF00000000000000007CFFFF00000000000000007C7FFF00000 00000000000FC7FFF8000000000000000F83FFF8000000000000001F83FFFC0000000000 00001F03FFFC000000000000001F01FFFC000000000000003F01FFFE000000000000003E 00FFFE000000000000007E00FFFF000000000000007C007FFF00000000000000FC007FFF 80000000000000F8007FFF80000000000000F8003FFF80000000000001F8003FFFC00000 00000001F0001FFFC0000000000003F0001FFFE0000000000003E0001FFFE00000000000 03E0000FFFE0000000000007E0000FFFF0000000000007C00007FFF000000000000FC000 07FFF800000000000F800007FFF800000000000F800003FFF800000000001F800003FFFC 00000000001F000001FFFC00000000003FFFFFFFFFFE00000000003FFFFFFFFFFE000000 00007FFFFFFFFFFF00000000007FFFFFFFFFFF00000000007FFFFFFFFFFF0000000000FC 0000007FFF8000000000F80000003FFF8000000001F80000003FFFC000000001F0000000 3FFFC000000001F00000001FFFC000000003F00000001FFFE000000003E00000000FFFE0 00000007E00000000FFFF000000007C00000000FFFF000000007C000000007FFF0000000 0FC000000007FFF80000000F8000000003FFF80000001F8000000003FFFC0000001F0000 000003FFFC0000007FC000000001FFFC0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FF FFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF 8051487CC75A>65 DI<000000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E0000 03FFFFFFFE00FE00000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001 FFFC000003FFFE0007FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003F FF000000001FFE007FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FF F00000000003FE03FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFF C00000000000FE0FFFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF 8000000000007E3FFF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF 0000000000003E7FFF000000000000007FFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF000000000000007FFF 000000000000007FFF000000000000007FFF000000000000007FFF8000000000003E3FFF 8000000000003E3FFF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFF C000000000003E0FFFC000000000007C0FFFC000000000007C07FFE000000000007C07FF E00000000000F803FFF00000000000F803FFF00000000001F801FFF80000000001F000FF FC0000000003E0007FFE0000000007E0003FFF000000000FC0001FFF800000001F80000F FFC00000003F000007FFF0000000FE000001FFFC000001FC000000FFFF80000FF8000000 3FFFF8007FF00000000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000 000FFFFFE00000000000003FFE000000474979C756>I69 DI<000000003FFE00000E000000000FFFFFC0001E000000007FFFFFF8003E00000003FF FFFFFE00FE0000000FFFFFFFFF81FE0000003FFFF800FFC3FE000000FFFF80000FF7FE00 0001FFFC000003FFFE000007FFF0000001FFFE00000FFFC00000007FFE00001FFF800000 003FFE00003FFF000000001FFE00007FFE000000000FFE0000FFFC0000000007FE0001FF F80000000007FE0003FFF00000000003FE0003FFF00000000001FE0007FFE00000000001 FE0007FFE00000000000FE000FFFC00000000000FE000FFFC000000000007E001FFFC000 000000007E001FFF8000000000007E003FFF8000000000007E003FFF8000000000003E00 3FFF8000000000003E007FFF8000000000003E007FFF0000000000003E007FFF00000000 000000007FFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF 0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF000000000000 0000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000 000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000007FFFFFFFFE 7FFF0000007FFFFFFFFE7FFF0000007FFFFFFFFE7FFF8000007FFFFFFFFE7FFF8000007F FFFFFFFE3FFF8000000000FFFE003FFF8000000000FFFE003FFF8000000000FFFE001FFF 8000000000FFFE001FFFC000000000FFFE000FFFC000000000FFFE000FFFC000000000FF FE0007FFE000000000FFFE0007FFE000000000FFFE0003FFF000000000FFFE0003FFF800 000000FFFE0001FFF800000000FFFE0000FFFC00000000FFFE00007FFE00000000FFFE00 003FFF00000000FFFE00001FFF80000000FFFE00000FFFE0000001FFFE000007FFF00000 03FFFE000001FFFC000007FFFE000000FFFF80001FFFFE0000003FFFFC00FFC7FE000000 0FFFFFFFFF83FE00000003FFFFFFFE00FE000000007FFFFFF8003E000000000FFFFFE000 0E00000000003FFE000000004F4979C75D>I73 D76 DI80 D<00007FF00007000007FFFF000F00001FFFFFC01F00 007FFFFFF03F0000FFFFFFFC7F0003FFC00FFEFF0007FE0000FFFF0007FC00003FFF000F F000001FFF001FF0000007FF001FE0000003FF003FE0000001FF003FC0000001FF007FC0 000000FF007FC00000007F007FC00000007F00FFC00000003F00FFC00000003F00FFC000 00003F00FFE00000003F00FFE00000001F00FFF00000001F00FFF80000001F00FFFC0000 001F00FFFF00000000007FFFC0000000007FFFFE000000007FFFFFE00000003FFFFFFF00 00003FFFFFFFF000001FFFFFFFFC00001FFFFFFFFF00000FFFFFFFFFC00007FFFFFFFFE0 0007FFFFFFFFF00003FFFFFFFFF80000FFFFFFFFFC00007FFFFFFFFE00003FFFFFFFFE00 000FFFFFFFFF000001FFFFFFFF0000003FFFFFFF80000001FFFFFF800000000FFFFFC000 000000FFFFC0000000001FFFC0000000000FFFE00000000007FFE00000000003FFE07800 000001FFE0F800000000FFE0F800000000FFE0F8000000007FE0F8000000007FE0F80000 00007FE0FC000000007FE0FC000000007FC0FC000000007FC0FE000000007FC0FE000000 007F80FF00000000FF80FF80000000FF80FFC0000000FF00FFE0000001FE00FFF8000003 FE00FFFE000007FC00FFFF80000FF800FFFFFC007FF000FE3FFFFFFFE000FC0FFFFFFF80 00F803FFFFFF0000F0007FFFF80000E00003FFC00000334979C742>83 D<3FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFF FFFFFFFF003FFFFFFFFFFFFFFFFF003FFF0003FFF8003FFF007FF80003FFF80007FF807F E00003FFF80001FF807FC00003FFF80000FF807F800003FFF800007F807F000003FFF800 003F807F000003FFF800003F807E000003FFF800001F807E000003FFF800001F807E0000 03FFF800000F807C000003FFF800000F807C000003FFF800000F807C000003FFF800000F 807C000003FFF800000F80FC000003FFF800000FC0F8000003FFF8000007C0F8000003FF F8000007C0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007C000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 0000000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FF FFFFFFFFF800000003FFFFFFFFFFF800004A467CC553>I<0007FFFC000000007FFFFFC0 000001FFFFFFF8000003FFFFFFFE000007FE001FFF000007FF0003FFC0000FFF8001FFE0 000FFF8000FFF0000FFF80007FF0000FFF80007FF8000FFF80007FF80007FF00003FFC00 07FF00003FFC0003FE00003FFC0000F800003FFC00000000003FFC00000000003FFC0000 0000003FFC00000000003FFC00000007FFFFFC000000FFFFFFFC000007FFFFFFFC00003F FFE03FFC0000FFFE003FFC0003FFF0003FFC0007FFC0003FFC000FFF00003FFC001FFE00 003FFC003FFC00003FFC007FF800003FFC007FF800003FFC00FFF000003FFC00FFF00000 3FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000007FFC007FF80000FF FC007FF80001EFFC003FFC0003EFFC003FFF0007CFFF000FFFC03F8FFFF807FFFFFF07FF FC01FFFFFC03FFFC007FFFF001FFFC0003FF80007FF8362E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0 0000000000FFFFC0000000000003FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00FFC0000 0001FFC07FFFC0000001FFC3FFFFF0000001FFCFFFFFFC000001FFDFF00FFF000001FFFF 8003FF800001FFFE0001FFC00001FFF800007FE00001FFF000007FF00001FFE000003FF8 0001FFE000001FFC0001FFE000001FFC0001FFE000001FFE0001FFE000001FFE0001FFE0 00000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF 8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE0 00000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF 8001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000001FFE0001FFE0 00001FFE0001FFE000001FFC0001FFE000001FFC0001FFE000003FF80001FFF000003FF8 0001FFF800007FF00001FFFC0000FFE00001FFFE0001FFC00001FFBF0007FF800001FF1F E01FFE000001FE0FFFFFFC000001FC03FFFFF0000001F800FFFF80000001F0001FF80000 0039487CC742>I<00001FFFC0000000FFFFF8000007FFFFFE00001FFFFFFF80007FFC00 FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF0003FFE00FFE0003FFE00FFE0003 FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC00003E007FF8000000007FF80000 00007FF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF80000 0000FFF800000000FFF800000000FFF800000000FFF800000000FFF8000000007FF80000 00007FF8000000007FFC000000003FFC000000003FFC000000001FFC000000F81FFE0000 00F80FFE000000F80FFF000001F007FF800003F003FFC00007E001FFE0000FC000FFF000 1F80007FFE00FF00001FFFFFFE000007FFFFF8000000FFFFE00000001FFE00002D2E7CAD 35>I<00000000007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000 000000FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000000FFC01 FFC0000000FFFF81FFC0000007FFFFE1FFC000001FFFFFF9FFC000007FFC03FFFFC00000 FFF0007FFFC00001FFC0001FFFC00003FF80000FFFC00007FF000007FFC0000FFE000003 FFC0000FFE000003FFC0001FFC000003FFC0001FFC000003FFC0003FFC000003FFC0003F FC000003FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC000FFF8000003 FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FF F8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003 FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC0003FF8000003FFC0003F FC000003FFC0003FFC000003FFC0001FFC000003FFC0001FFC000003FFC0000FFE000007 FFC00007FF00000FFFC00003FF00001FFFC00001FFC0003FFFC00000FFE000FFFFE00000 7FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FFFF800001FFFF03FFFF8000001FF803 FFFF8039487CC742>I<00001FFE00000001FFFFE0000007FFFFF800001FFFFFFE00007F FC07FF0000FFE001FF8001FFC0007FC003FF80003FE007FF00003FF00FFE00001FF01FFE 00000FF81FFC00000FF83FFC00000FFC3FFC000007FC7FFC000007FC7FF8000007FC7FF8 000007FE7FF8000007FEFFF8000007FEFFF8000007FEFFFFFFFFFFFEFFFFFFFFFFFEFFFF FFFFFFFEFFFFFFFFFFFCFFF800000000FFF800000000FFF800000000FFF8000000007FF8 000000007FF8000000007FFC000000003FFC000000003FFC000000003FFC0000001C1FFE 0000003E0FFE0000003E07FF0000007E07FF000000FC03FF800001F801FFC00003F0007F F0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE000000FFFFF80000000FFF80002F2E 7DAD36>I<00000000001F8000007FF000FFE00007FFFF03FFF0001FFFFFC7FFF0007FFF FFFFC7F800FFE03FFE0FF801FF800FFC0FF803FF0007FE0FF807FE0003FF07F007FE0003 FF07F00FFC0001FF81C00FFC0001FF80000FFC0001FF80001FFC0001FFC0001FFC0001FF C0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0000FFC0001FF80 000FFC0001FF80000FFC0001FF800007FE0003FF000007FE0003FF000003FF0007FE0000 01FF800FFC000000FFE03FF8000001FFFFFFF0000001DFFFFFC0000003C7FFFF00000003 C07FF000000007C0000000000007C0000000000007C0000000000007C0000000000007E0 000000000007F0000000000007F8000000000007FFFFFFF0000007FFFFFFFF000003FFFF FFFFE00003FFFFFFFFF80001FFFFFFFFFE0001FFFFFFFFFF0000FFFFFFFFFF80007FFFFF FFFF8003FFFFFFFFFFC00FFFFFFFFFFFC01FF800001FFFE03FE0000001FFE07FC0000000 7FF07FC00000003FF0FF800000001FF0FF800000001FF0FF800000001FF0FF800000001F F0FF800000001FF07FC00000003FE07FC00000003FE03FE00000007FC03FF0000000FFC0 1FFC000003FF800FFF00000FFF0003FFF000FFFC0000FFFFFFFFF000003FFFFFFFC00000 07FFFFFE000000003FFFC0000035447DAE3B>103 D<007FC00000000000FFFFC0000000 0000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FF C0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000 000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FF C0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000 000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FF C0000000000001FFC0000000000001FFC001FFC0000001FFC00FFFF8000001FFC03FFFFE 000001FFC0FFFFFF000001FFC1FC07FF800001FFC3E003FFC00001FFC7C001FFC00001FF CF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001FFFC0000FFF00001FFF80000FF F00001FFF00000FFF00001FFF00000FFF00001FFF00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFF FFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B487CC742>I<00FC0001FE0003 FF0007FF800FFFC01FFFE01FFFE01FFFE01FFFE01FFFE01FFFE00FFFC007FF8003FF0001 FE0000FC0000000000000000000000000000000000000000000000000000000000000000 0000007FC0FFFFC0FFFFC0FFFFC0FFFFC0FFFFC003FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF18497CC8 20>I<007FC000000000FFFFC000000000FFFFC000000000FFFFC000000000FFFFC00000 0000FFFFC00000000003FFC00000000001FFC00000000001FFC00000000001FFC0000000 0001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000000000 01FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001 FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FF C00000000001FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0 007FFFF801FFC0000FFE0001FFC00007F00001FFC0000FE00001FFC0003FC00001FFC000 7F800001FFC000FE000001FFC001FC000001FFC007F8000001FFC00FF0000001FFC01FC0 000001FFC03F80000001FFC0FF00000001FFC1FE00000001FFC3FF00000001FFCFFF8000 0001FFDFFFC0000001FFFFFFC0000001FFFFFFE0000001FFFFFFF0000001FFFCFFF80000 01FFF87FFC000001FFE03FFC000001FFC01FFE000001FFC01FFF000001FFC00FFF800001 FFC007FFC00001FFC003FFC00001FFC001FFE00001FFC001FFF00001FFC000FFF80001FF C0007FFC0001FFC0003FFC0001FFC0001FFE0001FFC0000FFF0001FFC0000FFF8001FFC0 000FFFC0FFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF80 7FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FFFFC000FFFFC000FFFFC000 03FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000 01FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000FFFFFF80FFFFFF80 FFFFFF80FFFFFF80FFFFFF8019487CC720>I<007FC001FFC00000FFE00000FFFFC00FFF F80007FFFC0000FFFFC03FFFFE001FFFFF0000FFFFC0FFFFFF007FFFFF8000FFFFC1FC07 FF80FE03FFC000FFFFC3E003FFC1F001FFE00003FFC7C001FFC3E000FFE00001FFCF0001 FFE78000FFF00001FFDE0000FFEF00007FF00001FFDC0000FFEE00007FF00001FFFC0000 FFFE00007FF80001FFF80000FFFC00007FF80001FFF00000FFF800007FF80001FFF00000 FFF800007FF80001FFF00000FFF800007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000 FFF000007FF800FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07F FFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF05C2E7CAD63 >I<007FC001FFC00000FFFFC00FFFF80000FFFFC03FFFFE0000FFFFC0FFFFFF0000FFFF C1FC07FF8000FFFFC3E003FFC00003FFC7C001FFC00001FFCF0001FFE00001FFDE0000FF E00001FFDC0000FFE00001FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FF F00000FFF00001FFF00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFF FFE0FFFFFFC07FFFFFE03B2E7CAD42>I<00000FFF0000000000FFFFF000000007FFFFFE 0000001FFFFFFF8000003FFC03FFC00000FFE0007FF00001FF80001FF80003FF00000FFC 0007FE000007FE000FFE000007FF000FFC000003FF001FFC000003FF803FFC000003FFC0 3FF8000001FFC03FF8000001FFC07FF8000001FFE07FF8000001FFE07FF8000001FFE0FF F8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8 000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF07FF800 0001FFE07FF8000001FFE07FF8000001FFE07FF8000001FFE03FFC000003FFC03FFC0000 03FFC01FFC000003FF801FFE000007FF800FFE000007FF0007FF00000FFE0003FF80001F FC0001FFC0003FF80000FFE0007FF000007FFC03FFE000001FFFFFFF80000007FFFFFE00 000000FFFFF0000000000FFF000000342E7DAD3B>I<007FC00FFC000000FFFFC07FFFC0 0000FFFFC3FFFFF00000FFFFCFFFFFFC0000FFFFDFF01FFF0000FFFFFF8007FF800003FF FE0001FFC00001FFF80000FFE00001FFF00000FFF00001FFE000007FF80001FFE000003F FC0001FFE000003FFC0001FFE000003FFE0001FFE000001FFE0001FFE000001FFF0001FF E000001FFF0001FFE000001FFF0001FFE000000FFF0001FFE000000FFF8001FFE000000F FF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FF E000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000F FF0001FFE000001FFF0001FFE000001FFF0001FFE000001FFE0001FFE000001FFE0001FF E000003FFC0001FFE000003FFC0001FFE000007FF80001FFF000007FF80001FFF80000FF F00001FFFC0001FFE00001FFFE0003FFC00001FFFF0007FF800001FFFFE03FFE000001FF EFFFFFFC000001FFE3FFFFF0000001FFE0FFFF80000001FFE01FF800000001FFE0000000 000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FF E0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000 000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FF E00000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC00000 0000FFFFFFC00000000039427CAD42>I<00000FFC0003C0000000FFFF0007C0000007FF FFC00FC000001FFFFFF01FC000003FFE03F81FC00000FFF000FC3FC00001FFE0003E7FC0 0003FFC0001F7FC00007FF80001FFFC0000FFF00000FFFC0000FFF000007FFC0001FFE00 0007FFC0001FFE000003FFC0003FFC000003FFC0003FFC000001FFC0007FFC000001FFC0 007FFC000001FFC0007FF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF800 0001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC0 00FFF8000001FFC000FFF8000001FFC000FFF8000001FFC0007FF8000001FFC0007FFC00 0001FFC0007FFC000001FFC0003FFC000001FFC0003FFC000001FFC0003FFE000003FFC0 001FFE000003FFC0001FFE000007FFC0000FFF00000FFFC00007FF80001FFFC00003FF80 003FFFC00001FFC0007FFFC00000FFF000FFFFC000007FFC07FBFFC000001FFFFFE3FFC0 000007FFFFC3FFC0000001FFFF03FFC00000001FF803FFC0000000000003FFC000000000 0003FFC0000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC0 000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC000000000 0003FFC0000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC0 0000000001FFFFFF8000000001FFFFFF8000000001FFFFFF8000000001FFFFFF80000000 01FFFFFF8039427CAD3F>I<00FF803F8000FFFF80FFF000FFFF83FFFC00FFFF87FFFE00 FFFF8FC3FF00FFFF8F07FF0003FF9E0FFF8001FFBC0FFF8001FFB80FFF8001FFF80FFF80 01FFF00FFF8001FFF007FF0001FFF007FF0001FFE003FE0001FFE000F80001FFE0000000 01FFE000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC0000000FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000 292E7CAD31>I<000FFF00E0007FFFF3E001FFFFFFE007FFFFFFE00FF800FFE01FC0001F E03F80000FE03F000007E07F000003E07F000003E0FF000003E0FF000003E0FF800003E0 FFC0000000FFF0000000FFFE000000FFFFF800007FFFFFC0007FFFFFF0003FFFFFFC001F FFFFFF000FFFFFFF8007FFFFFFC003FFFFFFE000FFFFFFF0003FFFFFF00003FFFFF80000 1FFFF8000000FFFC0000001FFC7800000FFCF8000007FCF8000003FCFC000003FCFC0000 03FCFE000003F8FE000003F8FF000003F8FF800007F0FFC0000FF0FFF0001FE0FFFC00FF C0FFFFFFFF80FC7FFFFE00F81FFFF800E003FF8000262E7CAD2F>I<0001F000000001F0 00000001F000000001F000000001F000000001F000000003F000000003F000000003F000 000007F000000007F000000007F00000000FF00000000FF00000001FF00000003FF00000 003FF00000007FF0000001FFF0000003FFF000000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0FF FFFFFFC0FFFFFFFFC000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0 000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF000 0000FFF0000000FFF0000000FFF0000000FFF001F000FFF001F000FFF001F000FFF001F0 00FFF001F000FFF001F000FFF001F000FFF001F000FFF001F0007FF001E0007FF803E000 3FF803E0003FFC07C0001FFE0F80000FFFFF800007FFFE000001FFFC0000001FF0002442 7EC12E>I<007FE000003FF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF0 00FFFFE0007FFFF000FFFFE0007FFFF00003FFE00001FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00001FFF00001FFE00001FFF00001FFE0 0001FFF00001FFE00003FFF00000FFE00007FFF00000FFE0000F7FF000007FE0001F7FF0 00007FF0003E7FF800003FFC00FC7FFFE0001FFFFFF87FFFE00007FFFFE07FFFE00001FF FF807FFFE000003FFE007FFFE03B2E7CAD42>III<7FFFFF801FFFFF007FFFFF80 1FFFFF007FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF00007FF80001FF0000 007FFC0001FC0000003FFE0003F80000001FFF0007F00000000FFF0007E000000007FF80 0FC000000003FFC01F8000000003FFE03F8000000001FFF07F0000000000FFF8FE000000 00007FF9FC00000000003FFFF800000000003FFFF000000000001FFFE000000000000FFF C0000000000007FFC0000000000003FFC0000000000001FFE0000000000001FFF0000000 000001FFF8000000000003FFFC000000000003FFFE000000000007FFFE00000000000FEF FF00000000001FCFFF80000000003F87FFC0000000007F03FFE000000000FE01FFE00000 0001FC00FFF000000001F8007FF800000003F0007FFC00000007F0003FFE0000000FE000 1FFF0000001FC0000FFF0000003F800007FF800000FF800007FFC000FFFFF8003FFFFFC0 FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC03A2E7EAD 3F>I<7FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807F FFFFC000FFFF8000FFF000000FE00000FFF800000FC00000FFF800000FC000007FFC0000 0F8000007FFC00001F8000003FFC00001F0000003FFE00003F0000001FFE00003E000000 1FFF00007E0000000FFF00007C0000000FFF8000FC00000007FF8000F800000007FFC001 F800000003FFC001F000000003FFE003F000000003FFE003F000000001FFF003E0000000 01FFF007E000000000FFF007C000000000FFF80FC0000000007FF80F80000000007FFC1F 80000000003FFC1F00000000003FFE3F00000000001FFE3E00000000001FFF7E00000000 000FFF7C00000000000FFFFC00000000000FFFFC000000000007FFF8000000000007FFF8 000000000003FFF0000000000003FFF0000000000001FFE0000000000001FFE000000000 0000FFC0000000000000FFC00000000000007F800000000000007F800000000000003F00 0000000000003F000000000000003F000000000000003E000000000000007E0000000000 00007C00000000000000FC000000001F8000F8000000003FC001F8000000007FE001F000 000000FFF003F000000000FFF003E000000000FFF007E000000000FFF00FC000000000FF F01F8000000000FFF03F80000000007FE07F00000000007F43FE00000000003FFFF80000 0000001FFFF0000000000007FFC0000000000001FE00000000000039427EAD3F>I E /FH 45 122 df<70F8FCFC7404040404080810102040060F7C840E>44 DI<70F8F8F87005057C840E>I<01F000071C000C060018030038 03803803807001C07001C07001C07001C0F001E0F001E0F001E0F001E0F001E0F001E0F0 01E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001C07001C07001C07803C038 03803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003 800380038003800380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038007C0FFFE0F217CA018>I<03F8000C1E0010070020 07804007C07807C07803C07807C03807C0000780000780000700000F00000E0000380003 F000001C00000F000007800007800003C00003C00003E02003E07003E0F803E0F803E0F0 03C04003C0400780200780100F000C1C0003F00013227EA018>51 D<000200000600000E00000E00001E00001E00002E00004E00004E00008E00008E00010E 00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E00800E00FFFF F8000E00000E00000E00000E00000E00000E00000E00001F0001FFF015217FA018>I<10 00801E07001FFF001FFE001FF80013E00010000010000010000010000010000010000010 F800130E001407001803801003800001C00001C00001E00001E00001E00001E07001E0F0 01E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013227EA018>I< 007E0001C1000300800601C00E03C01C03C0180180380000380000780000700000700000 F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0F001E0F001E0 7001E07001E07001E03801C03801C01803801C03000C0600070C0001F00013227EA018> I<01F800060E000803001001802001802000C06000C06000C06000C07000C07801803E01 003F02001FC4000FF80003F80003FC00067F00083F80100F803007C06001C06000E0C000 E0C00060C00060C00060C000606000406000C03000801803000E0E0003F00013227EA018 >56 D<01F000060C000C0600180700380380700380700380F001C0F001C0F001C0F001E0 F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E003E1E00001C0 0001C00001C0000380000380300300780700780600700C002018001030000FC00013227E A018>I<0001800000018000000180000003C0000003C0000003C0000005E0000005E000 000DF0000008F0000008F0000010F800001078000010780000203C0000203C0000203C00 00401E0000401E0000401E0000800F0000800F0000FFFF000100078001000780030007C0 020003C0020003C0040003E0040001E0040001E00C0000F00C0000F03E0001F8FF800FFF 20237EA225>65 D<0007E0100038183000E0063001C00170038000F0070000F00E000070 1E0000701C0000303C0000303C0000307C0000107800001078000010F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F800000078000000780000107C000010 3C0000103C0000101C0000201E0000200E000040070000400380008001C0010000E00200 00381C000007E0001C247DA223>67 D69 DI73 D77 DI80 D82 D<03F0200C0C601802603001E07000E06000 60E00060E00060E00020E00020E00020F00000F000007800007F00003FF0001FFE000FFF 0003FF80003FC00007E00001E00000F00000F0000070800070800070800070800070C000 60C00060E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF8780780786007 8018400780084007800840078008C007800C800780048007800480078004800780040007 800000078000000780000007800000078000000780000007800000078000000780000007 800000078000000780000007800000078000000780000007800000078000000780000007 80000007800000078000000FC00003FFFF001E227EA123>I<0FE0001838003C0C003C0E 0018070000070000070000070000FF0007C7001E07003C0700780700700700F00708F007 08F00708F00F087817083C23900FC1E015157E9418>97 D<0E0000FE00001E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F000E61C0 0E80600F00300E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E 0E001C0E003C0E00380F00700C80600C41C0083F0017237FA21B>I<01FE000703000C07 801C0780380300780000700000F00000F00000F00000F00000F00000F00000F000007000 007800403800401C00800C010007060001F80012157E9416>I<0000E0000FE00001E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E001F8E007 04E00C02E01C01E03800E07800E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F0 00E07000E07800E03800E01801E00C02E0070CF001F0FE17237EA21B>I<01FC00070700 0C03801C01C03801C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000F00000 7000007800203800201C00400E008007030000FC0013157F9416>I<003C00C6018F038F 030F070007000700070007000700070007000700FFF80700070007000700070007000700 0700070007000700070007000700070007000700070007807FF8102380A20F>I<000070 01F198071E180E0E181C07001C07003C07803C07803C07803C07801C07001C07000E0E00 0F1C0019F0001000001000001800001800001FFE000FFFC00FFFE03800F0600030400018 C00018C00018C000186000306000303800E00E038003FE0015217F9518>I<0E0000FE00 001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E1F800E60C00E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C 001E003E001E001C00000000000000000000000000000000000E00FE001E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A227FA10E >I<01C003E003E003E001C00000000000000000000000000000000001E00FE001E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F18061803E000B2C82A10F>I<0E0000FE00001E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E03FC0E01 F00E01C00E01800E02000E04000E08000E10000E38000EF8000F1C000E1E000E0E000E07 000E07800E03C00E01C00E01E00E00F00E00F8FFE3FE17237FA21A>I<0E00FE001E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B237FA20E>I<0E1F C07F00FE60E183801E807201C00F003C00E00F003C00E00E003800E00E003800E00E0038 00E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800 E00E003800E00E003800E00E003800E00E003800E00E003800E0FFE3FF8FFE27157F942A >I<0E1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F941B>I< 01FC000707000C01801800C03800E0700070700070F00078F00078F00078F00078F00078 F00078F000787000707800F03800E01C01C00E038007070001FC0015157F9418>I<0E1F 00FE61C00E80600F00700E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E00 1E0E001E0E003C0E003C0E00380F00700E80E00E41C00E3F000E00000E00000E00000E00 000E00000E00000E00000E00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F06 0F000E000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF010157F94 13>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C80 0CC00CC008E018D0308FC00E157E9413>I<02000200020002000600060006000E001E00 3E00FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E04 0E040708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E0070 0E00700E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F0060170 03827800FC7F18157F941B>III121 D E /FI 20 118 df45 D68 D73 D77 D80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E0780000E0 700000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F8000000 7C0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC0000FFFF00 000FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F8000000F8 8000007880000078800000788000007880000078C0000078C0000070E00000F0E00000E0 F00000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E003E001E 0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E001F001E 003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E0478003E 047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700 040F00041E001E1C003E3C003E7C003E7C0008780000F80000F80000F80000F80000F800 00F80000F80000F80000F800007800007C00007C00003C00011E00011E00020F00020700 0403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E00000 01E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E00000 01E0000001E0000001E0000001E0000001E0000001E0001F81E000F061E001C019E00780 05E00F0003E00E0003E01E0001E03C0001E03C0001E07C0001E0780001E0F80001E0F800 01E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0780001E07800 01E03C0001E03C0001E01C0001E01E0003E00E0005E0070009E0038011F000E061FF003F 81FF20327DB125>I<003F800000E0E0000380380007003C000E001E001E001E001C000F 003C000F007C000F0078000F8078000780F8000780F8000780FFFFFF80F8000000F80000 00F8000000F8000000F8000000F8000000780000007C0000003C0000003C0000801E0000 800E0001000F0002000780020001C00C0000F03000001FC000191F7E9E1D>I<0007E000 1C1000383800707C00E07C01E07C01C03803C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C000FFFFC0FFFFC003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00007E0007FFF007F FF0016327FB114>I<000000F0007F030801C1C41C0380E81C070070080F0078001E003C 001E003C003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E003C 000F007800070070000780E00009C1C000087F0000180000001800000018000000180000 00180000001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E0300000 F070000070E0000038E0000038E0000038E0000038E00000387000007070000070380000 E01C0001C00700070001C01C00003FE0001E2F7E9F21>I<07000F801F801F800F800700 000000000000000000000000000000000000000000000780FF80FF800F80078007800780 078007800780078007800780078007800780078007800780078007800780078007800780 0780078007800FC0FFF8FFF80D307EAF12>105 D<0780FE001FC000FF83078060F000FF 8C03C18078000F9001E2003C0007A001E4003C0007A000F4001E0007C000F8001E0007C0 00F8001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000 F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0 001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F000 1E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E 000FC001F8003F00FFFC1FFF83FFF0FFFC1FFF83FFF0341F7E9E38>109 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C000F000 07C000F000078000F000078000F000078000F000078000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F0000FC001 F800FFFC1FFF80FFFC1FFF80211F7E9E25>I<001FC00000F0780001C01C00070007000F 0007801E0003C01C0001C03C0001E03C0001E0780000F0780000F0780000F0F80000F8F8 0000F8F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8780000F07C0001F03C 0001E03C0001E01E0003C01E0003C00F00078007800F0001C01C0000F07800001FC0001D 1F7E9E21>I<0783E0FF8C18FF907C0F907C07A07C07C03807C00007C00007C000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0780000780000780000780000780000780000FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E00010E00010F00010F800007E00 003FF0001FFF000FFFC003FFE0003FF00001F80000F880003C80003C80001CC0001CC000 1CE0001CE00018F00038F00030CC0060C301C080FE00161F7E9E1A>I<00400000400000 400000400000400000C00000C00000C00001C00001C00003C00007C0000FC0001FFFE0FF FFE003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C01003C01003C01003C01003C01003C01003C01003 C01001C02001E02000E0400078C0001F00142C7FAB19>I<078000F000FF801FF000FF80 1FF0000F8001F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078001F000 078001F000078001F000038002F00003C004F00001C008F800007030FF80001FC0FF8021 1F7E9E25>I E /FJ 5 85 df<00000000C00000000000E00000000001E00000000003E0 0000000003E00000000007E00000000007E0000000000FE0000000000FE0000000001FE0 000000001FE00000000037E00000000067E00000000067E000000000C7E000000000C7F0 0000000183F00000000183F00000000303F00000000703F00000000603F00000000C03F0 0000000C03F00000001803F00000001803F00000003003F00000003003F00000006003F0 000000C003F0000000C003F00000018003F00000018003F8000003FFFFF8000003FFFFF8 0000060001F800000E0001F800000C0001F80000180001F80000180001F80000300001F8 0000300001F80000600001F80000E00001F80000C00001F80001C00001F80001C00001F8 0007C00001FC001FC00003FC00FFF8007FFFE0FFF8007FFFE02B327BB135>65 D<000FFFFFFE0000000FFFFFFF800000007F000FE00000007E0003F00000007E0000F800 00007E0000FC0000007E00007C000000FC00003E000000FC00003E000000FC00003F0000 00FC00001F000001F800001F000001F800001F800001F800001F800001F800001F800003 F000001F800003F000001F800003F000001F800003F000001F800007E000003F800007E0 00003F800007E000003F800007E000003F80000FC000003F00000FC000007F00000FC000 007F00000FC000007F00001F8000007E00001F800000FE00001F800000FE00001F800000 FC00003F000001FC00003F000001F800003F000001F800003F000003F000007E000003E0 00007E000007E000007E00000FC000007E00000F800000FC00001F800000FC00003F0000 00FC00007E000000FC0000FC000001F80001F0000001F80003E0000001F8000FC0000003 F8007F000000FFFFFFFC000000FFFFFFE000000031317BB036>68 D<000FFFFFFFFC000FFFFFFFFC00007F0001FC00007E00007C00007E00003C00007E0000 3C00007E0000180000FC0000180000FC0000180000FC0000180000FC0000180001F80000 180001F80000180001F80000180001F80000180003F00080100003F00180000003F00180 000003F00180000007E00300000007E00300000007E00700000007E01F0000000FFFFE00 00000FFFFE0000000FC01E0000000FC00E0000001F800C0000001F800C0000001F800C00 00001F800C0000003F00180000003F00080000003F00000000003F00000000007E000000 00007E00000000007E00000000007E0000000000FC0000000000FC0000000000FC000000 0000FC0000000001F80000000001F80000000001F80000000003F800000000FFFFF00000 00FFFFF00000002E317BB02F>70 D<000FFFFFF000000FFFFFFE0000007F003F8000007E 000FC000007E0007E000007E0003F000007E0001F80000FC0001F80000FC0001F80000FC 0001F80000FC0001F80001F80003F80001F80003F80001F80003F80001F80003F00003F0 0007F00003F00007E00003F0000FC00003F0000FC00007E0001F000007E0007E000007E0 00FC000007E007F000000FFFFFC000000FFFFF0000000FC00F8000000FC003C000001F80 03E000001F8001F000001F8001F000001F8001F800003F0001F800003F0001F800003F00 01F800003F0001F800007E0003F800007E0003F800007E0003F000007E0003F00000FC00 07F00000FC0007F00000FC0007F00800FC0007F00C01F80007F01801F80007F01801F800 03F03003F80003F030FFFFE001F0E0FFFFE000FFC0000000003F002E327BB034>82 D<07FFFFFFFFF00FFFFFFFFFF00FC00FE003F01E000FC000F01C000FC000E018000FC000 E038000FC0006030001F8000E030001F8000E060001F8000C060001F8000C060003F0000 C0C0003F0000C0C0003F0000C0C0003F0000C080007E00008000007E00000000007E0000 0000007E0000000000FC0000000000FC0000000000FC0000000000FC0000000001F80000 000001F80000000001F80000000001F80000000003F00000000003F00000000003F00000 000003F00000000007E00000000007E00000000007E00000000007E0000000000FC00000 00000FC0000000000FC0000000000FC0000000001F80000000001F80000000001F800000 00001F80000000003F00000000003F00000000003F0000000000FF00000000FFFFFF0000 00FFFFFF0000002C3173B033>84 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 795 908 a FJ(D)26 b(R)g(A)f(F)h(T)225 999 y FI(Do)r(cumen)n(t) 20 b(for)i(a)f(Standard)g(Message-P)n(assing)f(In)n(terface)621 1194 y FH(Message)c(P)o(assing)h(In)o(terface)e(F)l(orum)802 1320 y(August)i(14,)f(1993)77 1378 y(This)h(w)o(ork)f(w)o(as)h(supp)q (orted)g(in)f(part)h(b)o(y)e(ARP)l(A)h(and)h(NSF)e(under)h(gran)o(t)h (ASC-9310330,)i(the)192 1436 y(National)d(Science)f(F)l(oundation)i (Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q(erativ)o(e)76 1494 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o(y)e(the)h (Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1552 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 75 377 a FG(Con)m(ten)m(ts)75 645 y FF(Ac)o(kno)o(wledgmen)o (ts)1362 b(1)75 747 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(2)143 804 y FE(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(2)143 860 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(3)143 917 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(3)143 973 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(3)143 1030 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b FD(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b FE(4)143 1086 y(1.6)46 b(Organization)16 b(of)e(this)i(Do)q(cumen)o(t)34 b FD(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(4)75 1188 y FF(2)42 b(MPI)17 b(T)l(erms)f(and)i(Con)o(v)o(en)o (tions)1042 b(5)143 1245 y FE(2.1)46 b(Do)q(cumen)o(t)15 b(Con)o(v)o(en)o(tions)e FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b FE(5)143 1302 y(2.2)46 b(Pro)q(cedure)16 b(Sp)q(eci\014cation)44 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b FE(5)143 1358 y(2.3)46 b(Seman)o(tical)16 b(T)l(erms)f FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(6)143 1415 y(2.4)46 b(Data)14 b(T)o(yp)q(es)30 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b FE(6)248 1471 y(2.4.1)50 b(Opaque)16 b(ob)s(jects)21 b FD(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(6)248 1528 y(2.4.2)50 b(Arra)o(y)14 b(of)h(handles)35 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(7)248 1584 y(2.4.3)50 b(State)14 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(7)248 1641 y(2.4.4)50 b(Named)15 b(constan)o(ts)23 b FD(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(7)248 1697 y(2.4.5)50 b(Choice)20 b FD(:)i(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(8)143 1754 y(2.5)46 b(Pro)q(cesses)40 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(8)143 1810 y(2.6)46 b(Error)14 b(Handling)32 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FE(9)75 1912 y FF(3)42 b(Groups,)17 b(Con)o(texts,)f(and)i(Comm)o(unicators)783 b(11)143 1969 y FE(3.1)46 b(In)o(tro)q(duction)15 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FE(11)143 2025 y(3.2)46 b(Groups)12 b FD(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(12)143 2082 y(3.3)46 b(Con)o(texts)14 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FE(12)143 2138 y(3.4)46 b(Comm)o(unicators)20 b FD(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(13)248 2195 y(3.4.1)50 b(Prede\014ned)17 b(Comm)o(unicators)38 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(13)248 2251 y(3.4.2)50 b(Group)15 b(Managemen)o(t)38 b FD(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FE(13)248 2308 y(3.4.3)50 b(Group)15 b(Accessors)34 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(14)248 2364 y(3.4.4)50 b(Group)15 b(Constructors)37 b FD(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(15)248 2421 y(3.4.5)50 b(Group)15 b(Destructors)27 b FD(:)c(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)69 b FE(17)143 2477 y(3.5)46 b(Con)o(text)14 b(Managemen)o(t)45 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FE(17)248 2534 y(3.5.1)50 b(Con)o(text)14 b(Allo)q(cation)23 b FD(:)g(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(17)248 2591 y(3.5.2)50 b(Con)o(text)14 b(Deallo)q(cation)h FD(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(18)143 2647 y(3.6)46 b(In)o(tra-Comm)o(unication)15 b(Comm)o(unicator)g(Managemen)o (t)31 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(18)248 2704 y(3.6.1)50 b(Comm)o(unicator)15 b(Accessors)43 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(18)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 3 2 2 bop 248 45 a FE(3.6.2)50 b(Comm)o(unicator)15 b(Constructors)10 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(19)248 102 y(3.6.3)50 b(Comm)o(unicator)15 b(Destructors)36 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(21)143 159 y(3.7)46 b(In)o(ter-Comm)o(unication)16 b(Comm)o(unicator)e(Managemen)o(t)34 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(21)248 216 y(3.7.1)50 b(Discussion)18 b FD(:)k(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)69 b FE(21)248 272 y(3.7.2)50 b(Comm)o(unicator)15 b(Accessors)43 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(24)248 329 y(3.7.3)50 b(Comm)o(unicator)15 b(Constructors)10 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(24)248 386 y(3.7.4)50 b(Comm)o(unicator)15 b(Destructors)36 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(25)248 443 y(3.7.5)50 b(Supp)q(ort)16 b(for)e(User-Lev)o(el)j (Serv)o(ers)36 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(25)248 500 y(3.7.6)50 b(Comm)o(unicator)15 b(Constructors)f(using)i(Name)f (Service)32 b FD(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(26)248 556 y(3.7.7)50 b(F)l(urther)15 b(Discussion)29 b FD(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(27)143 613 y(3.8)46 b(Cac)o(heing)12 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(27)248 670 y(3.8.1)50 b(F)l(unctionalit)o(y)32 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(28)143 727 y(3.9)46 b(F)l(ormalizing)16 b(the)f(Lo)q(osely)h(Sync) o(hronous)g(Mo)q(del)g(\(Usage,)e(Safet)o(y\))j FD(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(30)248 784 y(3.9.1)50 b(Basic)16 b(Statemen)o(ts)22 b FD(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FE(30)248 840 y(3.9.2)50 b(Mo)q(dels)16 b(of)f(Execution)29 b FD(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(30)143 897 y(3.10)23 b(Motiv)m(ating)15 b(Examples)h FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(32)248 954 y(3.10.1)27 b(Curren)o(t)15 b(Practice)g(#1)27 b FD(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(32)248 1011 y(3.10.2)27 b(Curren)o(t)15 b(Practice)g(#2)27 b FD(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(32)248 1068 y(3.10.3)27 b(\(Appro)o(ximate\))15 b(Curren)o(t)f(Practice)i(#3) 35 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)69 b FE(33)248 1124 y(3.10.4)27 b(Example)16 b(#4)44 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(33)248 1181 y(3.10.5)27 b(Library)16 b(Example)g(#1)22 b FD(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(34)248 1238 y(3.10.6)27 b(Library)16 b(Example)g(#2)22 b FD(:)g(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)69 b FE(35)248 1295 y(3.10.7)27 b(In)o(ter-Comm)o (unication)16 b(Examples)g FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(37)75 1397 y FF(4)42 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)g(Comm)o(unication) 975 b(42)143 1454 y FE(4.1)46 b(In)o(tro)q(duction)15 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FE(42)143 1511 y(4.2)46 b(Basic)16 b(send)f(op)q(eration) 27 b FD(:)c(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(42)248 1568 y(4.2.1)50 b(Message)15 b(data)26 b FD(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(43)248 1624 y(4.2.2)50 b(Message)15 b(en)o(v)o(elop)q(e)j FD(:)k(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(44)143 1681 y(4.3)46 b(Basic)16 b(receiv)o(e)g(op)q(eration)h FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(45)248 1738 y(4.3.1)50 b(Return)16 b(status)j FD(:)k(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(46)143 1795 y(4.4)46 b(Seman)o(tics)15 b(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)40 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FE(47)143 1852 y(4.5)46 b(Data)14 b(T)o(yp)q(e)h(Matc)o(hing)22 b FD(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)69 b FE(49)143 1908 y(4.6)46 b(Data)14 b(con)o(v)o(ersion)g FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b FE(51)143 1965 y(4.7)46 b(Comm)o(unication)15 b(Mo)q(des)i FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(53)143 2022 y(4.8)46 b(Non)o(blo)q(c)o(king)16 b(comm)o(unication) j FD(:)j(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(55)248 2079 y(4.8.1)50 b(Comm)o(unication)16 b(Ob)s(jects)23 b FD(:)g(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(56)248 2136 y(4.8.2)50 b(Comm)o(unication)16 b(initiation)32 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FE(56)248 2192 y(4.8.3)50 b(Comm)o(unication)16 b(Completion)i FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(57)248 2249 y(4.8.4)50 b(Multiple)17 b(Completions)41 b FD(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(59)143 2306 y(4.9)46 b(Prob)q(e)15 b(and)g(Cancel)g FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(62)143 2363 y(4.10)23 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(ob)s(jects)i FD(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(65)143 2420 y(4.11)23 b(Send-receiv)o(e)d FD(:)i(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(68)143 2476 y(4.12)23 b(Null)16 b(pro)q(cesses)i FD(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(70)143 2533 y(4.13)23 b(Deriv)o(ed)15 b(datat)o(yp)q(es)42 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(71)248 2590 y(4.13.1)27 b(Datat)o(yp)q(e)14 b(constructors)27 b FD(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FE(72)248 2647 y(4.13.2)27 b(Additional)17 b(functions)30 b FD(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(77)248 2704 y(4.13.3)27 b(Use)16 b(of)e(general)i(datat)o(yp)q(es) f(in)h(comm)o(unication)36 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)69 b FE(79)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 4 3 3 bop 248 45 a FE(4.13.4)27 b(Examples)33 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(80)248 102 y(4.13.5)27 b(Correct)15 b(use)g(of)g(addresses)26 b FD(:)d(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(85)143 160 y(4.14)23 b(Univ)o(ersal)16 b(comm)o(unication)g(functions)24 b FD(:)f(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FE(86)248 217 y(4.14.1)27 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(ob)s(jects)e FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FE(87)248 274 y(4.14.2)27 b(Non)o(blo)q(c)o(king)17 b(comm)o(unication)e(initiation)31 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(87)248 332 y(4.14.3)27 b(Comm)o(unication)16 b(completion)31 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(88)248 389 y(4.14.4)27 b(Blo)q(c)o(king)17 b(comm)o(unication)g FD(:)23 b(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FE(88)248 446 y(4.14.5)27 b(Prob)q(e)16 b(and)f(cancel)23 b FD(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(88)248 503 y(4.14.6)27 b(Return)16 b(status)j FD(:)k(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(88)248 561 y(4.14.7)27 b(send-receiv)o(e)17 b(and)e(exc)o(hange)22 b FD(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)69 b FE(88)248 618 y(4.14.8)27 b(Deriv)o(ed)16 b(datat)o(yp)q(es)37 b FD(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(89)248 675 y(4.14.9)27 b(Summary)33 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(89)75 779 y FF(5)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(90)143 836 y FE(5.1)46 b(In)o(tro)q(duction)15 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(90)143 893 y(5.2)46 b(Comm)o(unication)15 b(F)l(unctions)24 b FD(:)f(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(91)143 950 y(5.3)46 b(Barrier)15 b(sync)o(hronization)44 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(91)143 1008 y(5.4)46 b(Data)14 b(mo)o(v)o(e)g(functions)29 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(91)248 1065 y(5.4.1)50 b(Broadcast)24 b FD(:)e(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(93)248 1122 y(5.4.2)50 b(Gather)13 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(93)248 1179 y(5.4.3)50 b(Scatter)c FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b FE(94)248 1237 y(5.4.4)50 b(All-to-all)17 b(broadcast)d FD(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(94)248 1294 y(5.4.5)50 b(All-to-all)17 b(scatter-gather)36 b FD(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FE(95)143 1351 y(5.5)46 b(Global)16 b(Compute)e(Op)q(erations)20 b FD(:)i(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b FE(95)248 1409 y(5.5.1)50 b(Reduce)45 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FE(96)248 1466 y(5.5.2)50 b(Scan)25 b FD(:)d(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(100)143 1523 y(5.6)g(Correctness)32 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(101)75 1626 y FF(6)c(Pro)q(cess)17 b(T)l(op)q(ologies)1224 b(105)143 1684 y FE(6.1)46 b(In)o(tro)q(duction)15 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FE(105)143 1741 y(6.2)g(Virtual)16 b(T)l(op)q(ologies)37 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(106)143 1798 y(6.3)g(Em)o(b)q(edding)16 b(in)g(MPI)22 b FD(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(106)143 1856 y(6.4)g(Ov)o(erview)16 b(of)e(the)i(prop)q(osed)f (MPI)g(functions)20 b FD(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(107)248 1913 y(6.4.1)k(Lo)o(w{lev)o(el)16 b(top)q(ology)f(functions)37 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b FE(108)248 1970 y(6.4.2)k(High{lev)o(el)17 b(top)q(ology)e(functions)25 b FD(:)d(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(109)248 2027 y(6.4.3)k(T)l(op)q(ology)15 b(inquiry)i(functions)42 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(111)248 2085 y(6.4.4)k(P)o (artitioning)16 b(of)e(Cartesian)h(structures)45 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(112)143 2142 y(6.5)g(A)15 b(simple)i(example)f FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(114)75 2245 y FF(7)c(Language)19 b(Binding)1242 b(116)143 2303 y FE(7.1)46 b(In)o(tro)q(duction)15 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FE(116)143 2360 y(7.2)g(Data)14 b(T)o(yp)q(es)h(and)g (Naming)h(Con)o(v)o(en)o(tions)43 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(116)143 2417 y(7.3)g(F)l(ortran)14 b(77)g(Binding)j(Issues)34 b FD(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(117)143 2474 y(7.4)g(C)15 b(Binding)i(Issues)35 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(117)143 2532 y(7.5)g(Sp)q(eci\014c)17 b(Bindings)32 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(117)248 2589 y(7.5.1)k(C)15 b(bindings)i(for)e(P)o(oin)o(t-to-P)o(oin)o(t)f(Comm)o (unication)k FD(:)k(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(118)248 2646 y(7.5.2)k(C)15 b(Bindings)i(for)e(Collectiv)o(e)h (Comm)o(unication)g(Chapter)29 b FD(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b FE(120)248 2704 y(7.5.3)k(F)l(ortran)14 b(Bindings)j(for)e (P)o(oin)o(t-to-P)o(oin)o(t)f(Routines)34 b FD(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(121)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 5 4 4 bop 248 45 a FE(7.5.4)50 b(F)l(ortran)14 b(Bindings)j(for)e (Collectiv)o(e)h(Comm)o(unciation)g(Chapter)i FD(:)k(:)h(:)f(:)g(:)h(:) f(:)46 b FE(123)75 147 y FF(8)c(En)o(vironmen)o(tal)18 b(Managemen)o(t)f(and)h(Inquiry)717 b(125)143 203 y FE(8.1)46 b(In)o(tro)q(duction)15 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(125)248 260 y(8.1.1)k(Program) 14 b(Startup)h(Issues)36 b FD(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(125)143 316 y(8.2)g(MPI)15 b(Program)f(Startup)g(Prop)q(osal{MPI)p 995 316 14 2 v 17 w(INIT)39 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(127)143 373 y(8.3)g(MPI-Sp)q(eci\014c)17 b(Enquiry)f(F)l(unctions)f FD(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(127)248 429 y(8.3.1)k(P)o(arallel)16 b(programming)39 b FD(:)22 b(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b FE(129)248 486 y(8.3.2)k(non-MPI)d FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(130)75 588 y FF(9)c(Pro\014ling)1458 b(131)143 644 y FE(9.1)46 b(Requiremen)o(ts)28 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(131)143 701 y(9.2)g(Discussion)21 b FD(:)i(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(131)143 757 y(9.3)g(Logic)16 b(of)e(the)i(design)29 b FD(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(132)143 813 y(9.4)g(Examples)36 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b FE(132)248 870 y(9.4.1)k(Pro\014ler)16 b(implemen)o(tation)35 b FD(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(132)248 926 y(9.4.2)k(MPI)15 b(library)h(implemen)o(tation)h FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(132)143 983 y(9.5)g(Multiple)17 b(lev)o(els)f(of)f(in)o(terception)25 b FD(:)d(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FE(134)75 1085 y FF(10)16 b(Initial)k(Implemen)o(tation)e (Subset)967 b(135)143 1141 y FE(10.1)23 b(In)o(tro)q(duction)15 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FE(135)143 1198 y(10.2)23 b(Criteria)15 b(and)h(Rational)c FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b FE(136)143 1254 y(10.3)23 b(Subset)15 b(F)l(unctionalit)o (y)31 b FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(137)248 1311 y(10.3.1)27 b(P)o(oin)o(t)15 b(to)g(P)o(oin)o(t)f(F)l (unctionalit)o(y)i FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(137)248 1367 y(10.3.2)27 b(Collectiv)o(e)17 b(Comm)o(unication)e(F)l (unctionalit)o(y)24 b FD(:)e(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FE(138)248 1423 y(10.3.3)27 b(T)l(op)q(ology)15 b(F)l(unctionalit)o(y)46 b FD(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(139)248 1480 y(10.3.4)27 b(Language)15 b(Binding)41 b FD(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(140)248 1536 y(10.3.5)27 b(En)o(vironmen)o(tal)16 b(Managemen)o(t)e(and)h (Inquiry)g FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FE(140)248 1593 y(10.3.6)27 b(Pro\014ling)16 b(F)l(unctionalit)o(y)24 b FD(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(140)248 1649 y(10.3.7)27 b(Maps,)15 b(Groups)f(and)i(Con)o(texts)f FD(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FE(140)248 1706 y(10.3.8)27 b(Subset)16 b(T)l(esting)38 b FD(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b FE(140)75 1808 y FF(MPI)17 b(F)l(unction)h(Index)1252 b(143)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 0 6 0 5 bop 875 722 a FF(Abstract)75 828 y FE(The)18 b(Message)f(P)o (assing)h(In)o(terface)f(F)l(orum)h(\(MPIF\),)e(with)i(participation)h (from)e(o)o(v)o(er)g(40)g(organiza-)75 885 y(tions,)f(has)h(b)q(een)g (meeting)g(since)h(Jan)o(uary)e(1993)f(to)h(discuss)h(and)g(de\014ne)g (a)f(set)h(of)f(library)h(in)o(terface)75 941 y(standards)f(for)h (message)f(passing.)25 b(MPIF)16 b(is)i(not)e(sanctioned)i(or)e(supp)q (orted)h(b)o(y)g(an)o(y)g(o\016cial)g(stan-)75 998 y(dards)e (organization.)166 1054 y(This)22 b(is)f(a)g(draft)f(of)h(what)g(will)h (b)q(ecome)g(the)f(Final)h(Rep)q(ort,)h(V)l(ersion)f(1.0,)f(of)f(the)i (Message)75 1111 y(P)o(assing)c(In)o(terface)h(F)l(orum.)29 b(This)19 b(do)q(cumen)o(t)g(con)o(tains)g(all)g(the)g(tec)o(hnical)h (features)e(prop)q(osed)h(for)75 1167 y(the)c(in)o(terface.)20 b(This)c(cop)o(y)f(of)g(the)g(draft)g(w)o(as)f(pro)q(cessed)i(b)o(y)f (L)1174 1161 y FB(a)1195 1167 y FE(T)1220 1181 y(E)1246 1167 y(X)g(on)g(August)g(14,)g(1993.)166 1224 y(MPIF)j(in)o(vites)h (commen)o(ts)f(on)h(the)f(tec)o(hnical)i(con)o(ten)o(t)e(of)g(MPI,)g (as)g(w)o(ell)i(as)e(on)g(the)h(editorial)75 1280 y(presen)o(tation)14 b(in)g(the)g(do)q(cumen)o(t.)19 b(Commen)o(ts)13 b(receiv)o(ed)i(b)q (efore)e(July)i(1,)e(1993)g(will)i(b)q(e)f(considered)h(in)75 1336 y(pro)q(ducing)i(the)e(\014nal)h(draft)e(of)h(V)l(ersion)h(1.0)e (of)h(the)g(Message)g(P)o(assing)g(In)o(terface)g(Sp)q(eci\014cation.) 166 1393 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o (terface,)g(simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g (used)75 1449 y(standard)g(for)f(writing)i(message-passing)f(programs.) 24 b(As)17 b(suc)o(h)h(the)f(in)o(terface)g(should)h(establishing)75 1506 y(a)d(practical,)h(p)q(ortable,)f(e\016cien)o(t,)g(and)h (\015exible)h(standard)e(for)f(message)h(passing.)p eop %%Page: 1 7 1 6 bop 75 75 a FG(Ac)m(kno)m(wledgmen)m(ts)166 282 y FE(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f(w)o(as)e(carried)i(out)f (b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o(as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d) h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P)o(assing)g(In)o (terface)75 395 y(\(MPI\),)f(man)o(y)h(p)q(eople)i(serv)o(ed)e(in)h(p)q (ositions)g(of)f(resp)q(onsibilit)o(y:)143 478 y FA(\017)23 b FE(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o(er,)g(Con)o(v)o (eners)g(and)g(Meeting)h(Chairs)143 566 y FA(\017)23 b FE(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes)143 653 y FA(\017)23 b FE(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i (Lusk,)f(P)o(oin)o(t)g(to)g(P)o(oin)o(t)f(Comm)o(unications)143 741 y FA(\017)23 b FE(Al)16 b(Geist,)e(Marc)h(Snir,)h(Collectiv)o(e)g (Comm)o(unications)143 828 y FA(\017)23 b FE(Rolf)15 b(Hemp)q(el,)i(Pro)q(cess)e(T)l(op)q(ologies)143 915 y FA(\017)23 b FE(Scott)14 b(Berryman,)h(Ewing)g(Lusk,)h(Language)f (Binding)143 1003 y FA(\017)23 b FE(Ewing)15 b(Lusk,)h(Correctness)143 1090 y FA(\017)23 b FE(William)17 b(Gropp,)d(En)o(vironmen)o(tal)i (Managemen)o(t)e(and)h(Inquiry)143 1177 y FA(\017)23 b FE(James)15 b(Co)o(wnie,)g(Pro\014ling)143 1265 y FA(\017)23 b FE(T)l(on)o(y)16 b(Skjellum,)h(Lyndon)g(Clark)o(e,)f(Marc)g(Snir,)h (Ric)o(hard)g(Little\014eld,)i(Marc)d(Sears,)g(Comm)o(u-)189 1321 y(nication)g(Con)o(texts)143 1409 y FA(\017)23 b FE(Stev)o(en)15 b(Huss-Lederman,)h(Initial)h(Implemen)o(tation)f (Subset)143 1496 y FA(\017)23 b FE(Stev)o(e)15 b(Otto,)f(Editor.)166 1580 y(Do)h(w)o(e)f(w)o(an)o(t)h(an)g(organization/compan)o(y)f(list)i (here?)166 1636 y(The)22 b(follo)o(wing)h(list)f(includes)i(some)e(of)f (the)h(activ)o(e)g(participan)o(ts)h(in)f(the)g(MPI)g(pro)q(cess)g(not) 75 1693 y(men)o(tioned)16 b(ab)q(o)o(v)o(e:)100 1736 y(Ed)f(Anderson)245 b(Jo)q(e)16 b(Baron)206 b(Eric)15 b(Barszcz)100 1793 y(Rob)g(Bjornson)230 b(Anne)16 b(Elster)173 b(Jim)16 b(F)l(eeney)100 1849 y(Vince)g(F)l(ernando)192 b(Sam)15 b(Fineb)q(erg)135 b(Jon)15 b(Flo)o(w)o(er)100 1906 y(Daniel)h(F)l(ry)o(e)274 b(Ian)16 b(Glendinning)90 b(Adam)15 b(Green)o(b)q(erg)100 1962 y(Rob)q(ert)g(Harrison)180 b(Leslie)17 b(Hart)190 b(T)l(om)14 b(Haupt)100 2019 y(Don)h(Heller)290 b(T)l(om)15 b(Henderson)97 b(C.)14 b(T.)h(Ho)o(w)o(ard)f(Ho)100 2075 y(Stev)o(en)h(Huss-Lederman)51 b(John)16 b(Kap)q(enga)123 b(Bob)15 b(Leary)100 2131 y(Arth)o(ur)g(Maccab)q(e)178 b(P)o(eter)15 b(Madams)120 b(Alan)16 b(Main)o(w)o(aring)100 2188 y(Oliv)o(er)g(McBry)o(an)185 b(Phil)17 b(McKinley)123 b(Charles)15 b(Mosher)100 2244 y(Dan)g(Nessett)260 b(P)o(eter)15 b(P)o(ac)o(heco)129 b(Ho)o(w)o(ard)14 b(P)o(almer)100 2301 y(P)o(aul)h(Pierce)278 b(Sanja)o(y)15 b(Rank)m(a)137 b(P)o(eter)14 b(Rigsb)q(ee)100 2357 y(Arc)o(h)h(Robison)234 b(Eric)o(h)16 b(Sc)o(hikuta)118 b(Am)o(buj)15 b(Singh)100 2414 y(Alan)h(Sussman)222 b(Rob)q(ert)16 b(T)l(omlinson)51 b(Rob)q(ert)15 b(G.)g(V)l(oigt)100 2470 y(Dennis)h(W)l(eeks)228 b(Stephen)17 b(Wheat)102 b(Stev)o(e)15 b(Zenith)166 2525 y(The)d(follo)o(wing)h(organizations)f(made)g(the)h(draft)e(a)o(v)m (ailable)j(b)o(y)e(anon)o(ymous)g(FTP)f(access)i(and/or)75 2582 y(mail)h(serv)o(ers:)k(Oak)c(Ridge)g(National)g(Lab)q(oratory)l(,) e(These)i(outlets)f(w)o(ere)g(instrumen)o(tal)h(in)g(distribut-)75 2638 y(ing)i(the)f(do)q(cumen)o(t.)166 2695 y(T)l(alk)g(ab)q(out)g (\014nancial)i(supp)q(ort)f(here.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 8 2 7 bop 75 356 a Fz(Section)35 b(1)75 564 y FG(In)m(tro)s(duction)41 b(to)g(MPI)75 805 y Fy(1.1)59 b(Overview)19 b(and)g(Goals)75 906 y FE(Message)11 b(passing)i(is)f(a)g(paradigm)f(used)i(widely)g(on) f(certain)h(classes)f(of)f(parallel)j(mac)o(hines;)f(esp)q(ecially)75 963 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1019 y(pro)q(cesses)12 b(comm)o(unicating)h(through)e(messages)g(is) i(w)o(ell)f(understo)q(o)q(d.)20 b(Ov)o(er)12 b(the)f(last)h(ten)g(y)o (ears,)f(sub-)75 1076 y(stan)o(tial)16 b(progress)g(has)g(b)q(een)h (made)f(in)h(casting)g(signi\014can)o(t)g(applications)h(in)f(this)f (paradigm.)23 b(Eac)o(h)75 1132 y(v)o(endor)c(has)h(implemen)o(ted)h (their)f(o)o(wn)f(v)m(arian)o(t.)33 b(More)19 b(recen)o(tly)l(,)j(sev)o (eral)d(systems)g(ha)o(v)o(e)h(demon-)75 1188 y(strated)12 b(that)h(a)g(message)f(passing)i(system)f(can)g(b)q(e)h(e\016cien)o (tly)h(and)e(b)q(e)h(p)q(ortably)f(implemen)o(ted.)21 b(It)13 b(is)75 1245 y(th)o(us)h(an)f(appropriate)h(time)g(to)f(try)g (to)g(de\014ne)i(b)q(oth)f(the)g(syn)o(tax)f(and)h(seman)o(tics)g(of)f (a)g(core)h(of)f(library)75 1301 y(routines)h(that)e(will)j(b)q(e)f (useful)h(to)e(a)g(wide)h(range)f(of)g(users)g(and)h(e\016cien)o(tly)g (implemen)o(table)i(on)d(a)g(wide)75 1358 y(range)i(of)g(computers.)166 1414 y(In)21 b(designing)i(MPI)e(w)o(e)f(ha)o(v)o(e)h(sough)o(t)f(to)g (mak)o(e)g(use)h(of)g(the)g(most)f(attractiv)o(e)f(features)i(of)f(a)75 1471 y(n)o(um)o(b)q(er)12 b(of)g(existing)h(message)e(passing)h (systems,)g(rather)f(than)h(selecting)h(one)f(of)g(them)g(and)g (adopting)75 1527 y(it)f(as)f(the)h(standard.)18 b(Th)o(us,)11 b(MPI)g(has)f(b)q(een)i(strongly)f(in\015uenced)i(b)o(y)d(w)o(ork)g(at) g(the)h(IBM)g(T.)f(J.)h(W)l(atson)75 1584 y(Researc)o(h)18 b(Cen)o(ter)g([1)o(,)g(2)o(],)g(In)o(tel's)g(NX/2)g([15)o(],)f(Express) h([14)o(],)g(nCUBE's)g(V)l(ertex)g([12)o(],)f(P4)h([3)o(],)g(and)75 1640 y(P)l(ARMA)o(CS)11 b([10)o(,)g(11)o(].)18 b(Other)11 b(imp)q(ortan)o(t)g(con)o(tributions)g(ha)o(v)o(e)g(come)g(from)f(Zip)q (co)q(de)i([16)o(,)f(17)o(],)g(Chimp)75 1697 y([6)o(,)k(7],)f(PVM)h([8) o(,)g(18],)f(and)h(PICL)h([9].)166 1753 y(One)j(of)f(the)g(ob)s(jectiv) o(es)g(of)g(this)g(pap)q(er)h(is)g(to)e(promote)h(a)f(discussion)j (within)g(the)e(concurren)o(t)75 1810 y(computing)d(researc)o(h)f(comm) o(unit)o(y)f(of)h(the)g(issues)h(that)f(m)o(ust)f(b)q(e)i(addressed)f (in)h(establishing)h(a)e(prac-)75 1866 y(tical,)j(p)q(ortable,)g(and)f (\015exible)j(standard)d(for)g(message)f(passing.)24 b(This)17 b(co)q(op)q(erativ)o(e)g(pro)q(cess)g(b)q(egan)75 1922 y(with)f(a)e(w)o(orkshop)h(on)g(standards)g(for)f(message)h (passing)h(held)g(in)g(April)h(1992)d([19)o(].)166 1979 y(The)k(main)h(adv)m(an)o(tages)e(of)h(establishing)i(a)e(message)g (passing)g(standard)g(are)g(p)q(ortabilit)o(y)h(and)75 2035 y(ease-of-use.)h(In)14 b(a)g(distributed)h(memory)e(comm)o (unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f(higher)h(lev)o (el)75 2092 y(routines)c(and/or)g(abstractions)f(are)h(build)i(up)q(on) e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i(routines)f(the)g(b)q (ene\014ts)75 2148 y(of)22 b(standardization)h(are)f(particularly)h (apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g (message)75 2205 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop) q(osed)g(here,)h(pro)o(vides)f(v)o(endors)h(with)f(a)g(clearly)h (de\014ned)h(base)75 2261 y(set)10 b(of)g(routines)g(that)g(they)g(can) h(implemen)o(t)g(e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o (vide)f(hardw)o(are)g(supp)q(ort)75 2318 y(for,)k(thereb)o(y)h (enhancing)i(scalabilit)o(y)l(.)166 2374 y(The)h(goal)g(of)g(the)g (Message)g(P)o(assing)g(In)o(terface)g(simply)i(stated)d(is)i(to)f(dev) o(elop)h(a)f(widely)h(used)75 2431 y(standard)e(for)f(writing)i (message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface) g(should)h(establishing)75 2487 y(a)d(practical,)h(p)q(ortable,)f (e\016cien)o(t,)g(and)h(\015exible)h(standard)e(for)f(message)h (passing.)166 2543 y(A)g(complete)h(list)g(of)f(goals)g(follo)o(w.)143 2647 y FA(\017)23 b FE(Design)11 b(an)f(application)i(programming)e(in) o(terface)h(\(not)f(necessarily)i(for)d(compilers)j(or)e(a)g(system)189 2704 y(implemen)o(tation)16 b(library\).)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 9 3 8 bop 75 -100 a Fx(1.2.)34 b(WHO)15 b(SHOULD)h(USE)g(THIS)g(ST)l(AND) o(ARD?)824 b FE(3)143 45 y FA(\017)23 b FE(Allo)o(w)15 b(e\016cien)o(t)h(comm)o(unication:)21 b(Av)o(oid)16 b(memory)e(to)h(memory)g(cop)o(ying)g(and)h(allo)o(w)f(o)o(v)o(erlap) 189 102 y(of)e(computation)g(and)h(comm)o(unication)g(and)g(o\017oad)f (to)g(comm)o(unication)h(copro)q(cessor,)g(where)189 158 y(a)o(v)m(ailable.)143 255 y FA(\017)23 b FE(Allo)o(w)15 b(\(but)g(no)h(mandate\))e(extensions)i(for)f(use)g(in)h(heterogeneous) f(en)o(vironmen)o(t.)143 352 y FA(\017)23 b FE(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C,)f(F)l(ortran)f(77,)g(F)l(ortran)g(90,)g(and) i(C++)f(bindings)i(for)e(in)o(terface.)143 449 y FA(\017)23 b FE(Assume)15 b(a)g(reliable)i(comm)o(unication)f(in)o(terface:)k (User)15 b(need)h(not)f(cop)q(e)h(with)f(comm)o(unication)189 506 y(failures.)21 b(Suc)o(h)15 b(failures)i(are)d(dealt)i(b)o(y)f(the) h(underlying)h(comm)o(unication)f(subsystem.)143 603 y FA(\017)23 b FE(F)l(o)q(cus)15 b(on)g(a)g(prop)q(osal)h(that)e(can)h (b)q(e)h(agreed)f(up)q(on)h(in)g(6)f(mon)o(ths.)143 700 y FA(\017)23 b FE(De\014ne)f(an)g(in)o(terface)g(that)f(is)h(not)f(to)q (o)g(di\013eren)o(t)h(from)f(curren)o(t)h(practice,)h(suc)o(h)f(as)g (PVM,)189 756 y(Express,)14 b(P4,)h(etc.)143 853 y FA(\017)23 b FE(De\014ne)12 b(an)f(in)o(terface)h(that)f(can)h(b)q(e)g(quic)o(kly) h(implemen)o(ted)g(on)f(man)o(y)f(v)o(endor's)g(platforms,)h(with)189 909 y(no)j(signi\014can)o(t)h(c)o(hanges)f(in)h(the)f(underlying)j (comm)o(unication)d(and)h(system)f(soft)o(w)o(are.)143 1007 y FA(\017)23 b FE(The)15 b(in)o(terface)h(should)g(not)f(con)o (tain)g(more)g(functions)h(than)f(are)g(really)h(necessary)l(.)143 1104 y FA(\017)23 b FE(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h (should)g(b)q(e)g(language)f(indep)q(enden)o(t.)75 1251 y Fy(1.2)59 b(Who)20 b(Should)g(Use)g(This)f(Standa)n(rd?)75 1354 y FE(This)d(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h(all) g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 1411 y(programs)21 b(in)j(F)l(ortran)d(77,)i(C,)f (F)l(ortran)g(90,)h(or)f(C++.)43 b(This)23 b(includes)i(individual)g (application)75 1467 y(programmers,)14 b(dev)o(elop)q(ers)j(of)e(soft)o (w)o(are)f(designed)j(to)e(run)h(on)g(parallel)h(mac)o(hines,)f(and)g (creators)e(of)75 1524 y(en)o(vironmen)o(ts,)g(and)f(to)q(ols.)19 b(In)14 b(order)f(to)g(b)q(e)h(attractiv)o(e)e(to)h(this)h(wide)g (audience,)h(the)f(standard)f(m)o(ust)75 1580 y(pro)o(vide)j(a)f (simple,)i(easy-to-use)e(in)o(terface)h(for)e(the)i(basic)g(user)f (while)i(not)e(seman)o(tically)i(precluding)75 1637 y(the)e(high-p)q (erformance)i(message-passing)e(op)q(erations)g(a)o(v)m(ailable)i(on)e (adv)m(anced)i(mac)o(hines.)75 1784 y Fy(1.3)59 b(What)20 b(Platfo)n(rms)g(Are)g(T)-5 b(a)n(rgets)19 b(F)n(o)n(r)i(Implementat)o (ion?)75 1887 y FE(The)c(attractiv)o(eness)f(of)g(the)h (message-passing)g(paradigm)f(at)g(least)h(partially)h(stems)e(from)g (its)h(wide)75 1944 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o(y)f(ma)o(y)g(run)h(on)g (distributed-memory)h(m)o(ultipro)q(cessors,)75 2000 y(net)o(w)o(orks)i(of)i(w)o(orkstations,)e(and)i(com)o(binations)g(of)g (all)g(of)f(these.)34 b(In)20 b(addition,)h(shared-memory)75 2057 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 2113 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 2170 y(th)o(us)d(should)i(b)q(e)f(b)q(oth) g(p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f (on)h(a)f(great)g(v)m(ariet)o(y)g(of)75 2226 y(mac)o(hines,)i (including)h(those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h (of)e(other)h(mac)o(hines,)g(parallel)75 2283 y(or)d(not,)f(connected)i (b)o(y)f(a)g(comm)o(unication)h(net)o(w)o(ork.)166 2340 y(The)21 b(in)o(terface)f(is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i (general)f(MIMD)f(programs,)g(as)g(w)o(ell)h(as)f(those)75 2396 y(written)d(in)g(the)g(more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g (Although)h(no)f(explicit)j(supp)q(ort)e(for)f(threads)g(is)75 2453 y(pro)o(vided,)g(the)f(in)o(terface)g(has)g(b)q(een)i(designed)f (so)f(as)g(not)g(to)f(prejudice)j(their)f(use.)75 2601 y Fy(1.4)59 b(What)20 b(Is)f(Included)g(In)g(The)g(Standa)n(rd?)75 2704 y FE(The)c(standard)g(includes)j(\(this)d(is)h(temp)q(orarily)f (as)g(inclusiv)o(e)j(as)d(p)q(ossible\):)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 10 4 9 bop 75 -100 a FE(4)925 b Fx(SECTION)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)143 45 y FA(\017)23 b FE(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 139 y FA(\017)23 b FE(Collectiv)o(e)16 b(op)q(erations)143 233 y FA(\017)23 b FE(Pro)q(cess)15 b(groups)143 327 y FA(\017)23 b FE(Comm)o(unication)15 b(con)o(texts)143 420 y FA(\017)23 b FE(A)15 b(simple)i(w)o(a)o(y)d(to) g(create)h(pro)q(cesses)h(for)e(the)i(SPMD)e(mo)q(del)143 514 y FA(\017)23 b FE(Bindings)17 b(for)d(F)l(ortran)g(77,)g(F)l (ortran)h(90,)f(C)h(and)g(C++)143 608 y FA(\017)23 b FE(A)15 b(mo)q(del)h(implemen)o(tation)143 702 y FA(\017)23 b FE(A)15 b(formal)g(sp)q(eci\014cation.)143 796 y FA(\017)23 b FE(Pro)q(cess)15 b(top)q(ology)143 889 y FA(\017)23 b FE(A)15 b(v)m(alidation)i(suite)75 1033 y Fy(1.5)59 b(What)20 b(Is)f(Not)h(Included)e(In)i(The)f(Standa)n(rd?)75 1134 y FE(The)c(standard)g(do)q(es)h(not)f(sp)q(ecify:)143 1240 y FA(\017)23 b FE(Explicit)17 b(shared-memory)e(op)q(erations)143 1334 y FA(\017)23 b FE(Op)q(erations)c(that)e(require)i(more)f(op)q (erating)h(system)f(supp)q(ort)g(than)g(is)h(curren)o(tly)g(standard;) 189 1391 y(for)14 b(example,)i(in)o(terrupt-driv)o(en)g(receiv)o(es,)g (remote)f(execution,)h(or)e(activ)o(e)i(messages)143 1484 y FA(\017)23 b FE(Program)13 b(construction)j(to)q(ols)143 1578 y FA(\017)23 b FE(Debugging)15 b(facilities)143 1672 y FA(\017)23 b FE(Auxiliary)17 b(functions)e(suc)o(h)h(as)f (timers)143 1766 y FA(\017)23 b FE(Explicit)17 b(supp)q(ort)e(for)g (threads)166 1872 y(There)g(are)f(man)o(y)h(features)f(that)g(ha)o(v)o (e)g(b)q(een)i(considered)g(and)f(not)g(included)i(in)f(this)f (standard.)75 1929 y(This)j(happ)q(ened)h(for)e(a)g(n)o(um)o(b)q(er)h (of)f(reasons,)g(one)g(of)g(whic)o(h)i(is)f(the)f(time)h(constrain)o(t) f(that)g(w)o(as)f(self)75 1985 y(imp)q(osed)g(in)h(\014nishing)g(the)e (standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g(included)j(can)e (alw)o(a)o(ys)e(b)q(e)i(o\013ered)f(as)75 2042 y(extensions)h(b)o(y)f (sp)q(eci\014c)i(implemen)o(tations.)75 2185 y Fy(1.6)59 b(Organization)20 b(of)g(this)f(Do)r(cument)75 2286 y FE(Describ)q(e)d(where)g(ev)o(erything)f(is)h(lo)q(cated)g(in)g(the)f (MPIF)g(do)q(cumen)o(t.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 11 5 10 bop 75 358 a Fz(Section)35 b(2)75 568 y FG(MPI)40 b(T)-10 b(erms)39 b(and)g(Con)m(v)m(en)m(tions)75 775 y Fw(1)166 850 y FE(This)14 b(Section)h(explains)g(notational)f(terms)f (and)h(con)o(v)o(en)o(tions)g(used)h(throughout)e(the)h(MPI)f(do)q(c-) 75 906 y(umen)o(t,)i(some)g(of)f(the)i(c)o(hoices)g(that)e(ha)o(v)o(e)h (b)q(een)h(made,)f(and)h(the)f(rationale)g(b)q(ehind)j(those)d(c)o (hoices.)75 1060 y Fy(2.1)59 b(Do)r(cument)18 b(Conventions)75 1165 y FE(Discussions)f(that)f(are)g(not)f(part)h(of)g(the)g(standard)g (are)g(mark)o(ed)f(with)i FF(\\Discussion")g FE(and)g(app)q(ear)75 1221 y(in)f(small)g(t)o(yp)q(e:)166 1362 y Fv(Discussion:)34 b Fu(A)14 b(\\discussion")g(lo)q(oks)f(lik)o(e)g(this.)75 1503 y FE(Advice)k(to)d(implemen)o(tors)i(of)f(MPI)g(will)i(app)q(ear)e (in)h(a)f(note)g(lik)o(e)h(this:)166 1644 y Fv(Implemen)o(tati)o(on)c (note:)36 b Fu(Advice)15 b(on)e(implemen)o(tation)e(go)q(es)j(here.)75 1880 y Fy(2.2)59 b(Pro)r(cedure)19 b(Sp)r(eci\014cation)75 1985 y FE(MPI)f(pro)q(cedures)h(are)f(sp)q(eci\014ed)i(using)f(a)f (language)h(indep)q(enden)o(t)h(notation.)29 b(The)18 b(parameters)f(of)75 2041 y(pro)q(cedure)f(calls)g(are)f(mark)o(ed)g (as)30 b Fu(IN)p FE(,)g Fu(OUT)16 b FE(or)29 b Fu(INOUT)p FE(.)16 b(The)f(meanings)h(of)f(these)g(are:)143 2142 y FA(\017)23 b FE(the)15 b(call)h(uses)g(but)f(do)q(es)h(not)e(up)q (date)i(a)f(parameter)f(mark)o(ed)30 b Fu(IN)p FE(,)143 2243 y FA(\017)23 b FE(the)15 b(call)h(ma)o(y)f(up)q(date)h(a)e (parameter)h(mark)o(ed)30 b Fu(OUT)p FE(,)143 2344 y FA(\017)23 b FE(the)15 b(call)h(b)q(oth)g(uses)f(and)g(up)q(dates)h(a)f (parameter)f(mark)o(ed)30 b Fu(INOUT)p FE(.)166 2445 y(If)22 b(a)f(parameter)f(is)i(a)f(handle)i(to)d(an)i(opaque)f(ob)s (ject,)h(and)f(the)h(ob)s(ject)f(is)g(up)q(dated)i(b)o(y)e(the)75 2501 y(pro)q(cedure)d(call,)g(then)f(the)g(parameter)f(is)h(mark)o(ed) 33 b Fu(OUT)p FE(.)17 b(This)g(is)h(done)f(ev)o(en)g(though)g(the)g (handle)75 2558 y(itself)d(is)g(not)f(mo)q(di\014ed)i({)e(w)o(e)g(use)g (the)27 b Fu(OUT)13 b FE(attribute)h(to)e(denote)i(that)e(what)h(the)g (handle)i Ft(r)n(efer)n(enc)n(es)75 2614 y FE(is)h(up)q(dated.)p 75 2661 720 2 v 127 2688 a FC(1)144 2704 y Fs(V)m(ersion)e(of)f(July)g (30,)g(1993)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 12 6 11 bop 75 -100 a FE(6)737 b Fx(SECTION)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)166 45 y FE(The)g(de\014nition)j(of)c(MPI)i (tries)f(to)g(a)o(v)o(oid,)g(to)f(the)i(largest)f(p)q(ossible)i(exten)o (t,)d(the)i(use)f(of)30 b Fu(INOUT)75 102 y FE(parameters,)14 b(b)q(ecause)i(suc)o(h)g(use)f(is)h(error-prone,)e(esp)q(ecially)k(for) d(scalar)g(parameters.)166 158 y(A)j(common)f(o)q(ccurrence)i(for)e (MPI)g(functions)i(is)f(a)f(parameter)g(that)g(is)h(used)g(as)35 b Fu(IN)18 b FE(b)o(y)f(some)75 214 y(pro)q(cesses)j(and)40 b Fu(OUT)20 b FE(b)o(y)g(other)f(pro)q(cesses.)34 b(Suc)o(h)20 b(parameters)f(are,)h(syn)o(tactically)l(,)i(an)39 b Fu(INOUT)75 271 y FE(parameter,)14 b(although,)h(seman)o(tically)l(,)h (it)g(is)f(not)g(used)h(in)g(one)f(call)i(b)q(oth)e(for)g(input)h(and)f (for)g(output.)166 410 y Fv(Discussion:)34 b Fu(W)m(e)13 b(migh)o(t)f(w)o(an)o(t)i(to)f(distinguish)h(this)g(use;)g(e.g.)36 b Fr(IN/OUT)13 b Fu(vs)28 b Fr(INOUT)p Fu(.)166 549 y FE(Another)17 b(frequen)o(t)g(situation)h(arises)f(when)h(a)e (parameter)h(v)m(alue)h(is)g(needed)g(only)g(b)o(y)f(a)g(subset)75 606 y(of)e(the)h(pro)q(cesses.)22 b(When)17 b(a)e(parameter)g(is)i(not) e(signi\014can)o(t)i(at)e(a)h(pro)q(cess)g(then)g(an)g(arbitrary)f(v)m (alue)75 662 y(can)g(b)q(e)h(passed)g(as)e(argumen)o(t.)75 805 y Fy(2.3)59 b(Semantical)18 b(T)-5 b(erms)75 906 y FE(A)15 b(comm)o(unication)h(op)q(eration)g(is)f(termed)g(to)g(b)q (e:)75 1010 y FF(non)o(blo)q(c)o(king)24 b FE(if)15 b(it)h(ma)o(y)e (return)h(b)q(efore)h(the)f(related)h(comm)o(unication)g(op)q(eration)f (completes;)75 1103 y FF(lo)q(cally-blo)r(c)o(king)26 b FE(if)18 b(return)g(from)f(the)h(pro)q(cedure)h(indicates)g(the)f (user)g(is)h(allo)o(w)o(ed)f(to)f(re-use)i(re-)189 1159 y(sources)c(\(suc)o(h)g(as)g(bu\013ers\))g(sp)q(eci\014ed)i(in)f(the)f (call;)75 1252 y FF(globally-blo)q(cking)25 b FE(if)c(return)f(from)g (the)g(pro)q(cedure)i(indicates)f(the)g(comm)o(unication)g(op)q (eration)189 1309 y(has)15 b(completed.)166 1412 y(An)20 b(op)q(eration)g(is)h FF(lo)q(cal)h FE(if)e(its)g(execution)h(do)q(es)f (not)g(en)o(tail)h(comm)o(unication,)g(and)f(do)q(es)g(not)75 1469 y(dep)q(end)c(on)f(the)g(state)f(of)g(other)g(pro)q(cesses;)h FF(global)p FE(,)h(if)g(its)e(completion)i(ma)o(y)e(dep)q(end)j(on)d (comm)o(uni-)75 1525 y(cation)h(o)q(ccurring;)h FF(collectiv)o(e)h FE(if)e(sev)o(eral)h(pro)q(cesses)f(need)h(to)f(participate)h(in)g(its) f(execution.)75 1668 y Fy(2.4)59 b(Data)19 b(T)-5 b(yp)r(es)75 1771 y Fq(2.4.1)49 b(Opaque)16 b(objects)75 1857 y FE(MPI)23 b(pro)q(cedures)g(use)g Ft(op)n(aque)h(obje)n(cts)p FE(.)42 b(These)23 b(hide)h(the)f(in)o(ternal)h(represen)o(tation)e(used,)j(th) o(us)75 1913 y(allo)o(wing:)c(similar)16 b(calls)h(in)f(C)f(and)g(F)l (ortran;)f(the)h(a)o(v)o(oidance)h(of)f(con\015icts)h(with)f(the)h(t)o (yping)f(rules)h(in)75 1970 y(these)f(languages;)g(future)g(extension)h (of)f(functionalit)o(y)l(.)21 b(The)16 b(mec)o(hanism)f(for)g(opaque)g (ob)s(jects)f(used)75 2026 y(here)i(lo)q(osely)g(follo)o(ws)f(the)g (POSIX)i(F)l(ortran)d(binding)j(standard.)166 2083 y(An)g(opaque)g(ob)s (ject)f(can)g(b)q(e)i(allo)q(cated)f(on)g(user)g(storage)e(or)h(on)h (system)f(\(i.e.)24 b(MPI)17 b(managed\))75 2139 y(storage.)23 b(W)l(e)17 b(call)h(the)f(former)f(a)g Ft(user)i(obje)n(ct)e FE(and)h(the)g(latter)g(a)f Ft(system)h(obje)n(ct)p FE(.)24 b(User)17 b(ob)s(jects)f(are)75 2195 y(allo)q(cated)e(and)f(deallo)q (cated)i(explicitly)g(b)o(y)e(the)g(user)g(as)g(a)g(regular)g (\(structured\))f(v)m(ariable.)21 b(The)13 b(t)o(yp)q(e)75 2252 y(de\014nitions)i(for)e(suc)o(h)h(ob)s(jects)f(are)g(pro)o(vided)i (b)o(y)e(MPI.)g(The)h(ob)s(jects)f(can)g(b)q(e)i(accessed)f(and)g(mo)q (di\014ed)75 2308 y(only)i(via)f(MPI)g(functions)h(that)f(tak)o(e)f (suc)o(h)i(ob)s(jects)e(as)h(parameters.)166 2365 y(System)d(ob)s (jects)g(are)g(accessed)i(only)f(via)f Ft(hand)r(les)p FE(.)19 b(In)13 b(C,)f(a)g(di\013eren)o(t)h(t)o(yp)q(e)g(of)f(handle)i (is)f(de\014ned)75 2421 y(b)o(y)g(MPI)g(for)g(eac)o(h)g(t)o(yp)q(e)h (of)f(opaque)g(system)g(ob)s(ject.)19 b(In)13 b(F)l(ortran,)g(all)h (handles)g(ha)o(v)o(e)f(t)o(yp)q(e)37 b Fp(INTEGER)p FE(.)75 2478 y(The)15 b(allo)q(cation)g(and)g(deallo)q(cation)h(of)e (suc)o(h)h(an)f(ob)s(ject,)g(and)g(an)o(y)h(access)f(or)g(up)q(date)h (to)f(it,)g(are)h(done)75 2534 y(only)h(via)f(MPI)g(calls)i(that)d (accept)i(as)e(parameters)h(handles)h(to)f(suc)o(h)g(ob)s(jects.)166 2591 y(An)k(opaque)g(ob)s(ject,)g(and)g(the)g(handle)h(to)e(suc)o(h)h (ob)s(ject)f(are)h(signi\014can)o(t)g(only)h(at)e(the)h(pro)q(cess)75 2647 y(where)g(the)g(ob)s(ject)f(w)o(as)g(created,)h(and)g(cannot)g(b)q (e)h(transferred)e(to)g(another)g(pro)q(cess)h(\(ob)s(jects)f(are)75 2704 y(lo)q(cal\).)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 13 7 12 bop 75 -100 a Fx(2.4.)34 b(D)o(A)l(T)l(A)15 b(TYPES)1375 b FE(7)166 45 y Fv(Discussion:)31 b Fu(This)11 b(ma)o(y)f(need)i(some)e (clari\014cation)g(if)h("\015atten/un\015atten")h(is)f(accepted)i(in)d (the)i(con)o(text)75 102 y(draft.)166 242 y FE(Opaque)k(system)e(ob)s (jects)h(are)g(allo)q(cated)h(and)f(deallo)q(cated)i(b)o(y)e(calls)h (that)f(are)f(sp)q(eci\014c)j(to)e(eac)o(h)75 299 y(ob)s(ject)k(t)o(yp) q(e.)32 b(These)20 b(are)f(listed)i(in)f(the)f(sections)h(where)g(the)f (ob)s(jects)g(are)g(describ)q(ed.)35 b(A)19 b(call)i(to)75 355 y(deallo)q(cate)15 b(an)f(opaque)h(system)e(ob)s(ject)h(do)q(es)g (not)g(necessarily)h(deallo)q(cate)h(the)e(ob)s(ject)f(immediately)l(,) 75 412 y(but)e(ma)o(y)f(only)h(mark)g(it)g(as)f(candidate)i(for)e (deallo)q(cation.)20 b(The)11 b(ob)s(ject)f(will)j(b)q(e)e(deallo)q (cated)h(as)f(as)f(so)q(on)75 468 y(as)17 b(there)g(there)g(is)h(no)f (p)q(ending)i(op)q(eration)e(that)g(is)g(using)h(this)g(ob)s(ject;)f (the)g(handle)i(then)e(b)q(ecomes)75 525 y(n)o(ull.)22 b(F)l(or)15 b(example,)h(if)f(a)g(comm)o(unication)i(ob)s(ject)d(is)i (freed)g(after)f(the)g(comm)o(unication)h(started)f(but)75 581 y(b)q(efore)h(it)h(completed,)g(then)f(the)h(p)q(ending)h(op)q (eration)e(is)h(not)e(a\013ected;)h(one)h(can)f(use)g(the)h(handle)g (in)75 637 y(calls)f(to)f Fo(MPI)p 317 637 14 2 v 16 w(W)l(AIT)g FE(or)g Fo(MPI)p 599 637 V 16 w(ST)l(A)l(TUS)p FE(.)h(In)g(fact,)f(one)g(m)o(ust)g(complete)h(the)f(comm)o(unication)i (suc)o(h)e(a)75 694 y(call.)25 b(Ho)o(w)o(ev)o(er,)15 b(once)i(the)f(op)q(eration)h(completes,)g(it)g(is)g(erroneous)f(to)g (re-use)h(the)f(freed)h(handle)h(for)75 750 y(a)d(new)g(comm)o (unication.)166 891 y Fv(Implemen)o(tati)o(on)j(note:)55 b Fu(Some)17 b(ob)r(jects,)k(suc)o(h)e(as)g(bu\013er)h(descriptor)g(ob) r(jects,)g(can)f(b)q(e)h(used)f(b)o(y)75 947 y(sev)o(eral,)c(distinct,) f(concurren)o(t)j(comm)o(unicatio)o(n)12 b(op)q(erations.)21 b(Th)o(us,)14 b(to)h(\014nd)f(out)h(whether)h(suc)o(h)f(ob)r(ject)h (can)75 1004 y(b)q(e)f(freed,)f(one)g(needs)h(a)f(reference)i(coun)o(t) e(of)f(p)q(ending)h(op)q(erations.)166 1227 y Fv(Discussion:)33 b Fu(Need)14 b(a)e(test)i(to)e(see)i(if)e(a)h(handle)f(is)h(n)o(ull.)k (In)12 b(C,)h(if)f(handles)h(are)g(p)q(oin)o(ters,)g(a)f(comparison)75 1284 y(to)g Fn(NULL)f Fu(can)h(b)q(e)h(used.)18 b(In)12 b(F)m(ortran,)g(w)o(e)g(ma)o(y)e(either)j(ha)o(v)o(e)f(a)g(function)f Fr(MPI)p 1300 1284 13 2 v 15 w(IS)p 1350 1284 V 14 w(NULL)i Fu(\(and)f(mak)o(e)f(it)g(a)o(v)n(ailable)75 1340 y(in)16 b(C,)g(to)q(o,)h(for)f(compatibilit)o(y)d(reasons\),)18 b(or)e(ha)o(v)o(e)h(a)f Fn(NULL)g Fu(v)n(alue)g(that)g(can)h(b)q(e)g (used)g(in)g(comparisons,)e(in)h(a)75 1397 y(manner)d(similar)e(to)j(a) g(C)g(n)o(ull)e(p)q(oin)o(ter.)166 1537 y FE(MPI)19 b(ma)o(y)g(pro)o (vide)h(prede\014ned)h(opaque)f(ob)s(jects)f(and)g(prede\014ned,)j (static)d(handles)i(to)e(these)75 1594 y(ob)s(jects.)g(Suc)o(h)d(ob)s (jects)f(ma)o(y)f(not)h(b)q(e)h(destro)o(y)o(ed.)75 1724 y Fq(2.4.2)49 b(Arra)o(y)16 b(of)h(handles)75 1813 y FE(An)i(MPI)g(call)h(ma)o(y)e(need)i(a)f(parameter)f(that)g(is)i(an)e Ft(arr)n(ay-of-hand)r(les)p FE(.)32 b(The)20 b(arra)o(y-of-handles)e (is)75 1870 y(a)h(regular)h(arra)o(y)e(with)i(en)o(tries)g(that)f(are)g (handles)i(to)e(ob)s(jects)g(of)g(the)h(same)f(t)o(yp)q(e)h(in)g (consecutiv)o(e)75 1926 y(lo)q(cations)j(in)h(the)e(arra)o(y)l(.)41 b(Whenev)o(er)23 b(suc)o(h)g(an)g(arra)o(y)e(is)i(used,)i(an)d (additional)47 b Fo(len)23 b FE(parameter)75 1983 y(is)g(required)h(to) d(indicate)j(the)f(n)o(um)o(b)q(er)g(of)f(v)m(alid)i(en)o(tries)f (\(unless)g(this)g(n)o(um)o(b)q(er)g(can)g(b)q(e)g(deriv)o(ed)75 2039 y(otherwise\).)75 2170 y Fq(2.4.3)49 b(State)75 2258 y FE(MPI)22 b(pro)q(cedures)h(use)g(at)e(v)m(arious)i(places)g (argumen)o(ts)e(with)h Ft(state)h FE(t)o(yp)q(es.)40 b(The)23 b(v)m(alues)g(of)f(suc)o(h)75 2315 y(data)16 b(t)o(yp)q(e)h(are)g(all)h(iden)o(ti\014ed)h(b)o(y)e(names,)g(and)g(no) g(op)q(eration)g(is)h(de\014ned)g(on)f(them.)25 b(F)l(or)16 b(example,)75 2371 y(the)42 b Fo(MPI)p 265 2371 14 2 v 16 w(COMM)p 423 2371 V 16 w(INIT)20 b FE(routine)h(has)g(a)g(state)f (t)o(yp)q(e)h(parameter)f(with)h(v)m(alues)41 b Fr(MPI)p 1632 2371 13 2 v 14 w(ST)m(AND)o(ARD)p FE(,)75 2428 y Fr(MPI)p 152 2428 V 14 w(READ)o(Y)p FE(,)14 b(etc.)75 2558 y Fq(2.4.4)49 b(Named)16 b(constants)75 2647 y FE(MPI)11 b(pro)q(cedures)h(sometimes)f(assign)g(a)g(sp)q(ecial)i(meaning)f(to)e (a)h(sp)q(ecial)i(v)m(alue)f(of)f(a)g(basic)g(t)o(yp)q(e)g(param-)75 2704 y(eter;)16 b(e.g.)44 b Fo(tag)15 b FE(is)i(an)f(in)o(teger)g(v)m (alued)h(parameter)e(of)h(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)f(op)q(erations,)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 8 14 8 13 bop 75 -100 a FE(8)737 b Fx(SECTION)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y FE(with)g(a)g(sp)q(ecial)i (wild-card)f(v)m(alue,)g Fr(MPI)p 770 45 13 2 v 14 w(ANY)p 869 45 V 15 w(T)m(A)o(G)p FE(.)d(Suc)o(h)j(parameters)e(will)j(ha)o(v)o (e)d(a)h(range)f(of)h(regular)75 102 y(v)m(alues,)j(whic)o(h)g(is)f(a)f (prop)q(er)h(subrange)g(of)f(the)h(range)g(of)f(v)m(alues)i(of)e(the)h (corresp)q(onding)h(basic)f(t)o(yp)q(e;)75 158 y(sp)q(ecial)g(v)m (alues)g(\(suc)o(h)e(as)29 b Fr(MPI)p 626 158 V 14 w(ANY)p 725 158 V 15 w(T)m(A)o(G)p FE(\))14 b(will)j(b)q(e)f(outside)g(the)f (regular)h(range.)k(The)15 b(range)g(of)g(reg-)75 214 y(ular)i(v)m(alues)h(can)f(b)q(e)h(queried,)g(and)f(sometimes)g(set,)g (using)h(en)o(vironmen)o(t)f(inquiry)h(or)f(en)o(vironmen)o(t)75 271 y(setting)e(functions)h(\(Chapter)f(8\).)166 411 y Fv(Implemen)o(tati)o(on)d(note:)35 b Fu(Implemen)o(tors)11 b(should)i(detect,)i(whenev)o(er)g(p)q(ossible,)e(illegal)e(uses)k(of)d (\\sp)q(e-)75 467 y(cial)h(v)n(alues".)19 b(Th)o(us,)14 b(the)h(use)g(of)f(the)29 b Fr(MPI)p 785 467 V 14 w(ANY)p 884 467 V 15 w(T)m(A)o(G)13 b Fu(v)n(alue)h(to)g(tag)g(a)g(message)g (sen)o(t)h(will)e(b)q(e)i(\015agged)f(as)g(an)75 524 y(error.)75 732 y Fq(2.4.5)49 b(Choice)75 820 y FE(MPI)22 b(functions)g(sometimes)f(use)h(parameters)f(with)h(a)f Ft(choic)n(e)g FE(\(or)g(union\))h(data)f(t)o(yp)q(e.)39 b(Distinct)75 876 y(calls)18 b(to)e(the)h(same)f(routine)h(ma)o(y)f (pass)h(b)o(y)f(reference)i(actual)e(parameters)g(of)g(di\013eren)o(t)h (t)o(yp)q(es.)25 b(The)75 933 y(mec)o(hanism)16 b(for)e(pro)o(viding)j (suc)o(h)e(parameters)f(will)j(di\013er)f(from)e(language)i(to)e (language.)166 1066 y Fv(Discussion:)47 b Fu(The)18 b(F)m(ortran)f(77)f (standard)i(sp)q(eci\014es)h(that)e(the)h(t)o(yp)q(e)g(of)f(actual)f (argumen)o(ts)h(need)h(to)75 1116 y(agree)12 b(with)e(the)i(t)o(yp)q(e) f(of)g(dumm)o(y)d(argumen)o(ts;)i(no)h(construct)i(equiv)n(alen)o(t)d (to)h(C)g(p)q(oin)o(ters)g(is)g(a)o(v)n(ailable.)k(Th)o(us,)c(it)75 1165 y(w)o(ould)e(seem)i(that)f(there)h(is)f(no)h(standard)f (conforming)e(mec)o(hanism)g(to)i(supp)q(ort)h(c)o(hoice)g(parameters.) 17 b(Ho)o(w)o(ev)o(er,)75 1215 y(most)10 b(F)m(ortran)i(compilers)e (either)j(don't)e(c)o(hec)o(k)h(t)o(yp)q(e)g(consistency)i(of)c(calls)i (to)f(external)h(routines,)g(or)g(supp)q(ort)g(a)75 1265 y(sp)q(ecial)g(mec)o(hanism)d(to)i(link)g(foreign)f(\(e.g.,)h(C\))h (routines.)18 b(W)m(e)11 b(accept)h(this)g(non-conformit)o(y)d(with)i (the)h(F)m(ortran)75 1315 y(77)g(standard.)18 b(I.e.,)12 b(w)o(e)i(accept)g(that)f(the)g(same)f(routine)h(ma)o(y)e(b)q(e)j (passed)f(an)g(actual)g(parameter)f(of)g(a)h(di\013eren)o(t)75 1365 y(t)o(yp)q(e)h(at)g(distinct)g(calls.)166 1415 y(Generic)j (routines)g(can)f(b)q(e)h(used)h(in)d(F)m(ortran)i(90)e(to)h(pro)o (vide)h(a)f(standard)g(conforming)f(solution.)24 b(This)75 1465 y(solution)13 b(will)f(b)q(e)j(consisten)o(t)g(with)f(our)f (nonstandard)i(conforming)c(F)m(ortran)j(77)f(solution.)75 1695 y Fy(2.5)59 b(Pro)r(cesses)75 1798 y FE(An)22 b(MPI)h(program)e (is)h(executed)h(b)o(y)g(sev)o(eral)f(autonomous)f(pro)q(cesses)i(that) e(eac)o(h)i(execute)f(their)75 1855 y(o)o(wn)d(co)q(de,)h(in)g(a)e (MIMD)h(st)o(yle.)31 b(The)19 b(co)q(des)h(executed)g(b)o(y)f(eac)o(h)g (pro)q(cess)g(need)h(not)f(b)q(e)g(iden)o(tical.)75 1911 y(The)h(pro)q(cesses)g(comm)o(unicate)g(via)g(calls)g(to)f(MPI)h(comm)o (unication)g(primitiv)o(es.)35 b(T)o(ypically)l(,)22 b(eac)o(h)75 1967 y(pro)q(cessor)17 b(executes)i(in)f(its)g(o)o(wn)f (address)g(space,)h(although)g(shared-memory)g(implemen)o(tations)g(of) 75 2024 y(MPI)i(are)f(p)q(ossible.)35 b(This)20 b(do)q(cumen)o(t)g(sp)q (eci\014es)i(the)d(b)q(eha)o(vior)h(of)g(a)f(parallel)i(program)e (assuming)75 2080 y(that)c(only)g(MPI)h(calls)g(are)f(used)h(for)e (comm)o(unication.)21 b(The)16 b(in)o(teraction)f(of)g(an)g(MPI)g (program)g(with)75 2137 y(other)g(p)q(ossible)i(means)e(of)g(comm)o (unication)h(\(e.g.,)d(shared)j(memory\))e(is)i(not)e(sp)q(eci\014ed.) 166 2194 y(MPI)d(do)q(es)g(not)g(sp)q(ecify)h(the)f(execution)h(mo)q (del)g(for)e(eac)o(h)h(pro)q(cess.)19 b(A)11 b(pro)q(cess)g(can)h(b)q (e)f(sequen)o(tial,)75 2250 y(or)f(can)h(b)q(e)h(m)o(ulti-threaded,)g (with)f(threads)g(p)q(ossibly)h(executing)g(concurren)o(tly)l(.)20 b(Care)10 b(has)g(b)q(een)i(tak)o(en)75 2307 y(to)j(mak)o(e)f(MPI)h (\\thread-safe",)f(b)o(y)i(a)o(v)o(oiding)f(the)g(use)h(of)f(implicit)i (global)f(states.)166 2364 y(MPI)i(do)q(es)g(not)f(sp)q(ecify)j(either) e(the)g(in)o(teraction)h(of)e(pro)q(cesses)h(with)h(signals,)g(in)f(a)g (UNIX)g(en-)75 2421 y(vironmen)o(t,)e(or)g(other)f(ev)o(en)o(ts)h(that) g(do)g(not)f(relate)i(to)e(MPI)h(comm)o(unication.)23 b(That)16 b(is,)g(signals)h(are)75 2477 y(not)c(signi\014can)o(t)h (from)e(the)h(view)h(p)q(oin)o(t)g(of)e(MPI,)h(and)g(implemen)o(tors)h (should)g(attempt)e(to)h(implemen)o(t)75 2533 y(MPI)h(so)g(that)f (signals)i(are)f(transparen)o(t:)19 b(an)14 b(MPI)g(call)h(susp)q (ended)h(b)o(y)e(a)g(signal)h(should)g(resume)g(and)75 2590 y(complete)h(after)e(the)i(signal)g(is)f(handled.)166 2647 y(MPI)j(do)q(es)g(not)f(pro)o(vide)h(mec)o(hanisms)g(to)f(sp)q (ecify)i(the)f(initial)i(allo)q(cation)f(of)e(pro)q(cesses)h(to)f(an)75 2704 y(MPI)d(computation)g(and)g(their)g(binding)i(to)d(ph)o(ysical)j (pro)q(cessors.)j(It)14 b(is)g(exp)q(ected)h(that)f(v)o(endors)f(will) 1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 9 15 9 14 bop 75 -100 a Fx(2.6.)34 b(ERR)o(OR)17 b(HANDLING)1239 b FE(9)75 45 y(pro)o(vide)20 b(mec)o(hanisms)h(to)e(do)g(so)h(either)g (at)f(load)h(time)g(or)g(at)f(run)h(time.)34 b(Suc)o(h)20 b(mec)o(hanisms)g(will)75 102 y(allo)o(w)c(the)f(sp)q(eci\014cation)j (of)d(the)h(initial)i(n)o(um)o(b)q(er)d(of)h(required)g(pro)q(cesses,)g (the)g(co)q(de)g(to)f(b)q(e)h(executed)75 158 y(b)o(y)k(eac)o(h)h (initial)i(pro)q(cess,)e(and)g(the)g(allo)q(cation)g(of)f(pro)q(cesses) h(to)f(pro)q(cessors.)36 b(Also,)21 b(the)g(curren)o(t)75 214 y(prop)q(osal)f(do)q(es)h(not)e(pro)o(vide)i(for)e(dynamic)i (creation)f(or)g(deletion)h(of)f(pro)q(cesses)g(during)h(program)75 271 y(execution)c(\(the)f(total)f(n)o(um)o(b)q(er)i(of)f(pro)q(cesses)g (is)h(\014xed\),)f(although)g(it)h(is)f(in)o(tended)i(to)d(b)q(e)i (consisten)o(t)75 327 y(with)h(suc)o(h)f(extensions.)27 b(Finally)l(,)19 b(the)e(curren)o(t)g(prop)q(osal)h(do)q(es)f(not)g(sp) q(ecify)h(a)f(naming)h(sc)o(heme)g(for)75 384 y(pro)q(cesses.)33 b(W)l(e)20 b(prop)q(ose)f(to)g(alw)o(a)o(ys)g(iden)o(tify)i(pro)q (cesses)f(according)g(to)f(their)h(relativ)o(e)g(rank)f(in)h(a)75 440 y(group,)15 b(so)f(that,)g(e\013ectiv)o(ely)l(,)i(pro)q(cesses)g (are)f(iden)o(ti\014ed)i(b)o(y)e(consecutiv)o(e)h(in)o(tegers.)75 592 y Fy(2.6)59 b(Erro)n(r)21 b(Handling)75 696 y FE(MPI)e(pro)o(vides) h(the)f(user)g(with)h(reliable)h(message)e(transmission.)32 b(A)19 b(message)g(sen)o(t)g(is)g(alw)o(a)o(ys)g(re-)75 752 y(ceiv)o(ed)e(correctly)l(,)e(and)h(the)g(user)g(do)q(es)f(not)h (need)g(to)f(c)o(hec)o(k)h(for)f(transmission)h(errors,)e(time-outs,)h (or)75 809 y(other)g(error)f(conditions.)22 b(In)15 b(other)g(w)o (ords,)f(MPI)h(do)q(es)h(not)f(pro)o(vide)h(mec)o(hanisms)f(for)g (dealing)i(with)75 865 y(failures)f(in)g(the)f(comm)o(unication)h (system.)j(If)c(the)g(MPI)g(implemen)o(tation)i(is)e(built)i(on)d(an)h (unreliable)75 922 y(underlying)k(mec)o(hanism,)e(then)g(it)g(is)h(the) e(job)h(of)f(the)h(implemen)o(ter)h(of)f(the)f(MPI)h(subsystem)g(to)f (in-)75 978 y(sulate)h(the)f(user)g(from)g(this)g(unreliabili)q(t)o(y)l (,)i(or)e(to)g(re\015ect)g(unreco)o(v)o(erable)h(errors)e(as)h(global)h (program)75 1035 y(failures.)k(Similarly)l(,)c(MPI)e(itself)h(pro)o (vides)g(no)f(mec)o(hanisms)h(for)e(handling)j(no)q(de)f(failures.)166 1092 y(Of)g(course,)h(MPI)f(programs)f(ma)o(y)h(still)i(b)q(e)f (erroneous.)23 b(A)16 b FF(program)i(error)d FE(can)i(o)q(ccur)f(when) 75 1149 y(an)f(MPI)g(call)g(is)h(called)g(with)f(an)g(incorrect)g (parameter)f(\(non-existing)i(destination)g(in)g(a)e(send)i(op)q(er-)75 1205 y(ation,)h(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i(op)q (eration,)f(etc.\))25 b(This)17 b(t)o(yp)q(e)g(of)f(error)g(w)o(ould)h (o)q(ccur)h(in)f(an)o(y)75 1262 y(implemen)o(tation.)31 b(In)19 b(addition,)h(a)f FF(resource)h(error)d FE(ma)o(y)h(o)q(ccur)h (when)g(a)f(program)f(exceeds)j(the)75 1318 y(amoun)o(t)d(of)f(a)o(v)m (ailable)j(system)e(resources)h(\(n)o(um)o(b)q(er)f(of)g(p)q(ending)i (messages,)e(system)g(bu\013ers,)g(etc.\).)75 1375 y(The)h(o)q (ccurrence)h(of)f(this)g(t)o(yp)q(e)g(of)g(error)f(dep)q(ends)j(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(in)h(the)75 1431 y(system)13 b(and)h(the)g(resource)f(allo)q(cation)i(mec)o(hanism) f(used;)h(this)f(ma)o(y)f(di\013er)h(from)f(system)g(to)g(system.)75 1488 y(The)i(recommended)g(implemen)o(tation)g(will)h(pro)o(vide)f (generous)g(limits)g(on)g(some)f(of)g(the)g(resources)g(so)75 1544 y(as)i(to)g(alleviate)i(the)f(p)q(ortabilit)o(y)g(problem)h(this)f (represen)o(ts.)24 b(One)17 b(can)g(also)f(write)h FF(safe)g FE(programs,)75 1600 y(that)d(are)h(not)g(sub)s(ject)g(to)g(resource)g (errors.)166 1658 y(All)21 b(MPI)f(calls)h(return)f(an)f(error)h (parameter)f(that)g(indicates)i(successful)g(completion)g(of)f(the)75 1715 y(op)q(eration,)15 b(or)g(the)g(error)g(condition)h(that)f(o)q (ccurred,)g(otherwise.)21 b(Ho)o(w)o(ev)o(er,)13 b(MPI)j(do)q(es)f(not) g(require)75 1771 y(the)f(user)f(to)g(c)o(hec)o(k)h(error)e (parameters.)19 b(It)13 b(is)h(recommended)h(to)e(pro)o(vide)h(an)f (implemen)o(tation)i(where)75 1828 y(the)g(en)o(vironmen)o(t)g(for)f (an)h(MPI)g(execution)g(can)g(b)q(e)h(set)e(so)h(that)f(an)o(y)g(error) g(encoun)o(tered)i(in)f(an)g(MPI)75 1884 y(call)k(will)h(cause)e(a)f (program)g(to)g(fail)i(with)f(a)g(suitable)h(error)e(message)h (returned)g(to)g(the)g(user.)28 b(The)75 1941 y(en)o(vironmen)o(t)15 b(for)f(an)g(MPI)h(execution)h(can)e(also)h(b)q(e)g(set)g(so)f(that)g (\(non-fatal\))g(errors)g(do)g(not)g(cause)h(a)75 1997 y(program)f(to)h(fail,)g(and)h(the)f(user)g(can)h(pro)o(vide)f(his)h (or)f(her)g(o)o(wn)g(error)g(reco)o(v)o(ery)f(co)q(de.)166 2055 y(MPI)f(calls)h(ma)o(y)f(initiate)h(op)q(erations)g(that)e(con)o (tin)o(ue)i(async)o(hronously)g(after)e(the)i(call)g(returned.)75 2111 y(Th)o(us,)22 b(the)g(op)q(eration)f(ma)o(y)g(return)g(with)h(a)f (co)q(de)h(indicating)h(successful)g(completion,)g(y)o(et)e(later)75 2168 y(cause)15 b(an)g(error)g(exception)h(to)e(b)q(e)i(raised.)k(If)15 b(there)g(is)h(a)f(subsequen)o(t)g(call)h(that)f(relates)g(to)f(the)h (same)75 2224 y(op)q(eration)f(\(e.g.,)f(a)g(call)i(that)e(v)o (eri\014es)i(that)e(an)h(async)o(hronous)f(op)q(eration)h(has)g (completed\))h(then)f(the)75 2281 y(error)f(parameter)g(asso)q(ciated)h (with)g(this)g(call)h(will)g(b)q(e)g(used)f(to)f(indicate)i(the)f (nature)g(of)f(the)h(error.)k(In)75 2337 y(a)c(few)h(cases,)f(the)g (error)g(ma)o(y)g(o)q(ccur)h(after)f(all)h(calls)h(that)d(relate)i(to)f (the)h(op)q(eration)f(ha)o(v)o(e)g(completed,)75 2394 y(so)j(that)g(no)g(error)g(parameter)g(can)h(b)q(e)g(used)g(to)f (indicate)i(the)f(nature)f(of)g(the)h(error)f(\(e.g.,)f(an)i(error)75 2450 y(in)f(a)e(send)h(with)g(the)g(ready)g(mo)q(de\).)21 b(Suc)o(h)16 b(an)g(error)f(m)o(ust)g(b)q(e)h(treated)g(as)f(fatal,)g (since)i(information)75 2507 y(cannot)e(b)q(e)h(returned)f(for)g(the)g (user)h(to)e(reco)o(v)o(er)h(from)f(it.)166 2647 y Fv(Implemen)o(tati)o (on)h(note:)45 b Fu(A)16 b(p)q(ossible)h(mec)o(hanism)c(is)j(to)g(ha)o (v)o(e)g(MPI)h(calls)e(generate)j(a)e(signal)f(that)75 2704 y(can)d(b)q(e)h(ignored)f(when)h(an)f(error)h(is)f(encoun)o (tered.)20 b(A)12 b(more)f(mo)q(dular)g(mec)o(hanism)e(that)k(allo)o (ws)e(one)h(to)g(handle)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 10 16 10 15 bop 75 -100 a FE(10)714 b Fx(SECTION)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Fu(an)f(exception)g (within)f(its)h(con)o(text)h(is)f(preferred.)166 184 y FE(The)21 b(b)q(eha)o(vior)g(of)g(erroneous)f(programs)g(is)h(not)g (de\014ned)h(b)o(y)e(this)i(do)q(cumen)o(t.)37 b(The)21 b(desired)75 241 y(b)q(eha)o(vior)d(is)f(that)f(a)h(relev)m(an)o(t)h (error)e(co)q(de)h(b)q(e)h(returned;)g(ho)o(w)o(ev)o(er,)e(e\016ciency) j(considerations)f(ma)o(y)75 297 y(dictate)e(that)e(the)h(system)g(do)q (es)h(not)e(detect)i(certain)f(t)o(yp)q(es)h(of)e(errors.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 11 17 11 16 bop 75 356 a Fz(Section)35 b(3)75 564 y FG(Groups,)40 b(Con)m(texts,)g(and)75 689 y(Comm)m(unicators)75 895 y Fw(1)75 1053 y Fy(3.1)59 b(Intro)r(duction)75 1154 y FE(W)l(e)17 b(de\014ne)h(the)e(concepts)h(of)g(group,)f(con)o(text,)g (and)h(comm)o(unicator)f(here.)25 b(W)l(e)17 b(discuss)h(op)q(erations) 75 1211 y(for)13 b(ho)o(w)g(these)h(should)g(b)q(e)g(used)g(to)f(pro)o (vide)h(safe)f(\(safer\))g(comm)o(unication)h(in)g(the)g(MPI)f(system.) 19 b(W)l(e)75 1267 y(start)11 b(b)o(y)h(discussing)i(in)o(tra-comm)o (unication)f(in)g(full)h(detail;)g(then)f(w)o(e)f(discuss)h(in)o (ter-comm)o(unication,)75 1324 y(whic)o(h)j(builds)g(on)f(the)g(data)f (structures)g(and)h(requiremen)o(ts)g(of)g(the)f(in)o(tra-comm)o (unication)i(sections.)75 1380 y(W)l(e)f(follo)o(w)g(with)g(discussion) h(of)e(formalizations)h(of)f(the)h(lo)q(osely)h(sync)o(hronous)f(mo)q (del)g(of)f(computing)75 1436 y(\(vis)h(a)g(vis)h(message)f(passing\))g (and)g(o\013er)g(examples.)166 1493 y(It)c(is)h(highly)g(desirable)h (that)d(pro)q(cesses)i(executing)g(a)f(parallel)i(pro)q(cedure)f(use)f (a)g(\\virtual)h(pro)q(cess)75 1549 y(name)20 b(space")f(lo)q(cal)i(to) e(the)h(in)o(v)o(o)q(cation.)34 b(Th)o(us,)20 b(the)g(co)q(de)g(of)g (the)f(parallel)j(pro)q(cedure)e(will)i(lo)q(ok)75 1606 y(iden)o(tical,)d(irresp)q(ectiv)o(e)g(of)e(the)g(absolute)h(addresses) f(of)g(the)h(executing)g(pro)q(cesses.)27 b(It)17 b(is)h(often)f(the)75 1662 y(case)h(that)f(parallel)i(application)h(co)q(de)e(is)h(built)g(b) o(y)f(comp)q(osing)g(sev)o(eral)g(parallel)i(mo)q(dules)f(\()p Ft(e.g.)p FE(,)e(a)75 1719 y(n)o(umerical)f(solv)o(er,)f(and)g(a)g (graphic)h(displa)o(y)g(mo)q(dule\).)21 b(Supp)q(ort)15 b(of)g(a)g(virtual)h(name)f(space)g(for)g(eac)o(h)75 1775 y(mo)q(dule)j(will)h(allo)o(w)e(for)f(the)h(comp)q(osition)g(of)g (mo)q(dules)h(that)e(w)o(ere)h(dev)o(elop)q(ed)h(separately)f(without) 75 1832 y(c)o(hanging)e(all)h(message)e(passing)h(calls)g(within)h(eac) o(h)f(mo)q(dule.)21 b(The)15 b(set)f(of)g(pro)q(cesses)h(that)f (execute)h(a)75 1888 y(parallel)i(pro)q(cedure)f(ma)o(y)f(b)q(e)h (\014xed,)g(or)f(ma)o(y)g(b)q(e)h(determined)h(dynamically)g(b)q(efore) f(the)g(in)o(v)o(o)q(cation.)75 1945 y(Th)o(us,)23 b(MPI)e(has)g(to)g (pro)o(vide)h(a)f(mec)o(hanism)h(for)f(dynamically)j(creating)d(sets)g (of)g(lo)q(cally)j(named)75 2001 y(pro)q(cesses.)35 b(W)l(e)20 b(alw)o(a)o(ys)f(n)o(um)o(b)q(er)h(pro)q(cesses)h(that)e(execute)h(a)g (parallel)i(pro)q(cedure)f(consecutiv)o(ely)l(,)75 2057 y(starting)15 b(from)g(zero,)h(and)g(call)g(this)h(n)o(um)o(b)q(ering)f FF(rank)i(in)h(group)p FE(.)i(Th)o(us,)16 b(a)f FF(group)h FE(is)g(an)g(ordered)75 2114 y(set)f(of)f(pro)q(cesses,)h(where)h(pro)q (cesses)f(are)g(iden)o(ti\014ed)i(b)o(y)e(their)g(ranks)g(when)g(comm)o (unication)h(o)q(ccurs.)166 2170 y(Comm)o(unication)23 b Fp(contexts)f FE(partition)h(the)h(message-passing)f(space)g(in)o(to) g(separate,)h(man-)75 2227 y(ageable)f(\\univ)o(erses.")44 b(Sp)q(eci\014cally)l(,)27 b(a)c(send)g(made)g(in)h(a)f(con)o(text)f (cannot)h(b)q(e)g(receiv)o(ed)h(in)g(an-)75 2283 y(other)18 b(con)o(text.)29 b(Con)o(texts)17 b(are)i(iden)o(ti\014ed)h(in)f(MPI)g (using)g(opaque)f FF(con)o(texts)h FE(that)e(reside)j(within)75 2340 y Fp(communicator)e FE(ob)s(jects.)31 b(The)20 b(con)o(text)e(mec) o(hanism)i(is)g(need)g(to)f(allo)o(w)g(predictable)i(b)q(eha)o(vior)f (in)75 2396 y(subprograms,)14 b(and)g(to)g(allo)o(w)h(dynamicism)h(in)g (message)e(usage)g(that)g(cannot)h(b)q(e)g(reasonably)g(an)o(tici-)75 2453 y(pated)g(or)e(managed.)20 b(Normally)l(,)15 b(a)f(parallel)i(pro) q(cedure)f(is)g(written)g(so)f(that)f(all)j(messages)e(pro)q(duced)75 2509 y(during)f(its)g(execution)h(are)e(also)g(consumed)h(b)o(y)g(the)f (pro)q(cesses)h(that)f(execute)h(the)f(parallel)i(pro)q(cedure.)75 2566 y(Ho)o(w)o(ev)o(er,)d(if)h(one)g(parallel)i(pro)q(cedure)f(calls)g (another,)e(then)i(it)f(migh)o(t)f(b)q(e)i(desirable)g(to)f(allo)o(w)g (suc)o(h)g(call)75 2622 y(to)17 b(pro)q(ceed)i(while)g(messages)e(are)g (p)q(ending)j(\(the)d(messages)h(will)h(b)q(e)f(consumed)h(b)o(y)e(the) h(pro)q(cedure)p 75 2661 720 2 v 127 2688 a FC(1)144 2704 y Fs(V)m(ersion)c(of)f(August)g(13,)g(1993)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 12 18 12 17 bop 75 -100 a FE(12)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y FE(after)g(the)g(call)h(returns\).)k(In)c(suc)o(h)g(case,)f(a)g (new)g(comm)o(unication)h(con)o(text)f(is)h(needed)g(for)f(the)g (called)75 102 y(parallel)i(pro)q(cedure,)e(ev)o(en)h(if)g(the)f (transfer)f(of)h(con)o(trol)g(is)h(sync)o(hronized.)166 161 y(The)e(comm)o(unication)i(domain)e(used)h(b)o(y)f(a)g(parallel)i (pro)q(cedure)g(is)e(iden)o(ti\014ed)j(b)o(y)d(a)g FF(comm)o(uni-)75 217 y(cator)p FE(.)29 b(Comm)o(unicators)17 b(bring)h(together)f(the)h (concepts)h(of)e(pro)q(cess)h(group)g(and)g(comm)o(unication)75 274 y(con)o(text.)29 b(A)18 b(comm)o(unicator)g(is)h(an)f(explicit)j (parameter)c(in)j(eac)o(h)e(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)75 330 y(op)q(eration.)31 b(The)19 b(comm)o(unicator)f(iden) o(ti\014es)j(the)d(comm)o(unication)i(con)o(text)e(of)g(that)g(op)q (eration;)j(it)75 387 y(iden)o(ti\014es)f(the)f(group)f(of)g(pro)q (cesses)g(that)g(can)g(b)q(e)h(in)o(v)o(olv)o(ed)h(in)f(this)g(comm)o (unication;)h(and)e(it)h(pro-)75 443 y(vides)g(the)g(translation)f (from)g(virtual)h(pro)q(cess)g(names,)g(whic)o(h)g(are)f(ranks)g (within)i(the)e(group,)h(in)o(to)75 500 y(absolute)13 b(addresses.)20 b(Collectiv)o(e)14 b(comm)o(unication)g(calls)g(also)f (tak)o(e)f(a)h(comm)o(unicator)f(as)h(parameter;)75 556 y(it)k(is)h(exp)q(ected)g(that)e(parallel)j(libraries)g(will)f(b)q(e)g (built)g(to)f(accept)g(a)g(comm)o(unicator)f(as)h(parameter.)75 613 y(Comm)o(unicators)d(are)h(represen)o(ted)h(b)o(y)f(opaque)g(MPI)h (ob)s(jects.)166 672 y(W)l(e)h(start)f(b)o(y)h(discussing)i(in)o (tra-comm)o(unication)e(in)h(full)h(detail;)f(then)g(w)o(e)e(discuss)j (in)o(ter-com-)75 728 y(m)o(unication,)14 b(whic)o(h)f(builds)i(on)e (the)g(data)f(structures)g(and)h(requiremen)o(ts)h(of)e(the)h(in)o (tra-comm)o(unicat-)75 785 y(ion)j(sections.)22 b(W)l(e)16 b(follo)o(w)g(with)g(discussion)h(of)e(formalizations)i(of)e(the)h(lo)q (osely)g(sync)o(hronous)g(mo)q(del)75 841 y(of)f(computing)h(\(vis)f(a) g(vis)g(message)g(passing\))g(and)h(o\013er)e(examples.)75 1001 y Fy(3.2)59 b(Groups)75 1108 y FE(A)17 b FF(group)g FE(is)g(an)g(ordered)g(set)f(of)h(pro)q(cess)g(iden)o(ti\014ers)h (\(henceforth)f(pro)q(cesses\);)g(pro)q(cess)g(iden)o(ti\014ers)75 1164 y(are)h(implemen)o(tation)j(dep)q(enden)o(t;)g(a)e(group)f(is)i (an)e(opaque)h(ob)s(ject.)30 b(Eac)o(h)19 b(pro)q(cess)g(in)g(a)g (group)f(is)75 1221 y(asso)q(ciated)d(with)h(an)f(in)o(teger)g FF(rank)p FE(,)g(starting)f(from)h(zero.)166 1280 y(Groups)d(are)g (represen)o(ted)h(b)o(y)f(opaque)h FF(group)h(ob)s(jects)p FE(,)e(and)h(hence)g(cannot)g(b)q(e)g(directly)g(trans-)75 1336 y(ferred)i(from)g(one)g(pro)q(cess)h(to)e(another.)75 1496 y Fy(3.3)59 b(Contexts)75 1603 y FE(A)15 b FF(con)o(text)h FE(is)g(the)f(MPI)g(mec)o(hanism)h(for)f(partitioning)h(comm)o (unication)g(space.)21 b(A)15 b(de\014ning)i(prop-)75 1659 y(ert)o(y)c(of)h(a)f(con)o(text)h(is)g(that)f(a)h(send)g(made)g (in)h(a)e(con)o(text)h(cannot)f(b)q(e)i(receiv)o(ed)g(in)g(another)e (con)o(text.)19 b(A)75 1716 y(con)o(text)c(is)g(an)g(opaque)h(ob)s (ject.)j(Only)d(one)g(comm)o(unicator)e(in)i(a)f(pro)q(cess)h(ma)o(y)e (bind)j(the)e(same)g(con-)75 1772 y(text.)24 b(Con)o(texts)15 b(ha)o(v)o(e)i(additional)h(attributes)e(for)g(in)o(ter-comm)o (unication,)i(to)e(b)q(e)i(discussed)g(b)q(elo)o(w.)75 1829 y(F)l(or)e(in)o(tra-comm)o(unication,)g(a)g(con)o(text)g(is)h (essen)o(tially)h(a)d(h)o(yp)q(er-tag)h(needed)i(to)e(mak)o(e)f(a)h (comm)o(uni-)75 1885 y(cator)e(safe)h(for)g(p)q(oin)o(t-to-p)q(oin)o(t) h(and)f(MPI-de\014ned)i(collectiv)o(e)g(comm)o(unication.)166 2027 y Fv(Discussion:)34 b Fu(Some)12 b(implemen)o(tations)f(ma)o(y)h (mak)o(e)g(a)h(con)o(text)i(to)e(b)q(e)i(a)e(pair)g(of)g(in)o(tegers,)h (eac)o(h)h(repre-)75 2084 y(sen)o(ting)g(\\h)o(yp)q(er)h(tags")f({)g (one)h(for)e(p)q(oin)o(t-to-p)q(oin)o(t)g(and)h(one)h(for)f (\(MPI-de\014ned\))h(collectiv)o(e)g(op)q(erations)f(on)g(a)75 2140 y(comm)o(unicator.)g(By)e(making)d(this)j(concept)h(opaque,)e(w)o (e)h(reliev)o(e)g(the)h(implemen)o(tor)c(of)i(the)h(requiremen)o(t)g (that)75 2197 y(this)h(is)g(the)g(only)f(w)o(a)o(y)g(to)h(implemen)o(t) d(con)o(texts)k(correctly)g(for)f(MPI.)166 2421 y Fv(Discussion:)19 b Fu(Among)14 b(other)i(reasons,)h(including)e(to)g(address)i(Jim)e(Co) o(wnie's)g(concerns)i(ab)q(out)f(safet)o(y)75 2478 y(and)c(to)g(mak)o (e)e(b)q(oth)i(p)q(oin)o(t-to-p)q(oin)o(t)f(and)h(collectiv)o(e)g(comm) o(unicati)o(on)d(safer)k(on)e(an)h(in)o(tra-comm)o(uni)o(ncator,)e(w)o (e)75 2534 y(ha)o(v)o(e)k(opted)h(to)g(mak)o(e)e(con)o(texts)i(opaque,) f(at)h(the)g(exp)q(ense)h(of)e(upsetting)h(those)h(who)e(w)o(an)o(t)g (to)g(b)q(e)h(able)g(to)f(set)75 2591 y(con)o(text)g(v)n(alues.)j(This) d(c)o(hange)f(is)g(crucial)g(to)g(abstracting)h(MPI)f(from)f(sp)q (eci\014c)i(implemen)o(tations,)c(and)j(forces)75 2647 y(sp)q(eci\014c)19 b(implem)o(en)o(tations)c(to)i(pro)o(vide)g(implem)o (en)o(tation-sp)q(eci\014c)e(functions)j(to)f(mak)o(e)e(the)j (connotation)f(of)75 2704 y(con)o(texts)e(with)f(sp)q(eci\014c)h(in)o (teger)f(v)n(alues.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 13 19 13 18 bop 75 -100 a Fx(3.4.)34 b(COMMUNICA)l(TORS)1224 b FE(13)75 45 y Fy(3.4)59 b(Communicato)n(rs)75 152 y FE(All)20 b(MPI)f(comm)o(unication)h(\(b)q(oth)f(p)q(oin)o(t-to-p)q (oin)o(t)h(and)f(collectiv)o(e\))i(functions)f(use)f FF(comm)o(unica-)75 209 y(tors)c FE(to)g(pro)o(vide)i(a)e(sp)q (eci\014c)j(scop)q(e)e(\(con)o(text)f(and)h(group)f(sp)q (eci\014cations\))j(for)d(the)h(comm)o(unication.)75 265 y(In)d(short,)g(comm)o(unicators)f(bring)h(together)f(the)h (concepts)g(of)g(group)f(and)h(con)o(text;)g(\(furthermore,)f(to)75 322 y(supp)q(ort)18 b(implemen)o(tation-sp)q(eci\014c)j(optimizations,) d(and)g(virtual)g(top)q(ologies,)g(they)g(\\cac)o(he")f(addi-)75 378 y(tional)f(information)g(opaquely\).)22 b(The)16 b(source)f(and)h(destination)h(of)e(a)g(message)g(is)i(iden)o(ti\014ed) g(b)o(y)f(the)75 434 y(rank)h(of)f(that)g(pro)q(cess)h(within)h(the)f (group;)g(no)g(a)g(priori)h(mem)o(b)q(ership)g(restrictions)f(on)g(the) g(pro)q(cess)75 491 y(sending)f(or)f(receiving)i(the)e(message)f(are)h (implied.)22 b(F)l(or)15 b(collectiv)o(e)i(comm)o(unication,)e(the)g (comm)o(uni-)75 547 y(cator)i(sp)q(eci\014es)j(the)e(set)f(of)h(pro)q (cesses)g(that)f(participate)i(in)g(the)f(collectiv)o(e)h(op)q (eration.)29 b(Th)o(us,)18 b(the)75 604 y(comm)o(unicator)g(restricts)h (the)f(\\spatial")h(scop)q(e)g(of)f(comm)o(unication,)i(and)f(pro)o (vides)g(lo)q(cal)h(pro)q(cess)75 660 y(addressing.)166 720 y(Comm)o(unicators)10 b(are)h(represen)o(ted)h(b)o(y)g(opaque)f FF(comm)o(unicator)i(ob)s(jects)p FE(,)f(and)g(hence)g(cannot)75 776 y(b)q(e)k(directly)g(transferred)f(from)g(one)g(pro)q(cess)g(to)g (another.)75 913 y Fo(Raison)j(d')o(^)-21 b(etre)17 b(fo)o(r)f (Contexts)j(and)f(Communicato)o(rs)44 b FE(Within)19 b(a)e(comm)o(unicator,)g(one)h(or)f(more)g(con-)75 969 y(texts)12 b(are)g(cac)o(hed.)20 b(W)l(e)12 b(w)o(an)o(t)g(to)g(mak)o (e)g(it)h(p)q(ossible)h(for)e(libraries)i(quic)o(kly)g(to)e(ac)o(hiev)o (e)h(additional)h(safe)75 1026 y(comm)o(unication)h(space)g(without)g (MPI-comm)o(unicator-based)g(sync)o(hronization.)20 b(The)15 b(only)g(w)o(a)o(y)f(to)75 1082 y(do)d(this)h(is)g(to)f(pro)o(vide)h(a) f(means)g(to)g(preallo)q(cate)h(man)o(y)f(con)o(texts,)g(and)g(bind)i (them)e(lo)q(cally)l(,)j(as)d(needed.)75 1139 y(This)16 b(c)o(hoice)h(w)o(eak)o(ens)e(the)h(o)o(v)o(erall,)g(inheren)o(t)h (\\safet)o(y")d(of)i(MPI,)f(if)h(programmed)f(in)i(this)f(w)o(a)o(y)l (,)f(but)75 1195 y(pro)o(vides)j(added)g(p)q(erformance)g(whic)o(h)g (library)h(designers)f(will)h(demand.)28 b(W)l(e)17 b(are)h(con)o (vinced)h(that)75 1251 y(the)c(means)g(w)o(e)g(ha)o(v)o(e)g(c)o(hosen)h (pro)o(vides)f(a)g(go)q(o)q(d)g(compromise)h(b)q(et)o(w)o(een)f (e\016ciency)i(and)e(safet)o(y)l(.)75 1390 y Fq(3.4.1)49 b(Prede\014ned)15 b(Communicato)o(rs)75 1481 y FE(Initial)i(comm)o (unicators)e(de\014ned)h(once)31 b Fo(MPI)p 885 1481 14 2 v 16 w(INIT)14 b FE(has)i(b)q(een)g(called)h(are)e(as)f(follo)o (ws:)143 1587 y FA(\017)23 b Fr(MPI)p 266 1587 13 2 v 14 w(COMM)p 410 1587 V 15 w(ALL)p FE(,)53 b(A)18 b(comm)o(unicator)g (of)f(all)i(pro)q(cesses)f(in)h(the)f(user)g(program)f(including)j(a) 189 1643 y(host)14 b(pro)q(cess)i(\(if)f(an)o(y\).)143 1749 y FA(\017)23 b Fr(MPI)p 266 1749 V 14 w(COMM)p 410 1749 V 15 w(PEER)p FE(,)38 b(A)13 b(comm)o(unicator)g(of)g(all)h(pro)q (cesses)g(excluding)h(a)e(host)g(pro)q(cess)h(\(if)f(an)o(y\).)189 1805 y(Note)19 b(that)g(in)h(the)g(absence)g(of)f(a)h(host)f(pro)q (cess)h(this)g(comm)o(unicator)f(will)i(not)e(b)q(e)i(just)e(the)189 1861 y(same)c(as)f Fr(MPI)p 435 1861 V 15 w(COMM)p 580 1861 V 14 w(ALL)i FE(since)g(it)g(will)h(con)o(tain)e(a)g(di\013eren)o (t)g(con)o(text.A)166 2050 y Fv(Discussion:)21 b Fu(In)16 b(the)h(host)g(pro)q(cess,)h(if)e(there)i(is)e(one,)h(what)g(should)f Fr(MPI)p 1404 2050 V 14 w(COMM)p 1548 2050 V 15 w(PEER)f Fu(b)q(e?)27 b(There)75 2106 y(are)15 b(t)o(w)o(o)f(consisten)o(t)i(p)q (ossibilities:)i(a)d(n)o(ull)e(comm)o(unicator;)f(a)i(comm)o(unicator)e (in)i(whic)o(h)g(the)i(group)e(con)o(tains)75 2162 y(only)f(the)i(host) f(pro)q(cess.)166 2304 y FE(MPI)g(implemen)o(tations)h(are)f(required)h (to)e(pro)o(vide)h(these)g(comm)o(unicators.)19 b(The)c(groups)e (corre-)75 2361 y(sp)q(onding)j(to)d(these)i(comm)o(unicators)f(do)g (not)g(app)q(ear)h(as)f(pre-de\014ned)i(constan)o(ts,)e(but)g(they)h (ma)o(y)e(b)q(e)75 2417 y(accessed)j(using)g Fo(MPI)p 459 2417 14 2 v 16 w(COMM)p 617 2417 V 16 w(GROUP)h FE(\(see)e(b)q(elo) o(w\).)75 2556 y Fq(3.4.2)49 b(Group)16 b(Management)75 2647 y FE(This)k(section)g(describ)q(es)h(the)f(manipulation)h(of)e (pro)q(cess)h(groups)f(in)h(MPI.)f(The)h(execution)h(of)e(the)75 2704 y(these)c(op)q(erations)h(do)f(not)g(require)h(in)o(terpro)q(cess) f(comm)o(unication.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 14 20 14 19 bop 75 -100 a FE(14)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fq(3.4.3)49 b(Group)16 b(Accesso)o(rs)75 313 y Fo(MPI)p 160 313 14 2 v 16 w(GROUP)p 328 313 V 18 w(SIZE\(group,)e(size\))117 459 y Fu(IN)171 b Fo(group)463 b Fu(handle)14 b(to)g(group)f(ob)r (ject.)117 672 y(OUT)124 b Fo(size)503 b Fu(is)14 b(the)g(in)o(teger)h (n)o(um)o(b)q(er)e(of)g(pro)q(cesses)k(in)c(the)i(group.)75 913 y Fo(MPI)p 160 913 V 16 w(GROUP)p 328 913 V 18 w(RANK\(group,)f (rank\))117 1060 y Fu(IN)171 b Fo(group)463 b Fu(handle)14 b(to)g(group)f(ob)r(ject.)117 1273 y(OUT)124 b Fo(rank)488 b Fu(is)15 b(the)h(in)o(teger)g(rank)f(of)g(the)g(calling)f(pro)q(cess) j(in)e(group,)g(or)905 1330 y Fr(MPI)p 982 1330 13 2 v 15 w(UNDEFINED)d Fu(if)h(the)i(pro)q(cess)h(is)d(not)h(a)g(mem)o(b)q (er.)75 1571 y Fo(MPI)p 160 1571 14 2 v 16 w(TRANSLA)l(TE)p 432 1571 V 17 w(RANKS)i(\(group)p 739 1571 V 16 w(a,)f(n,)g(ranks)p 956 1571 V 17 w(a,)g(group)p 1131 1571 V 16 w(b,)g(ranks)p 1298 1571 V 17 w(b\))117 1717 y Fu(IN)171 b Fo(group)p 445 1717 V 16 w(a)425 b Fu(handle)14 b(to)g(group)f(ob)r(ject)i(\\A") 117 1931 y(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)13 b(of)h(ranks)g(in)27 b Fr(ranks)p 1371 1931 13 2 v 16 w(a)14 b Fu(arra)o(y)117 2144 y(IN)171 b Fo(ranks)p 437 2144 14 2 v 16 w(a)433 b Fu(arra)o(y)14 b(of)f(zero)i(or)f(more)f(v)n (alid)f(ranks)i(in)g(group)f(\\A")117 2358 y(IN)171 b Fo(group)p 445 2358 V 16 w(b)424 b Fu(handle)14 b(to)g(group)f(ob)r (ject)i(\\B")117 2571 y(OUT)124 b Fo(ranks)p 437 2571 V 16 w(b)432 b Fu(arra)o(y)9 b(of)g(corresp)q(onding)h(ranks)g(in)f (group)g(\\B,")18 b Fr(MPI)p 1757 2571 13 2 v 14 w(UNDEFINED)905 2628 y Fu(when)d(no)e(corresp)q(ondence)k(exists.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 15 21 15 20 bop 75 -100 a Fx(3.4.)34 b(COMMUNICA)l(TORS)1224 b FE(15)75 45 y Fq(3.4.4)49 b(Group)16 b(Constructo)o(rs)75 181 y Fo(MPI)p 160 181 14 2 v 16 w(LOCAL)p 318 181 V 16 w(GROUP)p 486 181 V 18 w(UNION\(group1,)f(group2,)f(group)p 1088 181 V 17 w(out\))117 260 y Fu(IN)171 b Fo(group1)440 b Fu(\014rst)15 b(group)f(ob)r(ject)g(handle)117 339 y(IN)171 b Fo(group2)440 b Fu(second)15 b(group)f(ob)r(ject)h(handle) 117 417 y(OUT)124 b Fo(group)p 445 417 V 16 w(out)385 b Fu(group)14 b(ob)r(ject)h(handle)75 590 y Fo(MPI)p 160 590 V 16 w(LOCAL)p 318 590 V 16 w(GROUP)p 486 590 V 18 w(INTERSECT\(group1,)g(group2,)f(group)p 1191 590 V 17 w(out\))117 669 y Fu(IN)171 b Fo(group1)440 b Fu(\014rst)15 b(group)f(ob)r(ject)g(handle)117 748 y(IN)171 b Fo(group2)440 b Fu(second)15 b(group)f(ob)r(ject)h(handle)117 826 y(OUT)124 b Fo(group)p 445 826 V 16 w(out)385 b Fu(group)14 b(ob)r(ject)h(handle) 75 999 y Fo(MPI)p 160 999 V 16 w(LOCAL)p 318 999 V 16 w(GROUP)p 486 999 V 18 w(DIFFERENCE\(group1,)f(group2,)h(group)p 1216 999 V 16 w(out\))117 1078 y Fu(IN)171 b Fo(group1)440 b Fu(\014rst)15 b(group)f(ob)r(ject)g(handle)117 1157 y(IN)171 b Fo(group2)440 b Fu(second)15 b(group)f(ob)r(ject)h(handle) 117 1235 y(OUT)124 b Fo(group)p 445 1235 V 16 w(out)385 b Fu(group)14 b(ob)r(ject)h(handle)166 1361 y FE(The)g(set-lik)o(e)i (op)q(erations)e(are)g(de\014ned)h(as)f(follo)o(ws:)75 1462 y FF(union)24 b FE(All)19 b(elemen)o(ts)g(of)f(the)h(\014rst)f (group)g(\()p Fo(group1)p FE(\),)g(follo)o(w)o(ed)h(b)o(y)f(all)h (elemen)o(ts)g(of)f(second)h(group)189 1518 y(\()p Fo(group2)p FE(\))14 b(not)h(in)h(\014rst)75 1618 y FF(in)o(tersect)23 b FE(all)16 b(elemen)o(ts)g(of)f(the)g(\014rst)g(group)g(whic)o(h)h (are)e(also)i(in)g(the)f(second)h(group)75 1719 y FF(di\013erence)22 b FE(all)17 b(elemen)o(ts)e(of)g(the)g(\014rst)g(group)g(whic)o(h)h (are)f(not)g(in)h(the)f(second)h(group)75 1819 y(Note)d(that)f(for)g (these)i(op)q(erations)f(the)g(order)g(of)f(pro)q(cesses)i(in)g(the)f (output)g(group)f(is)i(determined)g(\014rst)75 1876 y(b)o(y)f(order)f (in)i(the)f(\014rst)f(group)h(\(if)g(p)q(ossible\))h(and)f(then)g(b)o (y)g(order)f(in)i(the)f(second)g(group)g(\(if)f(necessary\).)75 1981 y Fo(MPI)p 160 1981 V 16 w(LOCAL)p 318 1981 V 16 w(SUBGROUP\(group,)17 b(n,)e(ranks,)g(new)p 981 1981 V 17 w(group\))117 2060 y Fu(IN)171 b Fo(group)463 b Fu(handle)14 b(to)g(group)f(ob)r(ject)117 2138 y(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)9 b(of)f(elemen)o(ts)i(in)e(arra)o(y)h (ranks)h(\(and)f(size)h(of)18 b Fr(new)p 1802 2138 13 2 v 16 w(group)p Fu(\))117 2217 y(IN)171 b Fo(ranks)471 b Fu(arra)o(y)11 b(of)f(in)o(teger)i(ranks)f(in)f Fr(group)i Fu(to)f(app)q(ear)g(in)g Fr(new)p 1751 2217 V 16 w(group)p Fu(.)117 2295 y(OUT)124 b Fo(new)p 411 2295 14 2 v 17 w(group)372 b Fu(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i (preserving)h(the)f(order)905 2352 y(de\014ned)d(b)o(y)28 b Fr(ranks)p Fu(.)75 2478 y FE(The)18 b(function)g Fo(MPI)p 436 2478 V 16 w(LOCAL)p 594 2478 V 17 w(SUBGROUP)i FE(creates)d(a)g (group)h Fp(new)p 1267 2478 15 2 v 16 w(group)f FE(whic)o(h)i(is)f(a)g (subgroup)f(of)75 2534 y Fp(old)p 150 2534 V 17 w(group)10 b FE(suc)o(h)h(that)f(the)h(pro)q(cess)g(with)g(rank)f Fp(i)h FE(in)h Fp(new)p 1072 2534 V 16 w(group)e FE(is)i(the)e(pro)q (cess)i(with)f(rank)f Fp(ranks[i])75 2591 y FE(in)17 b Fp(old)p 204 2591 V 17 w(group)p FE(,)f(and)h(the)g(size)g(of)g Fp(new)p 753 2591 V 16 w(group)f FE(is)h Fp(n)p FE(.)24 b(Eac)o(h)17 b(of)f(the)h Fp(n)f FE(elemen)o(ts)i(of)e Fp(ranks)g FE(m)o(ust)g(b)q(e)h(a)75 2647 y(v)m(alid)i(rank)d(in)i Fp(old)p 423 2647 V 17 w(group)e FE(and)h(all)h(elemen)o(ts)g(m)o(ust)e (b)q(e)i(distinct,)g(else)g(the)f(program)f(is)h(erroneous.)75 2704 y(If)e(no)h(ranks)e(are)h(sp)q(eci\014ed,)i Fo(new)p 655 2704 14 2 v 17 w(group)f FE(has)f(no)g(mem)o(b)q(ers.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 16 22 16 21 bop 75 -100 a FE(16)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(LOCAL)p 318 45 V 16 w(EX)o(CL)p 444 45 V 16 w(SUBGROUP\(group,)i(n,)e(ranks,)h(new)p 1108 45 V 17 w(group\))117 126 y Fu(IN)171 b Fo(group)463 b Fu(handle)14 b(to)g(group)f(ob)r(ject)117 208 y(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)9 b(of)f(elemen)o(ts)i(in)e(arra)o(y)h (ranks)h(\(and)f(size)h(of)18 b Fr(new)p 1802 208 13 2 v 16 w(group)p Fu(\))117 291 y(IN)171 b Fo(ranks)471 b Fu(arra)o(y)9 b(of)g(in)o(teger)h(ranks)f(in)g Fr(group)h Fu(not)f(to)g(app)q(ear)h(in)f Fr(new)p 1805 291 V 16 w(group)117 373 y Fu(OUT)124 b Fo(new)p 411 373 14 2 v 17 w(group)372 b Fu(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o (e,)i(preserving)h(the)f(order)905 429 y(de\014ned)d(b)o(y)28 b Fr(ranks)p Fu(.)75 557 y FE(The)11 b(function)g Fo(MPI)p 422 557 V 16 w(LOCAL)p 580 557 V 16 w(EX)o(CL)p 706 557 V 16 w(SUBGROUP)i FE(creates)d(a)h(group)f Fp(new)p 1350 557 15 2 v 17 w(group)f FE(whic)o(h)j(is)f(a)f(subgroup)75 614 y(of)j Fp(old)p 200 614 V 17 w(group)g FE(suc)o(h)i(that)e Fp(new)p 621 614 V 17 w(group)g FE(con)o(tains)h(all)h(pro)q(cess)f(in) h Fp(old)p 1291 614 V 16 w(group)e FE(except)i(the)f Fp(n)g FE(pro)q(cesses)75 670 y(with)i(ranks)e Fp(ranks[i],)23 b(i)h(=)g(0,)f(...,)g(n)h(-)g(1)p FE(,)15 b(and)g(the)g(ordering)h(of)f (pro)q(cesses)g(in)h Fp(new)p 1696 670 V 17 w(group)e FE(is)75 727 y(iden)o(tical)20 b(to)d(the)h(ordering)g(in)h Fp(old)p 713 727 V 17 w(group)p FE(.)27 b(Eac)o(h)18 b(of)f(the)h Fp(n)g FE(elemen)o(ts)g(of)g Fp(ranks)f FE(m)o(ust)g(b)q(e)i(a)e(v)m(alid)75 783 y(rank)f(in)h Fp(old)p 310 783 V 17 w(group)f FE(and)h(all)g(elemen)o(ts)g(m)o(ust)f (b)q(e)h(distinct,)h(else)f(the)g(program)e(is)i(erroneous.)23 b(If)17 b(no)75 840 y(ranks)e(are)g(sp)q(eci\014ed,)i Fo(new)p 547 840 14 2 v 17 w(group)e FE(is)h(iden)o(tical)h(to)d Fo(group)p FE(.)75 947 y Fo(MPI)p 160 947 V 16 w(LOCAL)p 318 947 V 16 w(SUBGROUP)p 572 947 V 19 w(RANGES\(group,)i(n,)f(ranges,) g(new)p 1193 947 V 17 w(group\))117 1028 y Fu(IN)171 b Fo(group)463 b Fu(handle)14 b(to)g(group)f(ob)r(ject)117 1110 y(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)13 b(of)h(triplets)g(in)f (arra)o(y)h Fn(ranges)117 1193 y Fu(IN)171 b Fo(ranges)450 b Fu(a)19 b(one-dimensional)e(arra)o(y)i(of)g(in)o(teger)g(triplets:)29 b(pairs)20 b(of)905 1249 y(ranks)c(\(form:)j(b)q(eginning)c(through)g (end,)h(inclusiv)o(e\))f(to)g(b)q(e)905 1305 y(included)i(in)g(the)g (output)g(group)g Fr(new)p 1529 1305 13 2 v 16 w(group)p Fu(,)h(plus)e(a)h(con-)905 1362 y(stan)o(t)d(stride)h(\(often)f(1)g(or) g(-1\).)117 1444 y(OUT)124 b Fo(new)p 411 1444 14 2 v 17 w(group)372 b Fu(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o (e,)i(preserving)h(the)f(order)905 1501 y(de\014ned)d(b)o(y)28 b Fr(ranges)p Fu(.)75 1629 y FE(If)17 b(an)o(y)f(of)h(the)g(rank)f (sets)g(o)o(v)o(erlap,)h(then)g(call)h(is)f(erroneous.)24 b(If)17 b(no)g(ranges)f(are)h(sp)q(eci\014ed,)i(then)e(the)75 1685 y(output)e(group)g(has)g(no)g(mem)o(b)q(ers.)166 1745 y(The)f(functionalit)o(y)h(of)f(this)g(routine)g(is)h(sp)q (eci\014ed)h(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)f(expanding)h(the)f (arra)o(y)e(of)75 1802 y(ranges)i(to)g(an)g(arra)o(y)f(of)h(the)g (included)j(ranks)e(and)f(passing)h(the)f(resulting)i(arra)o(y)d(of)h (ranks)g(and)g(other)75 1858 y(argumen)o(ts)g(to)h(MPI)p 444 1858 V 16 w(LOCAL)p 618 1858 V 18 w(SUBGR)o(OUP)l(.)75 1965 y Fo(MPI)p 160 1965 V 16 w(LOCAL)p 318 1965 V 16 w(SUBGROUP)p 572 1965 V 19 w(EX)o(CL)p 701 1965 V 16 w(RANGES\(group,)h(n,)f(ranges,)g(new)p 1319 1965 V 17 w(group\))117 2046 y Fu(IN)171 b Fo(group)463 b Fu(handle)14 b(to)g(group)f(ob)r(ject)117 2129 y(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)9 b(of)f(elemen)o(ts)i(in)e(arra)o(y)h(ranks)h (\(and)f(size)h(of)18 b Fr(new)p 1802 2129 13 2 v 16 w(group)p Fu(\))117 2211 y(IN)171 b Fo(ranges)450 b Fu(a)17 b(one-dimensional)e(arra)o(y)i(of)g(\(three)i(in)o(teger\))f (consisting)905 2267 y(of)12 b(pairs)h(of)f(ranks)i(\(form:)h(b)q (eginning)e(through)f(end,)i(inclu-)905 2324 y(siv)o(e\))d(to)g(b)q(e)h (excluded)g(from)d(the)j(output)f(group)g Fr(new)p 1751 2324 V 16 w(group)p Fu(,)905 2380 y(plus)j(a)g(constan)o(t)g(stride)h (\(often)f(1)f(or)h(-1\).)117 2463 y(OUT)124 b Fo(new)p 411 2463 14 2 v 17 w(group)372 b Fu(new)18 b(group)g(deriv)o(ed)g(from) e(ab)q(o)o(v)o(e,)i(preserving)h(the)f(order)905 2519 y(de\014ned)d(b)o(y)28 b Fr(ranges)p Fu(.)75 2647 y FE(If)16 b(an)o(y)g(of)g(the)g(rank)g(sets)f(o)o(v)o(erlap,)h(then)h(call)g(is)f (erroneous.)23 b(If)16 b(there)g(are)g(no)g(ranges)g(sp)q(eci\014ed,)i (the)75 2704 y(output)d(group)g(is)h(the)f(same)g(as)g(the)g(original)h (group.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 23 17 22 bop 75 -100 a Fx(3.5.)34 b(CONTEXT)15 b(MANA)o(GEMENT)1051 b FE(17)166 45 y(The)14 b(functionalit)o(y)h(of)f(this)g(routine)g(is)h (sp)q(eci\014ed)h(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)f(expanding)h(the) f(arra)o(y)e(of)75 102 y(ranges)i(to)g(an)g(arra)o(y)f(of)h(the)g (included)j(ranks)e(and)f(passing)h(the)f(resulting)i(arra)o(y)d(of)h (ranks)g(and)g(other)75 158 y(argumen)o(ts)g(to)h(MPI)p 444 158 14 2 v 16 w(LOCAL)p 618 158 V 18 w(EX)o(CL)p 761 158 V 16 w(SUBGR)o(OUP)l(.)166 300 y Fv(Discussion:)f Fu(Please)f(prop)q(ose)f(additional)e(subgroup)i(functions,)g(b)q (efore)h(the)f(second)h(reading...Virtual)75 357 y(T)m(op)q(ologies)f (supp)q(ort?)166 582 y Fv(Discussion:)32 b Fu(What)22 b(do)g(p)q(eople)h(think)f(ab)q(out)g(these)i(lo)q(cal)d(op)q (erations?)44 b(More?)g(Less?)g(Note:)75 638 y(these)15 b(op)q(erations)g(do)f(not)f(explicitly)g(en)o(umerate)h(ranks,)g(and)g (therefore)i(are)e(more)f(scalable)h(if)f(implemen)o(ted)75 695 y(e\016cien)o(tly)p Fm(:)7 b(:)g(:)75 884 y Fo(MPI)p 160 884 V 16 w(GROUP)p 328 884 V 18 w(DUP\(group,)14 b(new)p 666 884 V 17 w(group\))117 965 y Fu(IN)171 b Fo(group)463 b Fu(extan)o(t)14 b(group)g(ob)r(ject)h(handle)117 1046 y(OUT)124 b Fo(new)p 411 1046 V 17 w(group)372 b Fu(new)15 b(group)e(ob)r(ject)i(handle)75 1174 y Fo(MPI)p 160 1174 V 16 w(GROUP)p 328 1174 V 18 w(DUP)k FE(duplicates)j(a)d (group)h(with)g(all)h(its)f(cac)o(hed)g(information,)h(replacing)g (nothing.)75 1231 y(This)16 b(function)g(is)g(essen)o(tial)g(to)e(the)h (supp)q(ort)h(of)f(virtual)g(top)q(ologies.)75 1371 y Fq(3.4.5)49 b(Group)16 b(Destructo)o(rs)75 1510 y Fo(MPI)p 160 1510 V 16 w(GROUP)p 328 1510 V 18 w(FREE\(group\))117 1590 y Fu(IN)171 b Fo(group)463 b Fu(frees)29 b Fr(group)15 b Fu(previously)e(de\014ned.)75 1718 y FE(This)i(op)q(eration)f(frees)h (a)f(handle)29 b Fo(group)15 b FE(whic)o(h)g(is)f(not)g(curren)o(tly)h (b)q(ound)g(to)f(a)g(comm)o(unicator.)19 b(It)14 b(is)75 1775 y(erroneous)h(to)g(attempt)f(to)g(free)i(a)f(group)f(curren)o(tly) i(b)q(ound)g(to)f(a)g(comm)o(unicator.)75 1936 y Fy(3.5)59 b(Context)18 b(Management)75 2046 y Fq(3.5.1)49 b(Context)16 b(Allo)q(cation)75 2185 y Fo(MPI)p 160 2185 V 16 w(CONTEXTS)p 414 2185 V 17 w(ALLOC\(comm,)d(n\))117 2265 y Fu(IN)171 b Fo(comm)454 b Fu(a)14 b(comm)o(unicator)117 2347 y(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)13 b(of)h(con)o(texts)h(to)e(allo)q (cate)75 2475 y FE(This)k(function)h(allo)q(cates)g(an)e(arra)o(y)g(of) g(con)o(texts,)h(whic)o(h)g(are)g(unique)h(within)g(the)f(group)g(b)q (ound)h(to)75 2531 y(the)13 b(comm)o(unicator)g Fp(comm)p FE(,)g(and)g(stores)g(them)g(in)h(the)f(un)o(used)h(con)o(text)f(store) f(of)h Fp(comm)p FE(.)19 b(The)13 b(function)75 2587 y(call)j(is)g(global)g(\(i.e.)k(collectiv)o(e\))c(in)h Fp(comm)p FE(.)166 2647 y FF(Note:)i FE(MPI)g(pro)o(vides)h(sp)q(ecial) h(con)o(textual)e(space)h(to)f(the)g(collectiv)o(e)i(op)q(erations)f (\(including)75 2704 y Fo(MPI)p 160 2704 V 16 w(CONTEXTS)p 414 2704 V 17 w(ALLOC)p FE(\))f(so)g(that,)h(despite)g(an)o(y)f (on-going)g(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unication)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 18 24 18 23 bop 75 -100 a FE(18)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y FE(on)k(the)g(comm)o(unicator)g(argumen)o(t,)g(the)g(collcetiv)o (e)i(op)q(eration)f(can)f(execute)h(safely)l(.)32 b(MPI)19 b(collec-)75 102 y(tiv)o(e)f(functions)g(will)h(ha)o(v)o(e)e(to)f(lo)q (c)o(k)i(out)f(m)o(ultiple)j(threads,)d(so)g(it)h(will)h(eviden)o(tly)g (ha)o(v)o(e)e(capabilities)75 158 y(una)o(v)m(ailable)g(to)e(the)g (user)h(program.)166 303 y Fv(Discussion:)166 366 y Fu(W)m(e)10 b(ha)o(v)o(e)g(to)h(retain)g(the)g(c)o(hic)o(k)o(en-and-egg)f(asp)q (ect)i(of)e(MPI)h(\()p Fl(i.e.)p Fu(,)f(use)h(a)g(comm)o(uni)o(cator)d (to)j(get)g(con)o(text\(s\))75 416 y(or)i(a)h(comm)o(unicator\),)c(to)j (get)h(thread)g(safet)o(y)m(.)k(Y)m(et,)13 b(w)o(e)h(w)o(an)o(t)f (libraries)g(to)g(con)o(trol)h(their)g(o)o(wn)f(fate)g(regarding)75 466 y(safet)o(y)m(,)c(not)h(to)f(rely)g(on)g(the)h(caller)f(to)h(pro)o (vide)f(a)g(quiescen)o(t)h(con)o(text.)17 b(W)m(e)9 b(ac)o(hiev)o(e)h (this)f(b)o(y)g(adding)g(the)h Fk(quiescen)o(t)75 515 y(prop)q(ert)o(y)20 b Fu(for)15 b(MPI)h(collectiv)o(e)g(comm)o (unication)c(functions.)24 b(W)m(e)15 b(m)o(ust,)g(in)g(fact,)h(push)g (this)f(requiremen)o(t)h(to)75 565 y(the)k(collectiv)o(e)e(c)o(hapter,) j(but)e(w)o(e)g(demonstrate)g(here)h(wh)o(y)f(our)g(particular)g (collectiv)o(e)g(routines)g(need)h(this)75 615 y(prop)q(ert)o(y)m(.)28 b(In)17 b(a)g(m)o(ulti-threaded)f(en)o(vironmen)o(t,)g(it)h(is)g(clear) g(that)g(eac)o(h)h(temp)q(orally)d(o)o(v)o(erlapping)h(call)g(to)h(a)75 665 y(collectiv)o(e)f(op)q(eration)f(m)o(ust)g(b)q(e)h(with)f(a)h (di\013eren)o(t)g(comm)o(unicator.)21 b(If)15 b(this)h(has)f(not)h(b)q (een)h(made)d(explicit,)h(it)75 715 y(m)o(ust)e(b)q(e.)166 777 y(One)f(can)f(ha)o(v)o(e)g(on-going)f(p)q(oin)o(t-to-p)q(oin)o(t)g (and)h(collectiv)o(e)g(comm)o(unications)d(on)j(a)g(single)g(comm)o (unicator.)75 827 y(A)i(con)o(text)g(is)f(de\014ned)h(to)g(b)q(e)g (su\016cien)o(tly)f(p)q(o)o(w)o(erful)g(to)g(k)o(eep)h(b)q(oth)g(p)q (oin)o(t-to-p)q(oin)o(t)e(and)h(collectiv)o(e)h(op)q(erations)75 877 y(distinct.)19 b(Hence,)c(it)f(is)g(alw)o(a)o(ys)f(safe)h(to)g (call)27 b Fr(MPI)p 894 877 13 2 v 15 w(COMM)p 1039 877 V 14 w(MAKE)14 b Fu(and)28 b Fr(MPI)p 1355 877 V 14 w(CONTEXTS)p 1588 877 V 15 w(ALLOC)p Fu(,)13 b(ev)o(en)i(if)75 927 y(p)q(ending)h(async)o(hronous)h(p)q(oin)o(t-to-p)q(oin)o(t)e(op)q (erations)i(are)f(on-going,)f(or)i(messages)f(ha)o(v)o(e)g(not)g(b)q (een)i(receiv)o(ed)75 977 y(but)g(are)h(on)f(the)h(receipt)g(queue.)32 b(With)17 b(these)j(rules,)f(no)f(quiescen)o(t)h(comm)o(unicator)d(is)h (required)j(in)d(order)75 1026 y(to)e(get)h(new)g(con)o(texts.)24 b(W)m(e)15 b(ha)o(v)o(e)h(added)g(demands)f(on)g(the)h(MPI)g(implem)o (en)o(tation)d(while)i(making)e(con)o(texts)75 1076 y(opaque)h(to)g (mak)o(e)e(this)i(simpler)f(to)g(realize)i(without)e(sa)o(ying)g(ho)o (w)h(it)f(m)o(ust)g(b)q(e)h(done.)166 1146 y(In)e(summary)m(,)d (libraries)i(ha)o(v)o(e)h(to)g(get)g(the)g(comm)o(unicator)e(as)i(a)f (base)i(argumen)o(t)d(to)i(retain)g(thread)h(safet)o(y)m(,)75 1202 y(but)d(they)h(can)f(alw)o(a)o(ys)f(safely)h(get)g(comm)o (unicatio)o(n)e(con)o(texts)j(to)e(do)h(further)h(w)o(ork.)17 b(The)10 b(concept)h(of)f(quiescence)75 1258 y(is)g(banished)h(to)f(b)q (e)g(a)g(small)e(detail)i(of)f(implemen)o(tation,)e(rather)k(than)g(a)e (cen)o(tral)i(tenet)g(of)f(library)f(design.)17 b(Users)75 1315 y(m)o(ust)9 b(still)h(w)o(orry)g(ab)q(out)h(temp)q(oral)e(safet)o (y)m(,)h(whic)o(h)g(is)g(not)h(guaran)o(teed)g(b)o(y)f(con)o(texts)h (alone)f(\(see)i(example)d(b)q(elo)o(w)75 1371 y(in)k(section)i (3.10.6\).)75 1649 y Fq(3.5.2)49 b(Context)16 b(Deallo)q(cation)75 1807 y Fo(MPI)p 160 1807 14 2 v 16 w(CONTEXTS)p 414 1807 V 17 w(FREE\(comm,)e(n\))117 1897 y Fu(IN)155 b Fo(comm)470 b Fu(a)15 b FE(comm)o(unicator)117 1998 y Fu(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)13 b(of)h(con)o(texts)h(to)e(free)75 2135 y FE(This)e(function)g(frees)f(\(i.e.,)h(returns)f(to)f(the)h(MPI) h(system\))e(up)i(to)e Fp(n)h FE(con)o(texts)g(from)f(the)i(un)o(used)g (con)o(text)75 2192 y(store)e(of)h(the)g(comm)o(unicator)g Fp(comm)p FE(,)g(whic)o(h)h(w)o(ere)f(previously)i(allo)q(cated)f(b)o (y)f(calling)22 b Fo(MPI)p 1630 2192 V 16 w(CONTEXTS)p 1884 2192 V 17 w(ALLOC)75 2248 y FE(\(b)q(elo)o(w\).)e(The)15 b(function)h(call)h(is)e(lo)q(cal)i(but)e FF(m)o(ust)f FE(b)q(e)i(called)h(as)e(though)g(it)g(is)h(collectiv)o(e.)75 2465 y Fy(3.6)59 b(Intra-Communication)17 b(Communicato)n(r)i (Management)75 2593 y Fq(3.6.1)49 b(Communicato)o(r)16 b(Accesso)o(rs)75 2704 y FE(The)f(follo)o(wing)h(are)f(all)h(lo)q(cal)h (\(non-comm)o(unicating\))e(op)q(erations.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 19 25 19 24 bop 75 -100 a Fx(3.6.)34 b(INTRA-COMMUNICA)l(TION)17 b(COMMUNICA)l(TOR)f(MANA)o(GEMENT)284 b FE(19)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(COMM)p 318 45 V 16 w(SIZE\(comm,)13 b(size\))117 122 y Fu(IN)171 b Fo(comm)454 b Fu(handle)14 b(to)g(comm)o(unicator)d(ob)r(ject.)117 198 y(OUT)124 b Fo(size)503 b Fu(is)22 b(the)g(in)o(teger)g(n)o(um)o(b)q(er)f(of)g (pro)q(cesses)k(in)c(the)h(group)g(of)905 254 y Fr(comm)p Fu(.)75 426 y Fo(MPI)p 160 426 V 16 w(COMM)p 318 426 V 16 w(RANK\(comm,)13 b(rank\))117 503 y Fu(IN)171 b Fo(comm)454 b Fu(handle)14 b(to)g(comm)o(unicator)d(ob)r(ject.)117 579 y(OUT)124 b Fo(rank)488 b Fu(is)17 b(the)g(in)o(teger)g(rank)g(of)f (the)i(calling)d(pro)q(cess)k(in)d(group)h(of)905 635 y Fr(comm)p Fu(,)h(or)39 b Fr(MPI)p 1195 635 13 2 v 14 w(UNDEFINED)18 b Fu(if)g(the)i(pro)q(cess)h(is)e(not)g(a)905 692 y(mem)o(b)q(er.)75 863 y Fo(MPI)p 160 863 14 2 v 16 w(COMM)p 318 863 V 16 w(GROUP\(comm,)14 b(group\))117 941 y Fu(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(ob)r(ject)k(handle)117 1016 y(OUT)124 b Fo(group)463 b Fu(group)14 b(ob)r(ject)h(handle)e(corresp)q(onding)i(to)f Fn(comm)75 1188 y Fo(MPI)p 160 1188 V 16 w(COMM)p 318 1188 V 16 w(CONTEXTS\(comm,)g(n\))117 1265 y Fu(IN)171 b Fo(comm)454 b Fu(handle)14 b(to)g(comm)o(unicator)d(ob)r(ject.)117 1341 y(OUT)124 b Fo(n)548 b Fu(the)11 b(n)o(um)o(b)q(er)f(of)g(con)o (texts)h(a)o(v)n(ailable)d(in)i(the)h(un)o(used)g(con)o(text)905 1397 y(store)k(asso)q(ciated)g(with)e Fn(comm)75 1588 y Fq(3.6.2)49 b(Communicato)o(r)16 b(Constructo)o(rs)75 1721 y Fo(MPI)p 160 1721 V 16 w(COMM)p 318 1721 V 16 w(DUP\(comm,)d(new)p 662 1721 V 17 w(comm\))117 1798 y Fu(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(ob)r(ject)k(handle) 117 1874 y(OUT)124 b Fo(new)p 411 1874 V 17 w(comm)363 b Fu(comm)o(unicator)11 b(ob)r(ject)k(handle)75 2045 y Fo(MPI)p 160 2045 V 16 w(COMM)p 318 2045 V 16 w(LOCAL)p 476 2045 V 17 w(DUP\(comm,)d(new)p 820 2045 V 17 w(comm\))117 2123 y Fu(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(ob)r(ject)k(handle)117 2198 y(OUT)124 b Fo(new)p 411 2198 V 17 w(comm)363 b Fu(comm)o(unicator)11 b(ob)r(ject)k(handle)75 2370 y Fo(MPI)p 160 2370 V 16 w(COMM)p 318 2370 V 16 w(COLL)p 446 2370 V 16 w(DUP\(comm,)e(new)p 790 2370 V 17 w(comm\))117 2447 y Fu(IN)171 b Fo(comm)454 b Fu(comm)o(unicator) 11 b(ob)r(ject)k(handle)117 2523 y(OUT)124 b Fo(new)p 411 2523 V 17 w(comm)363 b Fu(comm)o(unicator)11 b(ob)r(ject)k(handle) 75 2647 y FE(Eac)o(h)e(of)g(these)g(three)h(functions)g(duplicates)h (the)e(existing)h(comm)o(unicator)f Fp(comm)g FE(with)h(all)g(its)f (cac)o(hed)75 2704 y(information,)27 b(replacing)f(just)e(the)h(con)o (text,)g(and)g(creates)f(the)h(new)g(comm)o(unicator)f Fp(new)p 1753 2704 15 2 v 17 w(comm)p FE(.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 26 20 25 bop 75 -100 a FE(20)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(COMM)p 318 45 V 16 w(DUP)d FE(is)g(a)f(collectiv)o(e)i(op)q(eration)f(whic)o(h)h(will)g(tak)o(e)e (a)g(con)o(text)g(from)g(the)g(un)o(used)i(con)o(text)75 102 y(store)i(asso)q(ciated)h(with)g Fp(comm)f FE(if)h(suc)o(h)f(a)h (con)o(text)f(is)h(a)o(v)m(ailable,)h(else)f(it)g(will)h(allo)q(cate)f (a)g(new)g(con)o(text)75 158 y(for)g(the)h(comm)o(unicator)g Fp(new)p 600 158 15 2 v 17 w(comm)p FE(.)24 b Fo(MPI)p 832 158 14 2 v 16 w(COMM)p 990 158 V 16 w(LOCAL)p 1148 158 V 17 w(DUP)16 b FE(is)i(a)e(lo)q(cal)i(op)q(eration)g(whic)o(h)f (will)75 214 y(tak)o(e)12 b(a)h(con)o(text)f(from)g(the)i(un)o(used)f (con)o(text)g(store)f(asso)q(ciated)h(with)g Fp(comm)g FE(if)g(suc)o(h)g(a)g(con)o(text)f(is)i(a)o(v)m(ail-)75 271 y(able,)i(else)g(it)g(will)h(fail)f(and)g(return)g(the)f(n)o(ull)i (comm)o(unicator)e Fr(MPI)p 1250 271 13 2 v 15 w(COMM)p 1395 271 V 14 w(NULL)i FE(in)f Fp(new)p 1654 271 15 2 v 17 w(comm)p FE(.)k(The)75 327 y(function)14 b(is)g(lo)q(cal)g(but)g FF(m)o(ust)e FE(b)q(e)i(called)h(as)e(though)o(t)f(it)i(is)g(collectiv) o(e.)21 b Fo(MPI)p 1403 327 14 2 v 16 w(COMM)p 1561 327 V 16 w(COLL)p 1689 327 V 16 w(DUP)13 b FE(is)h(a)75 384 y(collectiv)o(e)j(op)q(eration)e(whic)o(h)g(will)i(allo)q(cate)e(a)g (new)g(con)o(text)f(for)g(the)h(comm)o(unicator)f Fp(new)p 1667 384 15 2 v 17 w(comm)p FE(.)19 b(See)75 440 y(also)c(the)g (considerations)i(of)d(program)g(correctness)h(b)q(elo)o(w.)75 550 y Fo(MPI)p 160 550 14 2 v 16 w(COMM)p 318 550 V 16 w(MAKE\(sync)p 562 550 V 17 w(comm,)e(comm)p 839 550 V 15 w(group,)h(comm)p 1106 550 V 15 w(new\))117 632 y Fu(IN)171 b Fo(sync)p 418 632 V 17 w(comm)356 b Fu(Comm)o(unicator)11 b(within)i(whic)o(h)h(con)o(text)g(is)g(unique)117 719 y(IN)155 b Fo(comm)p 438 719 V 15 w(group)347 b Fu(Group)14 b(whic)o(h)g(is)f(a)h(subset)h(of)e(the)i(group)f(of)f Fn(sync)p 1732 719 V 15 w(comm)117 805 y Fu(OUT)124 b Fo(comm)p 454 805 V 14 w(new)366 b Fu(The)15 b(new)f(comm)o(unicator.) 75 982 y Fo(MPI)p 160 982 V 16 w(COMM)p 318 982 V 16 w(LOCAL)p 476 982 V 17 w(MAKE\(sync)p 721 982 V 17 w(comm,)e(comm)p 997 982 V 15 w(group,)j(comm)p 1265 982 V 14 w(new\))117 1065 y Fu(IN)171 b Fo(sync)p 418 1065 V 17 w(comm)356 b Fu(Comm)o(unicator)11 b(within)i(whic)o(h)h(con)o(text)g(is)g(unique) 117 1151 y(IN)155 b Fo(comm)p 438 1151 V 15 w(group)347 b Fu(Group)14 b(whic)o(h)g(is)f(a)h(subset)h(of)e(the)i(group)f(of)f Fn(sync)p 1732 1151 V 15 w(comm)117 1238 y Fu(OUT)124 b Fo(comm)p 454 1238 V 14 w(new)366 b Fu(The)15 b(new)f(comm)o (unicator.)75 1415 y Fo(MPI)p 160 1415 V 16 w(COMM)p 318 1415 V 16 w(COLL)p 446 1415 V 16 w(MAKE\(sync)p 690 1415 V 17 w(comm,)f(comm)p 967 1415 V 15 w(group,)h(comm)p 1234 1415 V 15 w(new\))117 1498 y Fu(IN)171 b Fo(sync)p 418 1498 V 17 w(comm)356 b Fu(Comm)o(unicator)11 b(within)i(whic)o(h)h (con)o(text)g(is)g(unique)117 1584 y(IN)155 b Fo(comm)p 438 1584 V 15 w(group)347 b Fu(Group)14 b(whic)o(h)g(is)f(a)h(subset)h (of)e(the)i(group)f(of)f Fn(sync)p 1732 1584 V 15 w(comm)117 1670 y Fu(OUT)124 b Fo(comm)p 454 1670 V 14 w(new)366 b Fu(The)15 b(new)f(comm)o(unicator.)75 1800 y FE(Eac)o(h)22 b(of)f(these)h(three)g(functions)g(creates)g(a)f(new)h(comm)o(unicator) f Fp(new)p 1382 1800 15 2 v 17 w(comm)g FE(whic)o(h)i(com)o(bines)f(a) 75 1857 y(con)o(text)17 b(unique)i(within)g(the)f(group)f(of)h(the)g (existing)g(comm)o(unicator)g Fp(sync)p 1450 1857 V 16 w(comm)p FE(,)g(and)f(the)h(group)75 1913 y Fp(comm)p 174 1913 V 17 w(group)p FE(.)g(No)12 b(cac)o(hed)g(information)g (propagates)f(from)h Fp(sync)p 1226 1913 V 16 w(comm)g FE(to)f Fp(comm)p 1498 1913 V 17 w(new)p FE(.)18 b(F)l(or)12 b(pro)q(cesses)75 1970 y(in)19 b Fp(sync)p 230 1970 V 17 w(group)e FE(whic)o(h)i(are)f(not)g(in)h Fp(comm)p 832 1970 V 17 w(group)p FE(,)f(these)h(functions)g(return)f Fp(comm)p 1557 1970 V 16 w(new)g FE(as)g(the)h(n)o(ull)75 2026 y(comm)o(unicator)14 b Fr(MPI)p 448 2026 13 2 v 14 w(COMM)p 592 2026 V 15 w(NULL)p FE(.)h(If)f(the)g(set)g(of)g(pro)q (cesses)h(in)g Fp(comm)p 1324 2026 15 2 v 16 w(group)f FE(is)h(not)f(a)f(subset)i(of,)e(or)75 2083 y(equal)19 b(to,)e(the)i(set)e(of)h(pro)q(cesses)g(in)h(the)f(group)g(of)g Fp(sync)p 1101 2083 V 16 w(group)g FE(then)g(the)g(program)f(is)h (erroneous.)75 2139 y(This)12 b(error)f(condition)i(can)f(b)q(e)g(c)o (hec)o(k)o(ed.)19 b(If)12 b(the)g(group)f Fp(comm)p 1141 2139 V 17 w(group)g FE(is)h(not)f(iden)o(tical)j(in)e(all)h(pro)q (cesses)75 2195 y(calling)18 b(the)f(function,)h(then)f(the)g(program)e (is)i(again)g(erroneous.)24 b(This)18 b(error)e(condition)i(cannot)e(b) q(e)75 2252 y(c)o(hec)o(k)o(ed.)30 b Fo(MPI)p 354 2252 14 2 v 16 w(COMM)p 512 2252 V 17 w(MAKE)18 b FE(is)h(a)f(collectiv)o(e) j(op)q(eration)d(whic)o(h)i(will)g(tak)o(e)e(a)g(con)o(text)g(from)g (the)75 2308 y(un)o(used)h(con)o(text)e(store)g(asso)q(ciated)i(with)f Fp(sync)p 934 2308 15 2 v 17 w(comm)f FE(if)h(suc)o(h)h(a)e(con)o(text) h(is)g(a)o(v)m(ailable,)i(else)f(it)f(will)75 2365 y(allo)q(cate)i(a)f (new)h(con)o(text)f(for)f(the)i(comm)o(unicator)f Fp(new)p 1085 2365 V 17 w(comm)p FE(.)32 b Fo(MPI)p 1325 2365 14 2 v 15 w(COMM)p 1482 2365 V 17 w(LOCAL)p 1641 2365 V 16 w(MAKE)19 b FE(is)h(a)75 2421 y(lo)q(cal)h(op)q(eration)f(whic)o (h)g(will)i(tak)o(e)d(a)g(con)o(text)g(from)g(the)h(un)o(used)h(con)o (text)e(store)g(asso)q(ciated)h(with)75 2478 y Fp(sync)p 174 2478 15 2 v 17 w(comm)15 b FE(if)i(suc)o(h)g(a)f(con)o(text)f(is)i (a)o(v)m(ailable,)h(else)f(it)g(will)g(fail)h(and)e(return)g(the)h(n)o (ull)g(comm)o(unicator)75 2534 y Fr(MPI)p 152 2534 13 2 v 14 w(COMM)p 296 2534 V 15 w(NULL)24 b FE(in)f Fp(new)p 570 2534 15 2 v 17 w(comm)p FE(.)41 b(The)23 b(function)h(is)f(lo)q (cal)h(but)f FF(m)o(ust)e FE(b)q(e)j(called)g(as)e(though)h(it)75 2591 y(iscollectiv)o(e.)33 b Fo(MPI)p 416 2591 14 2 v 16 w(COMM)p 574 2591 V 16 w(COLL)p 702 2591 V 16 w(MAKE)19 b FE(is)g(a)f(collectiv)o(e)j(op)q(eration)e(whic)o(h)g(will)i(allo)q (cate)e(a)g(new)75 2647 y(con)o(text)13 b(for)f(the)h(comm)o(unicator)g Fp(new)p 748 2647 15 2 v 17 w(comm)p FE(.)18 b(See)c(also)f(the)g (considerations)h(of)f(program)f(correctness)75 2704 y(b)q(elo)o(w.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 27 21 26 bop 75 -100 a Fx(3.7.)34 b(INTER-COMMUNICA)l(TION)17 b(COMMUNICA)l(TOR)f(MANA)o(GEMENT)287 b FE(21)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(COMM)p 318 45 V 16 w(COLL)p 446 45 V 16 w(SUBSET\(comm,)14 b(colour,)g(k)o(ey)l(,)h(comm)p 1139 45 V 15 w(new\))117 122 y Fu(IN)155 b Fo(comm)470 b Fu(An)14 b(existing)g(comm)o(unicator)117 198 y(IN)155 b Fo(colour)471 b Fu(con)o(trol)14 b(of)f(subset)i(assignmen)o(t)117 273 y(IN)155 b Fo(k)o(ey)525 b Fu(con)o(trol)14 b(of)f(rank)h(assigmen) o(t)117 348 y(OUT)124 b Fo(comm)p 454 348 V 14 w(new)366 b Fu(A)14 b(new)h(comm)o(uni)o(cator.)166 556 y Fv(Discussion:)26 b Fu(Since)20 b(this)f(function)g(m)o(ust)f(b)q(e)i(collectiv)o(e)f(w)o (e)g(do)g(not)g(need)i(to)e(consider)h(whether)g(it)75 612 y(sync)o(hronises)15 b(to)e(allo)q(cate)g(a)g(new)h(con)o(text)g (when)g(the)g(un)o(used)g(con)o(text)h(store)f(is)f(empt)o(y)m(.)k(Ho)o (w)o(ev)o(er)c(w)o(e)h(should)75 668 y(probably)h(sa)o(y)g(whether)i (it)e(tak)o(es)h(a)g(con)o(text)g(from)e(the)i(store)g(or)g(not.)23 b(I)15 b(ha)o(v)o(e)h(said)f(that)h(it)f(allo)q(cates)g(a)h(new)75 725 y(con)o(text,)e(as)g(opp)q(osed)h(to)e(using)h(the)g(store,)h(but)f (w)o(e)g(ma)o(y)e(wish)i(to)f(c)o(hange)i(this.)75 930 y Fq(3.6.3)49 b(Communicato)o(r)16 b(Destructo)o(rs)75 1063 y Fo(MPI)p 160 1063 V 16 w(COMM)p 318 1063 V 16 w(FREE\(comm\))117 1140 y Fu(IN)171 b Fo(comm)454 b Fu(the)15 b(comm)o(unicator)c(to)j(b)q(e)g(destro)o(y)o(ed.)75 1265 y FE(This)21 b(function)f(destro)o(ys)g(the)g(comm)o(unicator)f Fp(comm)p FE(.)34 b(The)20 b(function)h(call)g(is)g(lo)q(cal)g(but)f FF(m)o(ust)f FE(b)q(e)75 1321 y(called)d(as)d(though)h(it)g(is)h (collectiv)o(e.)21 b(If)15 b(the)f(con)o(text)f(of)h Fp(comm)f FE(w)o(as)g(pre-allo)q(cated)j(b)o(y)e(the)g(user)g(calling) 75 1378 y Fo(MPI)p 160 1378 V 16 w(CONTEXTS)p 414 1378 V 17 w(ALLOC)g FE(then)g(it)h(is)f(returned)g(to)f(the)h(con)o(text)g (store)f(of)g(the)h(comm)o(unicator)g(from)75 1434 y(whic)o(h)20 b(it)f(w)o(as)f(tak)o(en.)30 b(Otherwise)19 b(the)g(con)o(text)f(is)i (returned)f(to)f(the)h(system.)30 b(It)19 b(is)g(erroneous)g(to)75 1491 y(free)i(a)g(comm)o(unicator)g(if)h(con)o(texts)e(from)h(the)g (con)o(text)f(sore)h(of)g(the)g(comm)o(unicator)g(are)g(in)h(use.)75 1547 y(If)e Fp(comm)f FE(w)o(as)f(created)i(b)o(y)f(calling)j(one)d(of) g Fo(MPI)p 939 1547 V 16 w(COMM)p 1097 1547 V 17 w(MAKE)p FE(,)g Fo(MPI)p 1357 1547 V 15 w(COMM)p 1514 1547 V 17 w(LOCAL)p 1673 1547 V 16 w(MAKE)g FE(or)75 1604 y Fo(MPI)p 160 1604 V 16 w(COMM)p 318 1604 V 16 w(COLL)p 446 1604 V 16 w(MAKE)13 b FE(then)g(the)h(underlying)h(group)d(is)i(not)f (destro)o(y)o(ed.)18 b(Otherwise)c(the)f(group)75 1660 y(is)j(destro)o(y)o(ed.)166 1799 y Fv(Discussion:)f Fu(Do)f(w)o(e)g(w)o (an)o(t)f(a)h(collectiv)o(e)g(v)o(ersion)g(of)f(this)h(function?)20 b FE(.)75 2026 y Fy(3.7)59 b(Inter-Communication)17 b(Communicato)n(r)i (Management)75 2127 y FE(This)f(section)h(in)o(tro)q(duces)f(the)g (concept)g(of)f(in)o(ter-comm)o(unication)i(and)f(describ)q(es)i(the)d (p)q(ortions)h(of)75 2184 y(MPI)h(that)f(supp)q(ort)h(it.)32 b(It)19 b(describ)q(es)i(supp)q(ort)e(for)f(writing)h(programs)f(whic)o (h)i(con)o(tain)f(user-lev)o(el)75 2240 y(serv)o(ers.)k(It)16 b(also)h(describ)q(es)h(a)e(name)g(service)h(whic)o(h)h(simpli\014es)g (writing)f(programs)e(con)o(taining)j(in)o(t-)75 2297 y(er-comm)o(unication.)75 2419 y Fq(3.7.1)49 b(Discussion)75 2505 y Fo(De\014nitions)17 b(of)e(Inter-Communication)f(and)i (Inter-Communicato)o(rs)75 2591 y FE(All)h(p)q(oin)o(t-to-p)q(oin)o(t)g (comm)o(unication)g(describ)q(ed)h(th)o(us)e(far)g(has)g(in)o(v)o(olv)o (ed)h(comm)o(unication)g(b)q(et)o(w)o(een)75 2647 y(pro)q(cesses)12 b(that)e(are)h(mem)o(b)q(ers)h(of)f(the)g(same)g(group.)18 b(The)12 b(source)f(pro)q(cess)h(in)g(send)g(or)f(the)g(destination)75 2704 y(pro)q(cess)j(in)h(receiv)o(e)f(\(the)g(\\target")e(pro)q(cess\)) i(is)g(sp)q(eci\014ed)i(using)e(a)g Fo(\(communicato)o(r,)d(rank\))j FE(pair.)20 b(The)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 28 22 27 bop 75 -100 a FE(22)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y FE(target)f(pro)q(cess)i(is)g(that)f(pro)q(cess)h(with)g(the)g (giv)o(en)g(rank)f(within)i(the)f(group)f(of)g(the)h(giv)o(en)g(comm)o (uni-)75 102 y(cator.)j(This)c(t)o(yp)q(e)g(of)f(comm)o(unication)i(is) f(called)i(\\in)o(tra-comm)o(unication")d(and)h(the)g(comm)o(unicator) 75 158 y(used)h(is)f(called)i(an)e(\\in)o(tra-comm)o(unicator.")166 219 y(In)20 b(mo)q(dular)g(and)g(m)o(ulti-discipli)q(nary)i (applications,)f(di\013eren)o(t)f(pro)q(cess)g(groups)f(execute)h(dif-) 75 275 y(feren)o(t)f(mo)q(dules)h(and)f(pro)q(cesses)g(within)h (di\013eren)o(t)f(mo)q(dules)h(comm)o(unicate)f(with)h(one)f(another)f (in)75 331 y(a)h(pip)q(eline)k(or)c(a)g(more)g(general)h(mo)q(dule)h (graph.)32 b(In)20 b(these)g(applications)h(the)f(most)e(natural)i(w)o (a)o(y)75 388 y(for)f(a)h(pro)q(cess)g(to)f(sp)q(ecify)i(a)f(target)e (pro)q(cess)i(is)h(b)o(y)e(the)h(rank)g(of)f(the)h(target)f(pro)q(cess) h(within)h(the)75 444 y(target)c(group.)27 b(In)19 b(applications)g (that)f(con)o(tain)g(in)o(ternal)g(user)h(lev)o(el)g(serv)o(ers,)f(eac) o(h)g(serv)o(er)f(ma)o(y)g(b)q(e)75 501 y(a)i(pro)q(cess)h(group)g (that)f(pro)o(vides)h(services)g(to)f(one)h(or)f(more)h(clien)o(ts,)h (and)f(eac)o(h)g(clien)o(t)h(ma)o(y)e(b)q(e)h(a)75 557 y(pro)q(cess)f(group)g(whic)o(h)h(uses)g(the)f(services)h(of)f(one)g (or)g(more)g(serv)o(ers.)31 b(In)20 b(these)f(applications)i(it)e(is)75 614 y(again)13 b(most)g(natural)g(to)g(sp)q(ecify)h(the)g(target)e(pro) q(cess)h(b)o(y)g(rank)g(within)i(the)e(target)f(group.)19 b(This)14 b(t)o(yp)q(e)75 670 y(of)h(comm)o(unication)g(is)h(called)g (\\in)o(ter-comm)o(unication")g(and)f(the)g(comm)o(unicator)f(used)i (is)f(called)i(an)75 727 y(\\in)o(ter-comm)o(unicator.")166 787 y(An)23 b(in)o(ter-comm)o(unication)g(op)q(eration)g(is)f(a)g(p)q (oin)o(t-to-p)q(oin)o(t)h(comm)o(unication)g(b)q(et)o(w)o(een)g(pro-)75 844 y(cesses)16 b(in)h(di\013eren)o(t)f(groups.)21 b(The)16 b(group)f(con)o(taining)h(a)g(pro)q(cess)g(that)f(initiates)i(an)e(in)o (ter-comm)o(un-)75 900 y(ication)i(op)q(eration)f(is)g(called)h(the)f (\\lo)q(cal)h(group,")e(that)g(is,)h(the)g(sender)h(in)f(a)g(send)g (and)g(the)g(receiv)o(er)75 957 y(in)g(a)e(receiv)o(e.)21 b(The)15 b(group)g(con)o(taining)g(the)g(target)f(pro)q(cess)h(is)g (called)i(the)e(\\remote)f(group,")g(that)g(is,)75 1013 y(the)j(receiv)o(er)i(in)f(a)f(send)h(and)f(the)h(sender)g(in)g(a)f (receiv)o(e.)27 b(As)17 b(in)i(in)o(tra-comm)o(unication,)f(the)f (target)75 1070 y(pro)q(cess)12 b(is)g(sp)q(eci\014ed)i(using)e(a)f Fo(\(communicato)o(r,)f(rank\))i FE(pair.)19 b(Unlik)o(e)13 b(in)o(tra-comm)o(unication,)g(the)e(rank)75 1126 y(is)16 b(relativ)o(e)f(to)g(the)g(remote)g(group.)166 1187 y(One)k(additional) i(needed)f(concept)f(is)g(the)g(\\group)f(leader.")31 b(The)19 b(pro)q(cess)g(with)g(rank)f(0)h(in)g(a)75 1243 y(pro)q(cess)f(group)f(is)i(designated)f(\\group)f(leader.")28 b(This)19 b(concept)f(is)g(used)h(in)f(supp)q(ort)g(of)f(user-lev)o(el) 75 1299 y(serv)o(ers,)d(and)i(elsewhere.)75 1443 y Fo(Prop)q(erties)g (of)f(Inter-Communication)g(and)g(Inter-Communicato)o(rs)75 1537 y FE(Here)g(is)h(a)f(summary)g(of)f(the)i(prop)q(erties)g(of)e(in) o(ter-comm)o(unication)j(and)e(in)o(ter-comm)o(unicators:)143 1647 y FA(\017)23 b FE(The)16 b(syn)o(tax)f(is)i(the)f(same)g(for)f(b)q (oth)h(in)o(ter-)h(and)f(in)o(tra-comm)o(unication.)23 b(The)16 b(same)g(comm)o(u-)189 1703 y(nicator)f(can)g(b)q(e)h(used)g (for)e(b)q(oth)i(send)f(and)h(receiv)o(e)g(op)q(erations.)143 1814 y FA(\017)23 b FE(A)15 b(target)f(pro)q(cess)h(is)h(addressed)g(b) o(y)f(its)g(rank)g(in)h(the)f(remote)g(group.)143 1924 y FA(\017)23 b FE(Comm)o(unications)12 b(using)i(an)e(in)o(ter-comm)o (unicator)h(are)f(guaran)o(teed)g(not)g(to)g(con\015ict)h(with)g(an)o (y)189 1980 y(comm)o(unications)i(that)g(use)h(a)e(di\013eren)o(t)i (comm)o(unicator.)143 2091 y FA(\017)23 b FE(An)15 b(in)o(ter-comm)o (unicator)g(cannot)g(b)q(e)h(used)g(for)f(collectiv)o(e)i(comm)o (unication.)143 2201 y FA(\017)23 b FE(A)15 b(comm)o(unicator)g(will)i (pro)o(vide)e(either)h(in)o(tra-)f(or)g(in)o(ter-comm)o(unication,)h (nev)o(er)f(b)q(oth.)143 2311 y FA(\017)23 b FE(Once)18 b(constructed,)f(the)g(remote)f(group)h(of)g(an)f(in)o(ter-comm)o (unicator)i(ma)o(y)e(not)g(b)q(e)i(c)o(hanged.)189 2368 y(Comm)o(unication)d(with)h(an)o(y)f(pro)q(cess)g(outside)h(of)f(the)g (remote)g(group)f(is)i(not)f(allo)o(w)o(ed.)166 2478 y(The)k(routine)g Fo(MPI)p 508 2478 14 2 v 16 w(COMM)p 666 2478 V 16 w(ST)l(A)l(T\(\))g FE(ma)o(y)f(b)q(e)h(used)g(to)f (determine)h(if)g(a)g(comm)o(unicator)f(is)h(an)75 2534 y(in)o(ter-)g(or)e(in)o(tra-comm)o(unicator.)29 b(In)o(ter-comm)o (unicators)18 b(can)g(b)q(e)h(used)g(as)f(argumen)o(ts)f(to)g(some)h (of)75 2591 y(the)f(other)f(comm)o(unicator)g(inquiry)i(routines)f (\(de\014ned)g(ab)q(o)o(v)o(e\).)24 b(In)o(ter-comm)o(unicators)16 b(cannot)g(b)q(e)75 2647 y(used)c(as)g(input)g(to)f(an)o(y)h(of)f(the)h (lo)q(cal)g(constructor)f(routines)h(for)f(in)o(tra-comm)o(unicators)h (or)f(the)h(con)o(text)75 2704 y(managemen)o(t)i(routines.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 29 23 28 bop 75 -100 a Fx(3.7.)34 b(INTER-COMMUNICA)l(TION)17 b(COMMUNICA)l(TOR)f(MANA)o(GEMENT)287 b FE(23)75 45 y Fo(Intercommunicato)o(r)14 b(Constructo)o(rs)i(and)g(Destructo)o(rs)75 133 y FE(Construction)h(of)f(an)g(in)o(ter-comm)o(unicator)h(requires)g (t)o(w)o(o)f(separate)g(collectiv)o(e)i(op)q(erations)f(\(one)f(in)75 189 y(the)11 b(lo)q(cal)h(group)f(and)g(one)g(in)h(the)f(remote)g (group\))f(and)h(a)g(p)q(oin)o(t-to-p)q(oin)o(t)h(op)q(eration)f(b)q (et)o(w)o(een)g(the)g(t)o(w)o(o)75 246 y(group)i(leaders.)20 b(These)14 b(op)q(erations)g(ma)o(y)e(b)q(e)i(p)q(erformed)g(with)g (explicit)h(sync)o(hronization)g(of)e(the)g(t)o(w)o(o)75 302 y(groups)e(b)o(y)h(calling)h Fo(MPI)p 503 302 14 2 v 16 w(COMM)p 661 302 V 17 w(PEER)p 790 302 V 17 w(MAKE\(\))p FE(.)k(The)12 b(explicit)i(sync)o(hronization)f(can)e(cause)h(dead-)75 359 y(lo)q(c)o(k)g(in)h(mo)q(dular)f(programs)f(with)h(cyclic)h(comm)o (unication)g(graphs.)18 b(So,)12 b(the)g(lo)q(cal)h(and)e(remote)h(op)q (er-)75 415 y(ations)i(can)f(b)q(e)i(decoupled)g(and)f(the)g (construction)f(p)q(erformed)h(\\lo)q(osely)h(sync)o(hronously")f(b)o (y)f(calling)75 471 y(the)d(t)o(w)o(o)f(routines)i Fo(MPI)p 484 471 V 15 w(COMM)p 641 471 V 17 w(PEER)p 770 471 V 17 w(MAKE)p 916 471 V 16 w(ST)l(ART\(\))f FE(and)h Fo(MPI)p 1286 471 V 16 w(COMM)p 1444 471 V 16 w(PEER)p 1572 471 V 17 w(MAKE)p 1718 471 V 16 w(FINISH\(\))p FE(.)166 611 y Fv(Discussion:)67 b Fr(MPI)p 544 611 13 2 v 14 w(COMM)p 688 611 V 15 w(PEER)p 807 611 V 14 w(MAKE)p 939 611 V 14 w(ST)m(ART\(\))16 b Fu(and)33 b Fr(MPI)p 1307 611 V 14 w(COMM)p 1451 611 V 15 w(PEER)p 1570 611 V 14 w(MAKE)p 1702 611 V 14 w(FINISH\(\))75 668 y Fu(are)16 b(b)q(oth)f(collectiv)o (e)g(op)q(erations)g(in)g(the)h(lo)q(cal)e(group.)22 b(They)15 b(ma)o(y)e(lea)o(v)o(e)i(a)g(non-blo)q(c)o(king)f(send)i(and) f(receiv)o(e)75 724 y(activ)o(e)k(b)q(et)o(w)o(een)h(the)f(t)o(w)o(o)g (calls,)g(where)h(the)f(group)g(leaders)g(exc)o(hange)h(lo)q(cal)e (comm)o(unicator)e(information)75 781 y(as)i(necessary)m(.)32 b(Ho)o(w)o(ev)o(er,)19 b(they)f(are)h(not)f(a)f(non-blo)q(c)o(king)g (collectiv)o(e)h(op)q(eration.)64 b FE(These)20 b(routines)g(can)75 920 y(construct)d(m)o(ultiple)j(in)o(ter-comm)o(unicators)d(with)h(a)f (single)i(call.)28 b(This)18 b(impro)o(v)o(es)f(p)q(erformance)h(b)o(y) 75 976 y(allo)o(wing)e(amortization)f(of)g(the)g(sync)o(hronization)h (o)o(v)o(erhead.)166 1034 y(The)e(in)o(ter-comm)o(unicator)h(ob)s (jects)f(are)g(destro)o(y)o(ed)f(in)j(the)e(same)g(w)o(a)o(y)f(as)h(in) o(tra-comm)o(unicator)75 1090 y(ob)s(jects,)g(b)o(y)h(calling)i Fo(MPI)p 535 1090 14 2 v 16 w(COMM)p 693 1090 V 16 w(FREE\(\))p FE(.)75 1216 y Fo(Supp)q(o)o(rt)g(fo)o(r)d(User-Level)h(Servers)75 1303 y FE(W)l(e)g(consider)g(the)f(primary)h(feature)f(of)g(user-lev)o (el)i(serv)o(ers)e(that)g(can)h(require)g(additional)h(supp)q(ort)e(is) 75 1360 y(that)c(the)h(serv)o(er)g(cannot)g(a)g(priori)g(kno)o(w)g(the) g(iden)o(ti\014cation)i(of)d(the)i(clien)o(ts,)g(whereas)f(the)g(clien) o(ts)i(m)o(ust)75 1416 y(a)g(priori)i(kno)o(w)e(the)h(iden)o (ti\014cation)h(of)e(the)h(serv)o(ers.)19 b(In)14 b(addition,)h(a)e (user-lev)o(el)j(serv)o(er)d(is)h(a)g(dedicated)75 1473 y(pro)q(cess)h(group)g(whic)o(h,)h(after)f(some)f(initialization,)k (pro)o(vides)e(a)f(giv)o(en)g(service)i(un)o(til)f(termination.)166 1530 y(The)h(supp)q(ort)f(for)g(user-lev)o(el)i(serv)o(ers)f(tak)o(es)e (in)o(to)i(accoun)o(t)f(the)h(prev)m(ailing)i(view)e(that)f(all)h(pro-) 75 1586 y(cesses)g(\(p)q(ossibly)g(excepting)h(a)e(host)f(pro)q(cess\)) i(are)f(initially)j(equiv)m(alen)o(t)f(mem)o(b)q(ers)e(of)g(the)g (group)g(of)75 1643 y(all)f(pro)q(cesses.)20 b(This)14 b(group)g(is)g(describ)q(ed)i(b)o(y)e(pre-de\014ned)i(in)o(tra-comm)o (unicator)e Fo(MPI)p 1612 1643 V 15 w(COMM)p 1769 1643 V 17 w(ALL)p FE(.)75 1699 y(The)k(user)h(splits)g(this)g(group)f(suc)o (h)g(that)g(pro)q(cesses)g(in)h(eac)o(h)g(parallel)h(serv)o(er)d(are)h (placed)i(within)f(a)75 1756 y(sp)q(eci\014c)e(sub-group.)22 b(The)16 b(non-serv)o(er)f(pro)q(cesses)h(are)g(placed)h(in)f(a)f (group)h(of)f(all)h(non-serv)o(ers.)21 b(Pro-)75 1812 y(vided)14 b(that)e(the)h(user)g(can)g(determine)h(the)f(ranks)g(of)f (the)h(serv)o(er)g(group)f(leaders)i(\()p Ft(i.e.,)f FE(rank)f(zero\))h(and)75 1869 y(assign)h(some)f(tags)g(for)g(clien)o (ts)i(to)e(send)h(a)f(message)g(to)g(the)h(group)f(leaders,)i(then)f(a) f(group)g(leader)i(can)75 1925 y(at)g(an)o(y)f(time)i(notify)f(a)g (serv)o(er)g(that)f(it)i(wishes)g(to)e(b)q(ecome)i(a)f(clien)o(t.)166 1983 y(MPI)22 b(pro)o(vides)h(a)f(routine,)i Fo(MPI)p 772 1983 V 15 w(COMM)p 929 1983 V 17 w(SPLITL\(\))p FE(,)e(that)f (splits)i(a)f(paren)o(t)g(group,)h(creates)75 2039 y(sub-groups)14 b(\(in)o(tra-comm)o(unicators\))g(according)g(to)g(supplied)i(k)o(eys,) e(and)h(returns)f(the)g(rank)g(of)g(eac)o(h)75 2095 y(sub-group)i (leader)h(\(relativ)o(e)f(to)g(the)g(paren)o(t)f(group\).)22 b(This)16 b(allo)o(ws)g(a)g(pro)q(cess)g(that)g(do)q(es)g(not)f(kno)o (w)75 2152 y(ab)q(out)k(a)g(sub-group)h(to)f(con)o(tact)f(that)h (sub-group)h(via)g(the)f(sub-group)h(leader,)h(using)f(the)f(paren)o(t) 75 2208 y(comm)o(unicator.)k(The)16 b(k)o(eys)h(ma)o(y)e(b)q(e)i(used)g (as)f(unique)i(tags.)k(This)17 b(information)f(ma)o(y)g(also)g(b)q(e)h (used)75 2265 y(as)e(input)h(to)f Fo(MPI)p 393 2265 V 15 w(COMM)p 550 2265 V 17 w(PEER)p 679 2265 V 17 w(MAKE\(\))p FE(,)f(for)g(example.)75 2390 y Fo(Name)h(Service)75 2478 y FE(MPI)f(pro)o(vides)h(a)g(name)f(service)h(to)f(simplify)j (construction)d(of)g(in)o(ter-comm)o(unicators.)20 b(This)15 b(service)75 2534 y(allo)o(ws)f(a)g(lo)q(cal)h(pro)q(cess)f(group)g(to) f(create)h(an)g(in)o(ter-comm)o(unicator)g(when)g(the)g(only)h(a)o(v)m (ailable)h(infor-)75 2591 y(mation)11 b(ab)q(out)h(the)g(remote)f (group)g(is)h(a)f(user-de\014ned)i(c)o(haracter)e(string.)19 b(A)12 b(sync)o(hronizing)h(v)o(ersion)e(is)75 2647 y(pro)o(vided)h(b)o (y)f(routine)g Fo(MPI)p 554 2647 V 16 w(COMM)p 712 2647 V 17 w(NAME)p 858 2647 V 16 w(MAKE\(\))p FE(.)18 b(A)11 b(lo)q(osely)h(sync)o(hronous)f(v)o(ersion)h(is)f(pro)o(vided)75 2704 y(b)o(y)j(routines)h Fo(MPI)p 396 2704 V 16 w(COMM)p 554 2704 V 17 w(NAME)p 700 2704 V 16 w(MAKE)p 845 2704 V 17 w(ST)l(ART\(\))f FE(and)h Fo(MPI)p 1224 2704 V 16 w(COMM)p 1382 2704 V 16 w(NAME)p 1527 2704 V 17 w(MAKE)p 1673 2704 V 16 w(FINISH\(\))p FE(.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 30 24 29 bop 75 -100 a FE(24)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fq(3.7.2)49 b(Communicato)o(r)16 b(Accesso)o(rs)75 178 y Fo(MPI)p 160 178 14 2 v 16 w(COMM)p 318 178 V 16 w(ST)l(A)l(T\(comm,)e(status\))117 256 y Fu(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)117 331 y(OUT)124 b Fo(status)460 b Fu(in)o(teger)15 b(status)166 455 y FE(This)e(lo)q(cal) h(routine)f(allo)o(ws)g(the)g(calling)h(pro)q(cess)f(to)f(determine)i (if)f(a)f(comm)o(unicator)h(is)g(an)g(in)o(ter-)75 512 y(comm)o(unicator)j(or)f(an)h(in)o(tra-comm)o(unicator.)22 b(This)17 b(returns)f(the)g(status)f(of)h(comm)o(unicator)f Fo(comm)p FE(.)75 568 y(V)l(alid)i(status)d(v)m(alues)i(are)29 b Fr(MPI)p 630 568 13 2 v 14 w(INTRA)p FE(,)g Fr(MPI)p 885 568 V 14 w(INTER)p FE(,)f Fr(MPI)p 1136 568 V 15 w(INV)m(ALID)p FE(.)166 625 y(When)18 b(an)f(in)o(ter-comm)o(unicator)h (is)g(used)g(as)f(an)g(input)i(argumen)o(t)d(to)h(the)h(comm)o (unicator)f(ac-)75 681 y(cessors)e(describ)q(ed)i(under)f(in)o (tra-comm)o(unication,)f(the)h(follo)o(wing)g(table)f(describ)q(es)i(b) q(eha)o(vior.)p 210 746 1531 2 v 209 803 2 57 v 664 786 a Fo(MPI)p 749 786 14 2 v 16 w(COMM)p 907 786 V 31 w FE(F)l(unction)f(Beha)o(vior)p 1739 803 2 57 v 209 859 V 655 842 a(\(in)g(In)o(ter-Comm)o(unication)f(Mo)q(de\))p 1739 859 V 210 861 1531 2 v 210 871 V 209 927 2 57 v 235 910 a Fo(MPI)p 320 910 14 2 v 15 w(COMM)p 477 910 V 17 w(SIZE\(\))p 790 927 2 57 v 195 w FE(returns)g(the)h(size)g(of)f (the)g(remote)f(group.)p 1739 927 V 209 984 V 235 967 a Fo(MPI)p 320 967 14 2 v 15 w(COMM)p 477 967 V 17 w(GROUP\(\))p 790 984 2 57 v 136 w FE(returns)h(the)h(remote)e(group.)p 1739 984 V 209 1040 V 235 1023 a Fo(MPI)p 320 1023 14 2 v 15 w(COMM)p 477 1023 V 17 w(RANK\(\))p 790 1040 2 57 v 165 w FE(returns)h Fr(MPI)p 1049 1023 13 2 v 15 w(UNDEFINED)p 1739 1040 2 57 v 209 1097 V 235 1080 a Fo(MPI)p 320 1080 14 2 v 15 w(COMM)p 477 1080 V 17 w(CONTEXTS\(\))p 790 1097 2 57 v 50 w FE(erroneous)p 1739 1097 V 210 1098 1531 2 v 166 1264 a Fv(Discussion:)g Fu(Should)f(there)h(b)q(e)f(an)o (y)g(other)g(status)h(v)n(alues?)166 1486 y Fv(Discussion:)g Fu(Do)q(es)g(this)f(routine)g(really)f(b)q(elong)h(under)g(in)o (ter-comm)o(unication?)75 1691 y Fq(3.7.3)49 b(Communicato)o(r)16 b(Constructo)o(rs)75 1824 y Fo(MPI)p 160 1824 14 2 v 16 w(COMM)p 318 1824 V 16 w(PEER)p 446 1824 V 17 w(MAKE\(my)p 668 1824 V 15 w(comm,)25 b(my)p 896 1824 V 15 w(leader,)h(p)q(eer)p 1142 1824 V 17 w(comm,)f(p)q(eer)p 1394 1824 V 17 w(rank,)h(tag,)g(num) p 1716 1824 V 17 w(comms,)75 1881 y(new)p 152 1881 V 17 w(comms\))117 1958 y Fu(IN)171 b Fo(my)p 395 1958 V 15 w(comm)381 b Fu(lo)q(cal)13 b(in)o(tra-comm)o(unicator)117 2033 y(IN)171 b Fo(my)p 395 2033 V 15 w(leader)386 b Fu(rank)14 b(of)f(lo)q(cal)g(group)h(leader)g(in)g Fr(my)p 1490 2033 13 2 v 14 w(comm)117 2108 y Fu(IN)171 b Fo(p)q(eer)p 417 2108 14 2 v 17 w(comm)357 b Fu(\\paren)o(t")14 b(in)o(tra-comm)o (uni)o(cator)117 2184 y(IN)171 b Fo(p)q(eer)p 417 2184 V 17 w(rank)391 b Fu(rank)14 b(of)f(remote)h(group)g(leader)g(in)27 b Fr(p)q(eer)p 1563 2184 13 2 v 17 w(comm)117 2259 y Fu(IN)171 b Fo(tag)510 b Fu(\\safe")14 b(tag)117 2334 y(IN)171 b Fo(num)p 420 2334 14 2 v 16 w(comms)338 b Fu(n)o(um)o(b)q(er)13 b(of)h(new)g(in)o(ter-comm)o(unicators)e(to)h (construct)117 2410 y(OUT)124 b Fo(new)p 411 2410 V 17 w(comms)346 b Fu(arra)o(y)14 b(of)f(new)h(in)o(ter-comm)o(unicators)166 2534 y FE(This)k(routine)f(constructs)g(an)g(arra)o(y)f(of)h(in)o (ter-comm)o(unicators)g(and)h(stores)e(it)i(in)g Fo(new)p 1716 2534 V 17 w(comms)p FE(.)75 2591 y(In)o(tra-comm)o(unicator)h Fo(my)p 551 2591 V 15 w(comm)f FE(describ)q(es)i(the)g(lo)q(cal)g (group.)32 b(In)o(tra-comm)o(unicator)19 b Fo(p)q(eer)p 1746 2591 V 17 w(comm)75 2647 y FE(describ)q(es)c(a)f(group)f(that)g (con)o(tains)h(the)g(leaders)h(\()p Ft(i.e.,)e FE(mem)o(b)q(ers)h(with) g(rank)f(zero\))h(of)f(b)q(oth)h(the)g(lo)q(cal)75 2704 y(and)f(remote)e(groups.)19 b(In)o(teger)12 b Fo(p)q(eer)p 707 2704 V 17 w(rank)h FE(is)g(the)f(rank)g(of)g(the)h(remote)f(leader) h(in)g Fo(p)q(eer)p 1575 2704 V 17 w(comm)p FE(.)k(In)o(teger)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 31 25 30 bop 75 -100 a Fx(3.7.)34 b(INTER-COMMUNICA)l(TION)17 b(COMMUNICA)l(TOR)f(MANA)o(GEMENT)287 b FE(25)75 45 y(tag)11 b(is)h(used)g(to)f(distinguish)i(this)f(op)q(eration)g(from)f(others)g (with)g(the)h(same)f(p)q(eer.)20 b(In)o(teger)11 b Fo(num)p 1729 45 14 2 v 16 w(comms)75 102 y FE(is)h(the)f(n)o(um)o(b)q(er)h(of)f (new)h(in)o(ter-comm)o(unicators)f(constructed.)19 b(This)12 b(routine)g(is)g(collectiv)o(e)h(in)f(the)g(lo)q(cal)75 158 y(group)i(and)h(sync)o(hronizes)g(with)g(the)f(remote)g(group.)19 b(Eac)o(h)14 b(of)g(the)g(in)o(ter-comm)o(unicators)h(pro)q(duced)75 214 y(pro)o(videsin)o(ter-comm)o(unication)i(with)e(the)h(remote)e (group.)75 320 y Fo(MPI)p 160 320 V 16 w(COMM)p 318 320 V 16 w(PEER)p 446 320 V 17 w(MAKE)p 592 320 V 16 w(ST)l(ART\(my)p 826 320 V 17 w(comm,)8 b(my)p 1039 320 V 16 w(leader,)j(p)q(eer)p 1271 320 V 17 w(comm,)e(p)q(eer)p 1507 320 V 17 w(rank,)i(tag,)f(num)p 1798 320 V 17 w(comms,)75 376 y(mak)o(e)p 178 376 V 15 w(id\))117 455 y Fu(IN)171 b Fo(my)p 395 455 V 15 w(comm)381 b Fu(lo)q(cal)13 b(in)o(tra-comm)o(unicator)117 534 y(IN)171 b Fo(my)p 395 534 V 15 w(leader)386 b Fu(rank)14 b(of)f(lo)q(cal)g (group)h(leader)g(in)g Fr(my)p 1490 534 13 2 v 14 w(comm)117 613 y Fu(IN)171 b Fo(p)q(eer)p 417 613 14 2 v 17 w(comm)357 b Fu(\\paren)o(t")14 b(in)o(tra-comm)o(uni)o(cator)117 692 y(IN)171 b Fo(p)q(eer)p 417 692 V 17 w(rank)391 b Fu(rank)14 b(of)f(remote)h(group)g(leader)g(in)27 b Fr(p)q(eer)p 1563 692 13 2 v 17 w(comm)117 771 y Fu(IN)171 b Fo(tag)510 b Fu(\\safe")14 b(tag)117 849 y(IN)171 b Fo(num)p 420 849 14 2 v 16 w(comms)338 b Fu(n)o(um)o(b)q(er)13 b(of)h(new)g(in)o (ter-comm)o(unicators)e(to)h(construct)117 928 y(OUT)124 b Fo(mak)o(e)p 437 928 V 15 w(id)422 b Fu(handle)14 b(for)27 b Fr(MPI)p 1193 928 13 2 v 15 w(COMM)p 1338 928 V 14 w(PEER)p 1456 928 V 14 w(MAKE)p 1588 928 V 15 w(FINISH\(\))166 1054 y FE(This)d(starts)e(o\013)g(a)p 533 1054 14 2 v 39 w Fo(PEER)p 662 1054 V 17 w(MAKE)h FE(op)q(eration,)i(returning)f(a) f(handle)h(for)f(the)g(op)q(eration)g(in)75 1111 y(\\)p Fo(mak)o(e)p 201 1111 V 15 w(id)p FE(.")42 b(It)23 b(is)g(collectiv)o (e)i(in)e Fo(my)p 761 1111 V 16 w(comm)p FE(.)40 b(It)22 b(do)q(es)h(not)g(w)o(ait)f(for)g(the)h(remote)f(group)g(to)g(do)75 1167 y Fo(MPI)p 160 1167 V 16 w(COMM)p 318 1167 V 16 w(MAKE)p 463 1167 V 16 w(ST)l(ART\(\))p FE(.)34 b(The)20 b(\\)p Fo(mak)o(e)p 925 1167 V 15 w(id)p FE(")f(handle)i(is)f (conceptually)i(similar)e(to)f(the)h(com-)75 1224 y(m)o(unication)d (handle)g(used)f(b)o(y)g(non-blo)q(c)o(king)h(p)q(oin)o(t-to-p)q(oin)o (t)f(routines.)22 b(A)16 b Fo(mak)o(e)p 1536 1224 V 15 w(id)g FE(handle)h(is)g(con-)75 1280 y(structed)f(b)o(y)f(a)g(\\)p 380 1280 V 16 w Fo(ST)l(ART)p FE(")i(routine)f(and)f(destro)o(y)o(ed)g (b)o(y)h(the)f(matc)o(hing)h(\\)p 1391 1280 V 16 w Fo(FINISH)p FE(")f(routine.)22 b(These)75 1337 y(handles)c(are)e(not)g(v)m(alid)j (for)c(an)o(y)i(other)f(use.)24 b(It)17 b(is)g(erroneous)g(to)e(call)j (this)f(routine)g(again)g(with)g(the)75 1393 y(same)f Fo(p)q(eer)p 273 1393 V 17 w(comm)p FE(,)d Fo(p)q(eer)p 513 1393 V 17 w(rank)j FE(and)g Fo(tag)p FE(,)g(without)g(calling)h Fo(MPI)p 1204 1393 V 16 w(COMM)p 1362 1393 V 16 w(MAKE)p 1507 1393 V 17 w(FINISH\(\))d FE(to)i(\014nish)75 1450 y(the)f(\014rst)g(call.)75 1555 y Fo(MPI)p 160 1555 V 16 w(COMM)p 318 1555 V 16 w(PEER)p 446 1555 V 17 w(MAKE)p 592 1555 V 16 w(FINISH\(mak)o(e)p 867 1555 V 15 w(id,)g(new)p 1018 1555 V 18 w(comms\))117 1634 y Fu(IN)171 b Fo(mak)o(e)p 437 1634 V 15 w(id)422 b Fu(handle)14 b(from)26 b Fr(MPI)p 1228 1634 13 2 v 14 w(COMM)p 1372 1634 V 15 w(PEER)p 1491 1634 V 14 w(MAKE)p 1623 1634 V 14 w(ST)m(ART\(\))117 1713 y Fu(OUT)124 b Fo(new)p 411 1713 14 2 v 17 w(comms)346 b Fu(arra)o(y)14 b(of)f(new)h(in)o(ter-comm)o(unicators)166 1839 y FE(This)f(completes)g(a)p 512 1839 V 29 w Fo(PEER)p 641 1839 V 17 w(MAKE)f FE(op)q(eration,)h(returning)g(an)f(arra)o(y)f (of)h Fo(num)p 1518 1839 V 17 w(comms)f FE(new)i(in)o(ter-)75 1896 y(comm)o(unicators)k(in)h Fo(new)p 524 1896 V 17 w(comms)p FE(.)24 b(\(Note)17 b(that)f Fo(num)p 1025 1896 V 16 w(comms)g FE(w)o(as)g(sp)q(eci\014ed)k(in)e(the)f(corresp)q (onding)75 1952 y(call)i(to)d Fo(MPI)p 303 1952 V 16 w(COMM)p 461 1952 V 17 w(PEER)p 590 1952 V 17 w(MAKE)p 736 1952 V 16 w(ST)l(ART\(\))p FE(.\))26 b(This)18 b(routine)g(is)g (collectiv)o(e)i(in)e(the)f Fo(my)p 1693 1952 V 16 w(comm)e FE(of)75 2009 y(the)d(corresp)q(onding)h(call)g(to)e Fo(MPI)p 656 2009 V 16 w(COMM)p 814 2009 V 16 w(PEER)p 942 2009 V 17 w(MAKE)p 1088 2009 V 16 w(ST)l(ART\(\))p FE(.)19 b(It)12 b(w)o(aits)g(for)f(the)h(remote)f(group)75 2065 y(to)16 b(call)i Fo(MPI)p 302 2065 V 16 w(COMM)p 460 2065 V 16 w(PEER)p 588 2065 V 17 w(MAKE)p 734 2065 V 16 w(ST)l(ART\(\))f FE(but)g(do)q(es)g(not)g(w)o(ait)f(for)g(the)h (remote)f(group)h(to)f(call)75 2121 y Fo(MPI)p 160 2121 V 16 w(COMM)p 318 2121 V 16 w(PEER)p 446 2121 V 17 w(MAKE)p 592 2121 V 16 w(FINISH\(\))p FE(.)75 2254 y Fq(3.7.4)49 b(Communicato)o(r)16 b(Destructo)o(rs)75 2343 y FE(In)o(ter-comm)o (unicator)i(ob)s(jects)g(are)g(destro)o(y)o(ed)f(in)i(the)g(same)f (fashion)g(as)g(in)o(tra-comm)o(uniation)h(ob-)75 2399 y(jects,)c(using)h Fo(MPI)p 397 2399 V 15 w(COMM)p 554 2399 V 17 w(FREE)g FE(describ)q(ed)h(ab)q(o)o(v)o(e.)75 2532 y Fq(3.7.5)49 b(Supp)q(o)o(rt)16 b(fo)o(r)g(User-Level)g(Servers) 75 2704 y Fv(Discussion:)f Fu(Do)q(es)g(this)f(routine)g(really)f(b)q (elong)h(under)g(in)o(ter-comm)o(unication?)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 32 26 31 bop 75 -100 a FE(26)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)166 45 y FE(This)i(collectiv)o(e)i(routine)e(is)h(used)f(to)f(mak)o(e)h(it) g(easier)g(for)f(pro)q(cesses)h(to)g(form)f(sub-groups)h(and)75 102 y(con)o(tact)d(user-lev)o(el)j(serv)o(ers.)75 205 y Fo(MPI)p 160 205 14 2 v 16 w(COMM)p 318 205 V 16 w(SPLITL\(comm,)12 b(k)o(ey)l(,)j(nk)o(eys,)h(leaders,)f(sub)p 1079 205 V 18 w(comm\))117 282 y Fu(IN)171 b Fo(comm)454 b Fu(extan)o(t)14 b(in)o(tra-comm)o(unicator)d(to)j(b)q(e)g(\\split")117 354 y(IN)171 b Fo(k)o(ey)509 b Fu(k)o(ey)14 b(for)g(sub-group)g(mem)o (b)q(ership)117 425 y(IN)171 b Fo(nk)o(eys)469 b Fu(n)o(um)o(b)q(er)13 b(of)h(k)o(eys)g(\(n)o(um)o(b)q(er)f(of)h(sub-groups\))117 496 y(OUT)124 b Fo(leaders)442 b Fu(ranks)14 b(of)g(sub-group)g (leaders)g(in)g(comm)117 567 y(OUT)124 b Fo(sub)p 400 567 V 17 w(comm)374 b Fu(in)o(tra-comm)o(unicator)6 b(describing)k (sub-group)f(of)g(calling)f(pro-)905 623 y(cess)166 748 y FE(This)17 b(routine)g(splits)h(the)f(group)f(describ)q(ed)j(b)o(y)d (in)o(tra-comm)o(unicator)h(comm)f(in)o(to)h(nk)o(eys)f(sub-)75 804 y(groups.)21 b(Eac)o(h)15 b(calling)j(pro)q(cess)e(m)o(ust)f(sp)q (ecify)i(a)e(v)m(alue)i(of)e(k)o(ey)h(in)g(the)g(range)f([0)p FD(:)8 b(:)g(:)d FE(\()p Fo(nk)o(eys)p FE(-1\)].)21 b(Pro-)75 861 y(cesses)g(sp)q(ecifying)i(the)d(same)h(k)o(ey)f(are)g(placed)i(in) g(the)f(same)f(sub-group.)36 b(Ranks)21 b(of)g(the)f(leaders)75 917 y(of)e(eac)o(h)h(sub-group)g(\(relativ)o(e)g(to)f Fo(comm)p FE(\))f(are)h(returned)h(in)h(in)o(teger)f(arra)o(y)e (leaders.)31 b(This)19 b(routine)75 973 y(returns)f(a)g(new)h(in)o (tra-comm)o(unicator,)f Fo(sub)p 862 973 V 18 w(comm)p FE(,)f(that)h(describ)q(es)i(the)e(sub-group)h(to)f(whic)o(h)h(the)75 1030 y(calling)e(pro)q(cess)e(b)q(elongs.)75 1148 y Fq(3.7.6)49 b(Communicato)o(r)16 b(Constructo)o(rs)e(using)j(Name)f(Service)75 1281 y Fo(MPI)p 160 1281 V 16 w(COMM)p 318 1281 V 16 w(NAME)p 463 1281 V 17 w(MAKE\(my)p 685 1281 V 15 w(comm,)d(name,)h (num)p 1055 1281 V 17 w(comms,)f(new)p 1306 1281 V 17 w(comms\))117 1358 y Fu(IN)171 b Fo(my)p 395 1358 V 15 w(comm)381 b Fu(lo)q(cal)13 b(in)o(tra-comm)o(unicator)117 1429 y(IN)171 b Fo(name)469 b Fu(\\name)10 b(kno)o(wn)h(to)f(b)q(oth)i (lo)q(cal)e(and)h(remote)f(group)h(leaders")117 1500 y(IN)171 b Fo(num)p 420 1500 V 16 w(comms)338 b Fu(n)o(um)o(b)q(er)13 b(of)h(new)g(in)o(ter-comm)o(unicators)e(to)h(construct)117 1571 y(OUT)124 b Fo(new)p 411 1571 V 17 w(comms)346 b Fu(Arra)o(y)14 b(of)f(new)i(in)o(ter-comm)o(unicators)166 1696 y FE(This)21 b(is)h(the)f(name-serv)o(ed)g(equiv)m(alen)o(t)h(of)f Fo(MPI)p 1041 1696 V 15 w(COMM)p 1198 1696 V 17 w(PEER)p 1327 1696 V 17 w(MAKE)g FE(in)g(whic)o(h)h(the)f(caller)75 1752 y(need)16 b(only)h(kno)o(w)e(a)g(name)h(for)f(the)g(p)q(eer)i (connection.)22 b(The)16 b(same)f(name)g(is)i(supplied)h(b)o(y)d(b)q (oth)h(lo)q(cal)75 1809 y(and)k(remote)f(groups.)34 b(The)20 b(name)g(is)g(remo)o(v)o(ed)g(from)f(the)h(in)o(ternal)h(name-serv)o (er)e(database)h(after)75 1865 y(b)q(oth)15 b(groups)g(ha)o(v)o(e)g (completed)h Fo(MPI)p 736 1865 V 16 w(COMM)p 894 1865 V 16 w(NAME)p 1039 1865 V 17 w(MAKE\(\))p FE(.)75 1969 y Fo(MPI)p 160 1969 V 16 w(COMM)p 318 1969 V 16 w(NAME)p 463 1969 V 17 w(MAKE)p 609 1969 V 16 w(ST)l(ART\(my)p 843 1969 V 16 w(comm,)d(name,)i(num)p 1215 1969 V 16 w(comms,)e(mak)o(e)p 1491 1969 V 16 w(id\))117 2046 y Fu(IN)171 b Fo(my)p 395 2046 V 15 w(comm)381 b Fu(lo)q(cal)13 b(in)o(tra-comm)o(unicator)117 2117 y(IN)171 b Fo(name)469 b Fu(\\name")10 b(kno)o(wn)g(to)h(b)q(oth)g(lo)q(cal)g(and)g(remote)f (group)h(leaders)117 2188 y(IN)171 b Fo(num)p 420 2188 V 16 w(comms)338 b Fu(n)o(um)o(b)q(er)13 b(of)h(new)g(in)o(ter-comm)o (unicators)e(to)h(construct)117 2259 y(OUT)124 b Fo(mak)o(e)p 437 2259 V 15 w(id)422 b Fu(handle)14 b(for)g(MPI)p 1186 2259 13 2 v 15 w(COMM)p 1339 2259 V 15 w(NAME)p 1482 2259 V 16 w(MAKE)p 1627 2259 V 15 w(FINISH\(\))75 2431 y Fo(MPI)p 160 2431 14 2 v 16 w(COMM)p 318 2431 V 16 w(NAME)p 463 2431 V 17 w(MAKE)p 609 2431 V 16 w(FINISH\(mak)o(e)p 884 2431 V 15 w(id,)h(new)p 1035 2431 V 17 w(comms\))117 2508 y Fu(IN)171 b Fo(mak)o(e)p 437 2508 V 15 w(id)422 b Fu(handle)14 b(from)e(MPI)p 1220 2508 13 2 v 15 w(COMM)p 1373 2508 V 16 w(NAME)p 1517 2508 V 15 w(MAKE)p 1661 2508 V 16 w(ST)m(AR)m(T\(\))117 2579 y(OUT)124 b Fo(new)p 411 2579 14 2 v 17 w(comms)346 b Fu(arra)o(y)14 b(of)f(new)h(in)o (ter-comm)o(unicators)166 2704 y FE(These)23 b(are)f(the)g Fo(ST)l(ART/FINISH)h FE(v)o(ersions)f(of)g Fo(MPI)p 1125 2704 V 16 w(COMM)p 1283 2704 V 17 w(NAME)p 1429 2704 V 16 w(MAKE\(\))p FE(.)41 b(They)22 b(ha)o(v)o(e)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 33 27 32 bop 75 -100 a Fx(3.8.)34 b(CA)o(CHEING)1399 b FE(27)75 45 y(sync)o(hronization)16 b(prop)q(erties)g(analogous)f(to)f(the)i (corresp)q(onding)p 1254 45 14 2 v 32 w Fo(PEER)p 1382 45 V 32 w FE(routines.)75 172 y Fq(3.7.7)49 b(F)o(urther)14 b(Discussion)75 259 y Fo(Securit)o(y)i(and)g(P)o(erfo)o(rmance)d (Issues)75 347 y FE(The)h(routines)g(in)g(this)g(section)g(do)f(not)g (in)o(tro)q(duce)i(insecurit)o(y)g(in)o(to)e(the)h(basic)g(usage)f(of)g (MPI.)g(Sp)q(ecif-)75 403 y(ically)l(,)k(they)e(do)g(not)g(allo)o(w)g (con)o(texts)g(to)g(b)q(e)g(b)q(ound)i(in)f(m)o(ultiple)h(usable)f (comm)o(unicators.)166 461 y(The)f(pro)o(vision)h(of)f(in)o(ter-comm)o (unication)i(do)q(es)e(not)g(adv)o(ersely)h(a\013ect)e(the)i(\(p)q (oten)o(tial\))f(p)q(erfor-)75 517 y(mance)g(of)g(in)o(tra-comm)o (unication.)75 642 y Fo(\\Under)h(the)g(Ho)q(o)q(d")g(Implementation)f (Notes)75 730 y FE(A)k(p)q(ossible)h(implemen)o(tation)g(of)e(a)g(comm) o(unicator)h(con)o(tains)f(a)h(single)h(group,)e(a)h(send)g(con)o (text,)f(a)75 786 y(receiv)o(e)f(con)o(text,)f(and)g(a)g(source)h (\(for)e(the)i(message)f(en)o(v)o(elop)q(e\).)24 b(This)17 b(structure)f(mak)o(es)f(in)o(tra-)i(and)75 843 y(in)o(ter-comm)o (unicators)e(basically)i(the)e(same)g(ob)s(ject.)166 900 y(The)21 b(in)o(tra-comm)o(unicator)f(has)h(the)f(prop)q(erties)i (that:)30 b(the)20 b(send-con)o(text)h(and)g(the)g(receiv)o(e-)75 956 y(con)o(text)c(are)g(iden)o(tical;)j(the)d(pro)q(cess)h(is)g(a)f (mem)o(b)q(er)g(of)g(the)h(group;)g(the)f(source)g(is)h(the)g(rank)f (of)g(the)75 1013 y(pro)q(cess)e(in)h(the)g(group.)166 1070 y(The)e(in)o(ter-comm)o(unicator)h(cannot)e(b)q(e)i(discussed)h (sensibly)g(without)e(considering)i(pro)q(cesses)e(in)75 1127 y(b)q(oth)19 b(the)g(lo)q(cal)h(and)f(remote)g(groups.)31 b(Imagine)19 b(a)g(pro)q(cess)g FF(P)g FE(in)h(group)e(G)h(whic)o(h)h (has)f(an)g(in)o(ter-)75 1183 y(comm)o(unicator)14 b Fp(Cp)p FE(,)f(and)i(a)f(pro)q(cess)g FF(Q)g FE(in)h(group)f(H)g(whic)o (h)h(has)f(an)g(in)o(ter-comm)o(unicator)g Fp(Cq)p FE(.)20 b(\(Note)75 1239 y(that)e(G)g(and)g(H)g(do)h(not)e(ha)o(v)o(e)h(to)g(b) q(e)h(distinct.\))30 b(The)19 b(in)o(ter-comm)o(unicators)f(ha)o(v)o(e) g(the)g(prop)q(erties)75 1296 y(that:)g(the)c(send)f(con)o(text)g(of)g Fp(Cp)g FE(is)h(iden)o(tical)h(to)e(the)g(receiv)o(e)h(con)o(text)f(of) g Fp(Cq)p FE(,)g(and)g(is)h(unique)h(in)f(H;)f(the)75 1352 y(receiv)o(e)i(con)o(text)e(of)h Fp(Cp)g FE(is)g(iden)o(tical)i (to)d(the)h(send)h(con)o(text)e(of)h Fp(Cq)p FE(,)f(and)i(is)f(unique)i (in)e(G;)g(the)g(group)f(of)75 1409 y Fp(Cp)i FE(is)g(H;)g(the)h(group) e(of)h Fp(Cq)g FE(is)h(G;)e(the)h(source)g(of)g Fp(Cp)g FE(is)h(the)f(rank)g(of)f FF(P)h FE(in)h(G,)f(whic)o(h)h(is)f(the)h (group)e(of)75 1465 y Fp(Cq)p FE(;)h(the)g(source)g(of)g Fp(Cq)g FE(is)g(the)h(rank)f(of)f(Q)i(in)g(H,)f(whic)o(h)h(is)g(the)f (group)g(of)g Fp(Cp)p FE(.)166 1523 y(It)h(is)g(easy)g(to)f(see)i(that) e(in)i(terms)e(of)g(these)i(\014elds,)g(the)f(in)o(tra-comm)o(unicator) f(is)i(a)e(sp)q(ecial)j(case)75 1579 y(of)f(the)g(in)o(ter-comm)o (unicator.)26 b(It)17 b(has)h FA(G)g FE(=)f FA(H)h FE(and)f(b)q(oth)h (con)o(texts)e(the)h(same.)26 b(This)18 b(ensures)g(that)75 1636 y(the)d(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unication)g(implemen)o (tation)g(for)f(in)o(tra-comm)o(unication)g(and)h(in)o(ter-com-)75 1692 y(m)o(unication)g(can)f(b)q(e)h(iden)o(tical.)75 1840 y Fy(3.8)59 b(Cacheing)75 1943 y FE(MPI)16 b(pro)o(vides)g(a)g (\\cac)o(heing")g(facilit)o(y)h(that)f(allo)o(ws)g(an)g(application)h (to)f(attac)o(h)f(arbitrary)g(pieces)i(of)75 2000 y(information,)d (called)h Ft(attributes)p FE(,)g(to)e(con)o(text,)g(group,)g(and)i (comm)o(unicator)e(descriptors;)h(it)g(pro)o(vides)75 2056 y(this)d(facilit)o(y)i(to)d(user)h(programs)f(as)h(w)o(ell.)19 b(A)o(ttributes)11 b(are)g(lo)q(cal)h(to)e(the)h(pro)q(cess)h(and)f (are)g(not)f(included)75 2113 y(if)j(the)f(descriptor)h(w)o(ere)f (someho)o(w)g(sen)o(t)g(to)f(another)h(pro)q(cess)1143 2096 y Fw(2)1163 2113 y FE(.)19 b(This)13 b(facilit)o(y)h(is)e(in)o (tended)i(to)e(supp)q(ort)75 2169 y(optimizations)22 b(suc)o(h)g(as)f(sa)o(ving)g(p)q(ersisten)o(t)h(comm)o(unication)g (handles)h(and)e(recording)h(top)q(ology-)75 2226 y(based)c(decisions)h (b)o(y)e(adaptiv)o(e)h(algorithms.)26 b(Ho)o(w)o(ev)o(er,)17 b(attributes)g(are)h(propagated)e(in)o(ten)o(tionally)75 2282 y(b)o(y)f(sp)q(eci\014c)i(MPI)e(routines.)166 2339 y(T)l(o)f(summarize,)g(cac)o(heing)h(is,)f(in)h(particular,)g(the)f (pro)q(cess)g(b)o(y)g(whic)o(h)h(implemen)o(tation-de\014ned)75 2396 y(data)g(\(and)g(virtual)g(top)q(ology)g(data\))g(is)g(propagated) g(in)h(groups)f(and)g(comm)o(unicators.)166 2536 y Fv(Discussion:)g Fu(A)o(ttribute)g(propagation)e(m)o(ust)g(b)q(e)h(discussed)i (carefully)m(.)p 75 2661 720 2 v 127 2688 a FC(2)144 2704 y Fs(The)d(deletion)i(of)e(\015atten/un\015atten)i(mak)o(es)f (this)f(p)q(oin)o(t)i(mo)q(ot.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 34 28 33 bop 75 -100 a FE(28)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fq(3.8.1)49 b(F)o(unctionalit)o(y)75 137 y FE(MPI)15 b(pro)o(vides)h(the)f(follo)o(wing)h(services)g(related)g(to)e(cac)o (heing.)21 b(They)15 b(are)g(all)h(pro)q(cess-lo)q(cal.)75 245 y Fo(MPI)p 160 245 14 2 v 16 w(A)l(TTRIBUTE)p 425 245 V 17 w(ALLOC\(n,handle)p 760 245 V 18 w(a)o(rra)o(y)l(,len\))117 325 y Fu(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)13 b(of)h(handles)g(to) g(allo)q(cate)117 407 y(OUT)124 b Fo(handle)p 459 407 V 17 w(a)o(rra)o(y)337 b Fu(p)q(oin)o(ter)10 b(to)f(arra)o(y)g(of)g (opaque)g(attribute)h(handling)e(structure)117 489 y(OUT)124 b Fo(len)517 b Fu(length)14 b(of)f(eac)o(h)i(opaque)f(structure)75 617 y FE(Allo)q(cates)19 b(a)e(new)h(attribute,)g(so)f(user)h(programs) f(and)h(functionalit)o(y)h(la)o(y)o(ered)f(on)f(top)h(of)f(MPI)h(can)75 673 y(access)d(attribute)g(tec)o(hnology)l(.)75 780 y Fo(MPI)p 160 780 V 16 w(A)l(TTRIBUTE)p 425 780 V 17 w(FREE\(handle)p 691 780 V 18 w(a)o(rra)o(y)l(,n\))117 861 y Fu(IN)171 b Fo(handle)p 459 861 V 17 w(a)o(rra)o(y)337 b Fu(arra)o(y)16 b(of)e(p)q(oin)o(ters)j(to)e(opaque)g(attribute)i(handling)d(struc-)905 917 y(tures)117 999 y(IN)171 b Fo(n)548 b Fu(n)o(um)o(b)q(er)13 b(of)h(handles)g(to)g(deallo)q(cate)75 1127 y FE(F)l(rees)h(attribute)g (handle.)75 1234 y Fo(MPI)p 160 1234 V 16 w(GET)p 264 1234 V 17 w(A)l(TTRIBUTE)p 530 1234 V 17 w(KEY\(k)o(eyval\))117 1315 y Fu(OUT)124 b Fo(k)o(eyval)455 b Fu(Pro)o(vide)14 b(the)h(in)o(teger)f(k)o(ey)g(v)n(alue)f(for)h(future)g(storing.)75 1443 y FE(Generates)h(a)g(new)g(cac)o(he)h(k)o(ey)l(.)75 1550 y Fo(MPI)p 160 1550 V 16 w(SET)p 259 1550 V 17 w(A)l (TTRIBUTE\(handle,)c(k)o(eyval,)f(attribute)p 993 1550 V 18 w(val,)f(attribute)p 1251 1550 V 18 w(len,)h(attribute)p 1510 1550 V 19 w(destructo)o(r)p 1718 1550 V 17 w(routine\))117 1687 y Fu(IN)171 b Fo(handle)449 b Fu(opaque)14 b(attribute)g(handle) 117 1769 y(IN)171 b Fo(k)o(eyval)455 b Fu(The)15 b(in)o(teger)f(k)o(ey) g(v)n(alue)f(for)h(future)g(storing.)117 1851 y(IN)171 b Fo(attribute)p 500 1851 V 18 w(val)336 b Fu(attribute)15 b(v)n(alue)e(\(opaque)h(p)q(oin)o(ter\))117 1933 y(IN)171 b Fo(attribute)p 500 1933 V 18 w(len)336 b Fu(length)14 b(of)f(attribute)i(\(in)e(b)o(ytes\))117 2014 y(IN)171 b Fo(attribute)p 500 2014 V 18 w(destructo)o(r)p 707 2014 V 17 w(routine)52 b Fu(What)14 b(one)g(calls)f(to)h(get)g(rid)g (of)f(this)h(attribute)g(later)75 2142 y FE(Stores)h(attribute)g(in)h (cac)o(he)f(b)o(y)g(k)o(ey)l(.)75 2249 y Fo(MPI)p 160 2249 V 16 w(TEST)p 290 2249 V 16 w(A)l(TTRIBUTE\(handle,k)o (eyval,attribute)p 1000 2249 V 20 w(ptr,len\))117 2330 y Fu(IN)171 b Fo(handle)449 b Fu(opaque)14 b(attribute)g(handle)117 2412 y(IN)171 b Fo(k)o(eyval)455 b Fu(The)15 b(in)o(teger)f(k)o(ey)g(v) n(alue)f(for)h(future)g(storing.)117 2494 y(OUT)124 b Fo(attribute)p 500 2494 V 18 w(ptr)335 b Fu(v)o(oid)13 b(p)q(oin)o(ter)h(to)g(attribute,)g(or)g(NULL)g(if)f(not)h(found)117 2576 y(OUT)124 b Fo(len)517 b Fu(length)14 b(in)f(b)o(ytes)i(of)e (attribute,)h(if)f(found.)75 2704 y FE(Retriev)o(e)j(attribute)f(from)g (cac)o(he)g(b)o(y)g(k)o(ey)l(.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 35 29 34 bop 75 -100 a Fx(3.8.)29 b(CA)o(CHEING)1404 b FE(29)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(DELETE)p 346 45 V 16 w(A)l(TTRIBUTE\(handle,)17 b(k)o(eyval\))117 124 y Fu(IN)171 b Fo(handle)449 b Fu(opaque)14 b(attribute)g(handle)117 204 y(IN)171 b Fo(k)o(eyval)455 b Fu(The)15 b(in)o(teger)f(k)o(ey)g(v)n (alue)f(for)h(future)g(storing.)75 330 y FE(Delete)i(attribute)f(from)f (cac)o(he)i(b)o(y)f(k)o(ey)l(.)75 462 y Fo(Example)75 552 y FE(Eac)o(h)k(attribute)g(consists)h(of)f(a)g(p)q(oin)o(ter)h(or)e (a)h(v)m(alue)i(of)e(the)g(same)g(size)i(as)e(a)g(p)q(oin)o(ter,)h(and) g(w)o(ould)75 609 y(t)o(ypically)14 b(b)q(e)f(a)f(reference)h(to)e(a)h (larger)g(blo)q(c)o(k)h(of)f(storage)f(managed)h(b)o(y)h(the)f(mo)q (dule.)20 b(As)12 b(an)g(example,)75 665 y(a)k(global)i(op)q(eration)e (using)i(cac)o(heing)f(to)f(b)q(e)h(more)g(e\016cien)o(t)g(for)f(all)h (con)o(texts)f(of)g(a)h(group)f(after)g(the)75 722 y(\014rst)f(call)h (migh)o(t)f(lo)q(ok)h(lik)o(e)g(this:)147 841 y Fp(static)23 b(int)g(gop_key_assigned)f(=)h(0;)96 b(/*)23 b(0)h(only)f(on)h(first)f (entry)g(*/)147 897 y(static)g(int)g(gop_key;)190 b(/*)23 b(key)h(for)f(this)h(module's)e(stuff)i(*/)147 1010 y (efficient_global_op)d(\(comm,)i(...\))147 1066 y(void)g(*comm;)147 1123 y({)194 1179 y(struct)g(gop_stuff_type)f(*gop_stuff;)70 b(/*)24 b(whatever)f(we)g(need)h(*/)194 1236 y(void)47 b(*group)24 b(=)f(mpi_comm_group\(comm\);)194 1349 y(if)h (\(!gop_key_assigned\))117 b(/*)23 b(get)h(a)f(key)h(on)f(first)h(call) f(ever)g(*/)194 1405 y({)h(gop_key_assigned)e(=)h(1;)242 1462 y(if)h(\()f(!)h(\(gop_key)f(=)h(mpi_Get_Attribute_Key\(\))o(\))d (\))j({)290 1518 y(mpi_abort)e(\("Insufficient)g(keys)i(available"\);) 242 1574 y(})194 1631 y(})194 1687 y(if)g(\(mpi_Test_Attribute)d (\(mpi_group_attr\(group\),gop_)o(key,&gop)o(_stuff\))o(\))194 1744 y({)j(/*)g(This)f(module)g(has)g(executed)g(in)h(this)f(group)g (before.)314 1800 y(We)g(will)h(use)f(the)g(cached)g(information)g(*/) 194 1857 y(})194 1913 y(else)194 1970 y({)h(/*)g(This)f(is)h(a)f(group) g(that)h(we)f(have)h(not)f(yet)h(cached)f(anything)f(in.)314 2026 y(We)h(will)h(now)f(do)h(so.)266 2083 y(*/)242 2139 y(gop_stuff)f(=)g(/*)h(malloc)f(a)h(gop_stuff_type)e(*/)242 2252 y(/*)i(...)f(fill)g(in)h(*gop_stuff)e(with)i(whatever)f(we)g(want) g(...)h(*/)242 2365 y(mpi_set_attribute)e(\(mpi_group_attr\(group\),)e (gop_key,)j(gop_stuff,)958 2421 y(gop_stuff_destructor\);)194 2478 y(})194 2534 y(/*)h(...)f(use)h(contents)f(of)g(*gop_stuff)g(to)g (do)h(the)f(global)g(op)h(...)f(*/)170 2591 y(})170 2704 y(gop_stuff_destructor)f(\(gop_stuff\))70 b(/*)23 b(called)g(by)h(MPI)f (on)h(group)f(delete)g(*/)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 36 30 35 bop 75 -100 a FE(30)409 b Fx(SECTION)16 b(3.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)170 45 y Fp(struct)23 b(gop_stuff_type)f(*gop_stuff;)170 102 y({)218 158 y(/*)i(...)f(free)h(storage)e(pointed)h(to)h(by)g (gop_stuff)e(...)i(*/)170 214 y(})166 404 y Fv(Discussion:)15 b Fu(The)e(cac)o(he)i(facilit)o(y)c(could)i(also)f(b)q(e)i(pro)o(vided) f(for)g(other)g(descriptors,)i(but)e(it)g(is)g(less)h(clear)75 461 y(ho)o(w)d(suc)o(h)g(pro)o(vision)g(w)o(ould)f(b)q(e)h(useful.)18 b(It)11 b(is)g(suggested)h(that)f(this)g(issue)h(b)q(e)g(review)o(ed)g (in)e(reference)k(to)d(Virtual)75 517 y(T)m(op)q(ologies.)75 744 y Fy(3.9)59 b(F)n(o)n(rmalizing)21 b(the)e(Lo)r(osely)g (Synchronous)g(Mo)r(del)g(\(Usage,)g(Safet)n(y\))75 847 y Fq(3.9.1)49 b(Basic)17 b(Statements)75 933 y FE(When)c(a)f(caller)i (passes)f(a)f(comm)o(unicator)h(\(whic)o(h)g(con)o(tains)g(a)f(con)o (text)g(and)h(group\))f(to)g(a)g(callee,)j(that)75 989 y(comm)o(unicator)g(m)o(ust)h(b)q(e)g(free)g(of)f(side)i(e\013ects)e (throughout)g(execution)i(of)e(the)h(subprogram)f(\(quies-)75 1046 y(cen)o(t\).)26 b(This)18 b(pro)o(vides)g(one)f(mo)q(del)h(in)g (whic)o(h)h(libraries)g(can)e(b)q(e)h(written,)f(and)h(w)o(ork)e (\\safely)l(.")27 b(F)l(or)75 1102 y(libraries)13 b(so)f(designated,)h (the)f(callee)h(has)f(p)q(ermission)h(to)e(do)h(whatev)o(er)f(comm)o (unication)i(it)f(lik)o(es)h(with)75 1159 y(the)19 b(comm)o(unicator,)g (and)g(under)h(the)f(ab)q(o)o(v)o(e)g(guaran)o(tee)f(kno)o(ws)g(that)h (no)g(other)f(comm)o(unications)75 1215 y(will)e(in)o(terfere.)k(Since) c(w)o(e)e(p)q(ermit)h(the)f(creation)g(of)g(new)h(comm)o(unicators)f (without)g(sync)o(hronization)75 1272 y(\(assuming)h(preallo)q(cated)i (con)o(texts\),)c(this)j(do)q(es)g(not)e(imp)q(ose)i(a)f(signi\014can)o (t)h(o)o(v)o(erhead.)166 1328 y(This)k(form)e(of)h(safet)o(y)f(is)i (analogous)f(to)f(other)h(common)g(computer)g(science)h(usages,)g(suc)o (h)f(as)75 1385 y(passing)c(a)f(descriptor)h(of)f(an)h(arra)o(y)e(to)h (a)g(library)i(routine.)k(The)15 b(library)g(routine)g(has)g(ev)o(ery)f (righ)o(t)h(to)75 1441 y(exp)q(ect)h(suc)o(h)f(a)g(descriptor)h(to)f(b) q(e)g(v)m(alid)i(and)f(mo)q(di\014able.)75 1564 y Fq(3.9.2)49 b(Mo)q(dels)17 b(of)g(Execution)75 1650 y FE(W)l(e)i(sa)o(y)f(that)f(a) i(parallel)h(pro)q(cedure)f(is)g Ft(active)g FE(at)f(a)g(pro)q(cess)h (if)g(the)g(pro)q(cess)g(b)q(elongs)g(to)f(a)g(group)75 1706 y(that)e(ma)o(y)h(collectiv)o(ely)i(execute)f(the)f(pro)q(cedure,) h(and)f(some)f(mem)o(b)q(er)h(of)g(that)f(group)h(is)g(curren)o(tly)75 1763 y(executing)c(the)f(pro)q(cedure)g(co)q(de.)20 b(If)12 b(a)f(parallel)j(pro)q(cedure)e(is)h(activ)o(e)e(at)h(a)f(pro)q(cess,)h (then)g(this)h(pro)q(cess)75 1819 y(ma)o(y)e(b)q(e)i(receiving)g (messages)e(p)q(ertaining)i(to)f(this)g(pro)q(cedure,)h(ev)o(en)f(if)g (it)g(do)q(es)h(not)e(curren)o(tly)h(execute)75 1876 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 1996 y Fo(Nonreentrant)i(pa)o(rallel)d(p)o(ro)q(cedures)75 2082 y FE(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o(y)g(p)q (oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(o)q(cation)g(of)f(a) g(parallel)75 2139 y(pro)q(cedure)14 b(can)f(b)q(e)g(activ)o(e)g(at)f (an)o(y)h(pro)q(cess.)19 b(That)12 b(is,)i(concurren)o(t)e(in)o(v)o(o)q (cations)i(of)e(the)h(same)f(parallel)75 2195 y(pro)q(cedure)k(ma)o(y)e (o)q(ccur)h(only)g(within)h(disjoin)o(t)f(groups)f(of)h(pro)q(cesses.) 20 b(F)l(or)14 b(example,)h(all)h(in)o(v)o(o)q(cations)75 2252 y(of)f(parallel)i(pro)q(cedures)f(in)o(v)o(olv)o(e)g(all)h(pro)q (cesses,)e(pro)q(cesses)h(are)f(single-threaded,)i(and)e(there)h(are)f (no)75 2308 y(recursiv)o(e)h(in)o(v)o(o)q(cations.)166 2365 y(In)21 b(suc)o(h)f(a)g(case,)h(a)f(con)o(text)f(can)i(b)q(e)f (statically)h(allo)q(cated)g(to)f(eac)o(h)g(pro)q(cedure.)36 b(The)20 b(static)75 2421 y(allo)q(cation)h(can)e(b)q(e)h(done)g(in)h (a)e(pream)o(ble,)i(as)e(part)g(of)g(initialization)j(co)q(de.)34 b(Or,)20 b(it)g(can)f(b)q(e)i(done)75 2478 y(a)d(compile/link)j(time,)e (if)g(the)g(implemen)o(tation)g(has)g(additional)h(mec)o(hanisms)f(to)e (reserv)o(e)i(con)o(text)75 2534 y(v)m(alues.)h(Comm)o(unicators)12 b(to)g(b)q(e)h(used)g(b)o(y)f(the)h(di\013eren)o(t)f(pro)q(cedures)i (can)e(b)q(e)h(build)i(in)e(a)f(pream)o(ble,)h(if)75 2591 y(the)f(executing)g(groups)f(are)h(statically)g(de\014ned;)i(if)e (the)f(executing)i(groups)e(c)o(hange)h(dynamically)l(,)i(then)75 2647 y(a)j(new)g(comm)o(unicator)f(has)h(to)f(b)q(e)i(built)g(whenev)o (er)f(the)g(executing)h(group)f(c)o(hanges,)f(but)h(this)h(new)75 2704 y(comm)o(unicator)d(can)h(b)q(e)g(built)h(using)f(the)f(same)g (preallo)q(cated)i(con)o(text.)j(If)c(the)f(parallel)i(pro)q(cedures) 1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 37 31 36 bop 75 -100 a Fx(3.9.)33 b(F)o(ORMALIZING)15 b(THE)f(LOOSEL)l(Y)i (SYNCHR)o(ONOUS)g(MODEL)e(\(USA)o(GE,)f(SAFETY\))p FE(31)75 45 y(can)d(b)q(e)h(organized)g(in)o(to)f(libraries,)j(so)d(that)f(only) i(one)f(pro)q(cedure)i(of)e(eac)o(h)g(library)h(can)f(b)q(e)h (concurren)o(tly)75 102 y(activ)o(e)k(at)g(eac)o(h)g(pro)q(cessor,)g (then)g(it)h(is)f(su\016cien)o(t)h(to)f(allo)q(cate)h(one)f(con)o(text) g(p)q(er)g(library)l(.)75 236 y Fo(P)o(a)o(rallel)f(p)o(ro)q(cedures)i (that)g(a)o(re)f(nonreentrant)i(within)f(each)g(executing)h(group)75 327 y FE(This)k(co)o(v)o(ers)g(the)g(case)f(where,)j(at)d(an)o(y)g(p)q (oin)o(t)i(in)f(time,)h(for)f(eac)o(h)g(pro)q(cess)g(group,)g(there)g (can)g(b)q(e)75 383 y(at)d(most)g(one)h(activ)o(e)g(in)o(v)o(o)q (cation)g(of)g(a)f(parallel)i(pro)q(cedure)g(b)o(y)f(a)f(pro)q(cess)h (mem)o(b)q(er.)31 b(Ho)o(w)o(ev)o(er,)19 b(it)75 439 y(migh)o(t)i(b)q(e)h(p)q(ossible)i(that)c(the)i(same)f(pro)q(cedure)h (is)g(concurren)o(tly)g(in)o(v)o(ok)o(ed)g(in)g(t)o(w)o(o)e(partially)j (\(or)75 496 y(completely\))17 b(o)o(v)o(erlapping)f(groups.)k(F)l(or) 15 b(example,)i(the)e(same)g(collectiv)o(e)j(comm)o(unication)e (function)75 552 y(ma)o(y)e(b)q(e)i(concurren)o(tly)g(in)o(v)o(ok)o(ed) g(on)f(t)o(w)o(o)f(partially)i(o)o(v)o(erlapping)g(groups.)166 611 y(In)f(suc)o(h)f(a)g(case,)g(a)f(con)o(text)h(is)g(asso)q(ciated)h (with)f(eac)o(h)g(parallel)i(pro)q(cedure)f(and)f(eac)o(h)g(executing) 75 668 y(group,)j(so)h(that)e(o)o(v)o(erlapping)j(execution)f(groups)f (ha)o(v)o(e)g(distinct)i(comm)o(unication)f(con)o(texts.)27 b(\(One)75 724 y(do)q(es)19 b(not)f(need)i(a)f(di\013eren)o(t)f(con)o (text)h(from)f(eac)o(h)h(group;)g(one)g(merely)h(needs)f(a)g (\\coloring")g(of)f(the)75 781 y(groups,)d(so)g(that)g(One)h(can)g (generate)g(the)f(comm)o(unicators)g(for)g(eac)o(h)h(parallel)h(pro)q (cedure)g(when)f(the)75 837 y(execution)g(groups)e(are)h(de\014ned.)21 b(Here,)14 b(again,)h(one)g(only)g(need)h(one)e(con)o(text)h(for)f(eac) o(h)g(library)l(,)i(if)f(no)75 894 y(t)o(w)o(o)f(pro)q(cedures)i(from)e (the)i(same)f(library)h(can)f(b)q(e)h(concurren)o(tly)g(activ)o(e)f(in) h(the)f(same)g(group.)166 953 y(Note)f(that,)g(for)h(collectiv)o(e)h (comm)o(unication)g(libraries,)g(w)o(e)f(do)g(allo)o(w)g(sev)o(eral)g (concurren)o(t)g(in)o(v)o(o-)75 1009 y(cations)h(within)h(the)f(same)g (group:)21 b(a)15 b(broadcast)h(in)g(a)g(group)g(ma)o(y)f(b)q(e)h (started)g(at)f(a)g(pro)q(cess)i(b)q(efore)75 1065 y(the)e(previous)h (broadcast)f(in)h(that)e(group)h(ended)h(at)f(another)g(pro)q(cess.)20 b(In)c(suc)o(h)f(a)g(case,)g(one)g(cannot)75 1122 y(rely)e(on)f(con)o (text)g(mec)o(hanisms)h(to)f(disam)o(biguate)h(successiv)o(e)h(in)o(v)o (o)q(cations)e(of)g(the)h(same)f(parallel)i(pro-)75 1178 y(cedure)f(within)h(the)f(same)f(group:)18 b(the)13 b(pro)q(cedure)g (need)h(b)q(e)f(implemen)o(ted)h(so)e(as)g(to)g(a)o(v)o(oid)h (confusion.)75 1235 y(F)l(or)f(example,)i(for)e(broadcast,)g(one)h(ma)o (y)f(need)i(to)e(carry)g(additional)j(information)e(in)g(messages,)g (suc)o(h)75 1291 y(as)19 b(the)g(broadcast)g(ro)q(ot,)g(to)g(help)i(in) f(suc)o(h)g(disam)o(biguation;)i(one)d(also)g(relies)i(on)e(preserv)m (ation)h(of)75 1348 y(message)e(order)f(b)o(y)h(MPI.)g(With)g(suc)o(h)g (an)g(approac)o(h,)g(w)o(e)g(ma)o(y)f(b)q(e)i(gaining)g(p)q (erformance,)f(but)g(w)o(e)75 1404 y(lo)q(ose)f(mo)q(dularit)o(y)l(.)27 b(It)17 b(is)g(not)g(su\016cien)o(t)g(to)g(implemen)o(t)h(the)f (parallel)i(pro)q(cedure)f(so)e(that)h(it)g(w)o(orks)75 1461 y(correctly)k(in)h(isolation,)g(when)g(in)o(v)o(ok)o(ed)f(only)g (once;)j(it)d(needs)g(to)g(b)q(e)g(implemen)o(ted)i(so)d(that)g(an)o(y) 75 1517 y(n)o(um)o(b)q(er)c(of)g(successiv)o(e)h(in)o(v)o(o)q(cations)f (will)i(execute)f(correctly)l(.)22 b(Of)17 b(course,)e(the)i(same)e (approac)o(h)h(can)75 1574 y(b)q(e)g(used)g(for)e(other)h(parallel)i (libraries.)75 1708 y Fo(W)o(ell-nested)g(pa)o(rallel)d(p)o(ro)q (cedures)75 1799 y FE(Calls)h(of)g(parallel)h(pro)q(cedures)g(are)e(w)o (ell)i(nested)f(if)g(a)g(new)g(parallel)h(pro)q(cedure)g(is)f(alw)o(a)o (ys)f(in)o(v)o(ok)o(ed)h(in)75 1855 y(a)j(subset)h(of)e(a)h(group)g (executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 1912 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 1970 y(In)i(suc)o(h)f(a)g(case,)h(a)f(new)g (con)o(text)g(need)h(to)f(b)q(e)h(dynamically)h(allo)q(cated)f(for)f (eac)o(h)g(new)h(in)o(v)o(o)q(ca-)75 2027 y(tion)g(of)f(a)g(parallel)i (pro)q(cedure.)28 b(Ho)o(w)o(ev)o(er,)16 b(a)i(stac)o(k)e(mec)o(hanism) i(can)g(b)q(e)g(used)g(for)f(allo)q(cating)i(new)75 2083 y(con)o(texts.)27 b(Th)o(us,)18 b(a)g(p)q(ossible)h(mec)o(hanism)g(is)f (to)f(allo)q(cate)i(\014rst)e(a)h(large)f(n)o(um)o(b)q(er)i(of)e(con)o (text's)g(\(up)75 2140 y(to)f(the)h(upp)q(er)h(b)q(ound)g(on)f(the)g (depth)g(of)g(nested)g(parallel)h(pro)q(cedure)g(calls\),)g(and)f(then) g(use)g(a)g(lo)q(cal)75 2196 y(stac)o(k)d(managemen)o(t)g(of)g(these)h (con)o(text's)f(on)g(eac)o(h)h(pro)q(cess)g(to)f(create)h(a)f(new)h (comm)o(unicator)g(\(using)75 2253 y Fo(MPI)p 160 2253 14 2 v 16 w(COMM)p 318 2253 V 16 w(MAKE)p FE(\))g(for)f(eac)o(h)i(new)f (in)o(v)o(o)q(cation.)75 2387 y Fo(The)h(General)f(case)75 2478 y FE(In)22 b(the)g(general)g(case,)h(there)e(ma)o(y)g(b)q(e)h(m)o (ultiple)i(concurren)o(tly)e(activ)o(e)g(in)o(v)o(o)q(cations)g(of)f (the)g(same)75 2534 y(parallel)i(pro)q(cedure)g(within)g(the)f(same)f (group;)j(in)o(v)o(o)q(cations)e(ma)o(y)f(not)h(b)q(e)g(w)o (ell-nested.)41 b(A)22 b(new)75 2591 y(con)o(text)17 b(need)i(to)e(b)q(e)h(created)g(for)f(eac)o(h)g(in)o(v)o(o)q(cation.)28 b(It)18 b(is)g(the)g(user)g(resp)q(onsibilit)o(y)i(to)d(mak)o(e)g(sure) 75 2647 y(that,)k(if)h(t)o(w)o(o)d(distinct)j(parallel)h(pro)q(cedures) e(are)g(in)o(v)o(ok)o(ed)g(concurren)o(tly)h(on)e(o)o(v)o(erlapping)i (sets)e(of)75 2704 y(pro)q(cesses,)15 b(then)h(con)o(text)e(allo)q (cation)j(or)d(comm)o(unicator)h(creation)h(is)f(prop)q(erly)h(co)q (ordinated.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 38 32 37 bop 75 -100 a FE(32)414 b Fx(SECTION)16 b(3.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fy(3.10)59 b(Motivating)19 b(Examples)75 230 y Fv(Discussion:)c Fu(The)e(in)o(tra-comm)o(uni)o(cation)c(examples)j(w)o(ere)i(\014rst)f (presen)o(ted)i(at)d(the)i(June)f(MPI)g(meeting;)e(the)75 287 y(in)o(ter-comm)o(unication)g(routines)j(\(when)h(added\))f(are)g (new.)75 494 y Fq(3.10.1)49 b(Current)15 b(Practice)h(#1)75 581 y FE(Example)g(#1a:)170 678 y Fp(int)24 b(me,)f(size;)170 734 y(...)170 791 y(mpi_init\(\);)170 847 y (mpi_comm_rank\(MPI_COMM_ALL,)e(&me\);)170 904 y (mpi_comm_size\(MPI_COMM_ALL,)g(&size\);)170 1017 y(printf\("Process)h (\045d)i(size)f(\045d\\n",)g(me,)h(size\);)170 1073 y(...)75 1169 y FE(Example)19 b(#1a)f(is)h(a)f(do-nothing)h(program)e(that)h (initializes)k(itself)d(legally)l(,)i(and)d(refers)g(to)g(the)h(the)75 1226 y(\\all")13 b(comm)o(unicator,)f(and)h(prin)o(ts)f(a)g(message.)19 b(This)13 b(example)g(do)q(es)g(not)f(imply)i(that)e(MPI)g(supp)q(orts) 75 1282 y(prin)o(tf-lik)o(e)17 b(comm)o(unication)f(itself.)75 1339 y(Example)g(#1b:)170 1436 y Fp(int)24 b(me,)f(size;)170 1492 y(...)170 1549 y(mpi_init\(\);)170 1605 y (mpi_comm_rank\(MPI_COMM_ALL,)e(&me\);)71 b(/*)23 b(local)g(*/)170 1662 y(mpi_comm_size\(MPI_COMM_ALL,)e(&size\);)i(/*)g(local)g(*/)170 1775 y(if\(\(me)g(\045)h(2\))g(==)f(0\))242 1831 y(mpi_send\(...,)f (\(\(me)h(+)h(1\))g(\045)f(size\),)h(MPI_COMM_ALL\);)170 1888 y(else)242 1944 y(mpi_recv\(...,)e(\(\(me)h(-)h(1)g(+)g(size\))f (\045)h(size\),)f(MPI_COMM_ALL\);)170 2000 y(...)75 2097 y FE(Example)16 b(#1b)f(sc)o(hematically)i(illustrates)g(message)e(exc) o(hanges)h(b)q(et)o(w)o(een)g(\\ev)o(en")f(and)h(\\o)q(dd")f(pro-)75 2153 y(cesses)h(in)g(the)f(\\all")g(comm)o(unicator.)75 2278 y Fq(3.10.2)49 b(Current)15 b(Practice)h(#2)170 2365 y Fp(void)24 b(*data;)170 2421 y(int)g(me;)170 2478 y(int)g(count;)170 2534 y(...)170 2591 y(mpi_init\(\);)170 2647 y(mpi_comm_rank\(MPI_COMM_ALL,)d(&me\);)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 39 33 38 bop 75 -100 a Fx(3.10.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b FE(33)170 45 y Fp(if\(me)24 b(==)f(0\))170 102 y({)266 158 y(/*)g(get)h(input,)f(create)g(buffer)g(``data'')g(*/) 266 214 y(...)170 271 y(})170 384 y(mpi_broadcast\(data,)f(count,)h (MPI_BYTE,)f(0,)i(MPI_COMM_ALL\);)170 497 y(...)75 577 y FE(This)16 b(example)g(illustrates)g(the)f(use)h(of)f(a)g(collectiv)o (e)h(comm)o(unication.)75 696 y Fq(3.10.3)49 b(\(App)o(ro)o(ximate\))16 b(Current)f(Practice)g(#3)170 782 y Fp(int)24 b(me,)f(count,)g(count2;) 170 838 y(void)h(*send_buf,)e(*recv_buf,)h(*send_buf2,)f(*recv_buf2;) 170 894 y(MPI_Group)h(MPI_GROUP_ALL,)f(grp0,)h(grprem;)170 951 y(MPI_Comm)g(commslave;)170 1007 y(static)g(int)h(ranks[])f(=)h ({0};)170 1064 y(...)170 1120 y(mpi_init\(\);)170 1177 y(mpi_comm_group\(MPI_COMM_ALL,)c(&MPI_GROUP_ALL\);)170 1233 y(mpi_comm_rank\(MPI_COMM_ALL,)h(&me\);)47 b(/*)23 b(local)g(*/)170 1290 y(mpi_local_subgroup\(group_all,)d(1,)k(ranks,)f (&grp0\);)g(/*)g(local)h(*/)170 1346 y(mpi_group_difference\(group_al)o (l,)d(grp0,)i(&grprem\);)g(/*)g(local)h(*/)170 1403 y (mpi_comm_make\(MPI_COMM_ALL,)d(grprem,)i(&commslave\);)170 1515 y(if\(me)h(!=)f(0\))170 1572 y({)266 1628 y(/*)g(compute)g(on)h (slave)f(*/)266 1685 y(...)266 1741 y(mpi_reduce\(send_buf,)e (recv_buff,)i(count,)g(MPI_BYTE,)f(commslave\);)266 1798 y(...)170 1854 y(})170 1911 y(/*)i(zero)f(falls)h(through)e (immediately)h(to)g(this)h(reduce,)f(others)g(do)g(later...)g(*/)170 1967 y(mpi_reduce\(send_buf2,)e(recv_buff2,)i(count2,)g(MPI_BYTE,)f (MPI_COMM_ALL\);)75 2048 y FE(This)c(example)h(illustrates)g(ho)o(w)e (a)h(group)f(consisting)i(of)e(all)i(but)f(the)g(zeroth)f(pro)q(cess)h (of)g(the)g(\\all")75 2104 y(group)13 b(is)h(created,)f(and)h(then)g (ho)o(w)f(a)g(comm)o(unicator)g(is)h(formed)f(\()g Fo(commslave)p FE(\))e(for)i(that)g(new)h(group.)75 2161 y(The)19 b(new)g(comm)o (unicator)g(is)g(used)h(in)f(a)g(collectiv)o(e)h(call,)h(and)e(all)h (pro)q(cesses)f(execute)h(a)e(collectiv)o(e)75 2217 y(call)k(in)g(the) 40 b Fr(MPI)p 403 2217 13 2 v 15 w(COMM)p 548 2217 V 14 w(ALL)22 b FE(con)o(text.)37 b(This)21 b(example)h(illustrates)g(ho) o(w)f(the)g(t)o(w)o(o)e(comm)o(unica-)75 2274 y(tors)d(\(whic)o(h)i(p)q (ossess)f(distinct)i(con)o(texts\))d(protect)h(comm)o(unication.)27 b(That)16 b(is,)i(comm)o(unication)g(in)75 2330 y Fr(MPI)p 152 2330 V 14 w(COMM)p 296 2330 V 15 w(ALL)e FE(is)f(insulated)i(from)e (comm)o(unication)h(in)31 b Fo(commslave)p FE(,)13 b(and)i(vice)i(v)o (ersa.)166 2387 y(In)h(summary)l(,)f(for)g(comm)o(unication)h(with)g (\\group)e(safet)o(y)l(,")h(con)o(texts)g(within)h(comm)o(unicators)75 2443 y(m)o(ust)d(b)q(e)g(distinct.)75 2561 y Fq(3.10.4)49 b(Example)17 b(#4)75 2647 y FE(The)e(follo)o(wing)g(example)g(is)g (mean)o(t)f(to)f(illustrate)j(\\safet)o(y")d(b)q(et)o(w)o(een)h(p)q (oin)o(t-to-p)q(oin)o(t)i(and)e(collectiv)o(e)75 2704 y(comm)o(unication.)20 b(MPI)12 b(guaran)o(tees)g(that)g(a)g(single)i (comm)o(unicator)e(can)h(do)f(safe)g(p)q(oin)o(t-to-p)q(oin)o(t)i(and) -32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 40 34 39 bop 75 -100 a FE(34)414 b Fx(SECTION)16 b(3.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y FE(collectiv)o(e)i(comm)o(unication.)75 140 y Fp(#define)23 b(TAG_ARBITRARY)f(12345)75 197 y(#define)h(SOME_COUNT)166 b(50)170 253 y(int)24 b(me;)170 310 y(MPI_Group)f(MPI_GROUP_ALL,)f (subgroup;)170 366 y(int)i(ranks[])f(=)h({2,)f(4,)h(6,)f(8};)170 423 y(...)170 479 y(mpi_init\(\);)170 536 y (mpi_comm_group\(MPI_COMM_ALL,)d(&MPI_GROUP_ALL\);)170 592 y(mpi_local_subgroup\(MPI_GROUP_)o(ALL,)h(4,)i(ranks,)g (&subgroup\);)g(/*)g(local)h(*/)170 648 y(mpi_group_rank\(subgroup,)d (&me\);)119 b(/*)23 b(local)g(*/)170 761 y (mpi_comm_make\(MPI_COMM_ALL,)e(subgroup,)h(&the_comm\);)170 874 y(if\(me)i(!=)f(MPI_UNDEFINED\))170 931 y({)266 987 y(/*)g(asynchronous)g(receive:)f(*/)266 1044 y(mpi_irecv\(handle,)f (start,)i(count,)h(MPI_DOUBLE,)528 1100 y(MPI_SRC_ANY,)f (TAG_ARBITRARY,)f(the_comm\);)170 1157 y(})170 1269 y(for\(i)i(=)f(0;)h (i)g(<)f(SOME_COUNT,)g(i++\))266 1326 y(mpi_reduce\(...,)f(the_comm\);) 75 1449 y Fq(3.10.5)49 b(Lib)o(ra)o(ry)17 b(Example)f(#1)75 1536 y FE(The)f(main)h(program:)170 1631 y Fp(int)24 b(done)f(=)h(0;)170 1687 y(user_lib_t)f(*libh_a,)g(*libh_b;)170 1744 y(void)h(*dataset1,)e(*dataset2;)170 1800 y(...)170 1857 y(mpi_init\(\);)170 1913 y(...)170 1970 y (init_user_lib\(MPI_COMM_ALL,)f(&libh_a\);)170 2026 y (init_user_lib\(MPI_COMM_ALL,)g(&libh_b\);)170 2083 y(...)170 2139 y(user_start_op\(libh_a,)g(dataset1\);)170 2195 y(user_start_op\(libh_a,)g(dataset2\);)170 2252 y(...)170 2308 y(while\(!done\))170 2365 y({)266 2421 y(/*)i(work)h(*/)266 2478 y(...)266 2534 y(mpi_reduce\(...,)e(MPI_COMM_ALL\);)266 2591 y(...)266 2647 y(/*)h(see)h(if)g(done)f(*/)266 2704 y(...)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 41 35 40 bop 75 -100 a Fx(3.10.)29 b(MOTIV)-5 b(A)l(TING)15 b(EXAMPLES)1056 b FE(35)170 45 y Fp(})170 102 y(user_end_op\(libh_a\);) 170 158 y(user_end_op\(libh_b\);)75 250 y FE(The)15 b(user)h(library)g (initialization)i(co)q(de:)75 342 y Fp(void)23 b(init_user_lib\(void)f (*comm,)h(user_lib_t)f(**handle\))75 399 y({)147 455 y(user_lib_t)g(*save;)147 512 y(void)h(*context;)147 568 y(void)g(*group;)147 624 y(int)g(len;)147 737 y (user_lib_initsave\(&save)o(\);)e(/*)j(local)f(*/)147 794 y(mpi_contexts_alloc\(comm)o(,)e(1\);)147 850 y (mpi_comm_dup\(comm,)g(&\(save)i(->)h(comm\)\);)147 963 y(/*)f(other)g(inits)h(*/)147 1020 y(*handle)e(=)i(save;)75 1076 y(})75 1168 y FE(Notice)17 b(that)e(the)h(comm)o(unicator)32 b Fo(comm)14 b FE(passed)i(to)f(the)i(library)f Ft(is)g FE(needed)h(to)f(allo)q(cate)g(new)h(con-)75 1225 y(texts.)75 1281 y(User)e(start-up)g(co)q(de:)75 1373 y Fp(void)23 b(user_start_op\(user_lib_t)e(*handle,)i(void)g(*data\))75 1430 y({)170 1486 y(mpi_irecv\(&\(handle)f(->)h(irecv_handle\),)f(...,) i(handle)f(->)g(comm\);)170 1543 y(mpi_isend\(&\(handle)f(->)h (isend_handle\),)f(...,)i(handle)f(->)g(comm\);)75 1599 y(})75 1691 y FE(User)15 b(clean-up)i(co)q(de:)75 1783 y Fp(void)23 b(user_end_op\(user_lib_t)e(*handle\))75 1840 y({)170 1896 y(mpi_wait\(handle)h(->)i(isend_handle\);)170 1953 y(mpi_wait\(handle)e(->)i(irecv_handle\);)75 2009 y(})75 2130 y Fq(3.10.6)49 b(Lib)o(ra)o(ry)17 b(Example)f(#2)75 2216 y FE(The)f(main)h(program:)170 2308 y Fp(int)24 b(ma,)f(mb;)170 2365 y(...)170 2421 y(list_a)g(:=)h(``[0,1]'';)170 2478 y(list_b)f(:=)h(``[0,2{,3}]'';)170 2591 y (mpi_local_subgroup\(MPI_GROUP_)o(ALL,)d(2,)i(list_a,)g(&group_a\);)170 2647 y(mpi_local_subgroup\(MPI_GROUP_)o(ALL,)e(2\(3\),)i(list_b,)g (&group_b\);)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 42 36 41 bop 75 -100 a FE(36)414 b Fx(SECTION)16 b(3.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)170 45 y Fp(mpi_comm_make\(MPI_COMM_ALL,)21 b(group_a,)h(&comm_a\);)170 102 y(mpi_comm_make\(MPI_COMM_ALL,)f(group_b,)h(&comm_b\);)170 214 y(mpi_comm_rank\(comm_a,)f(&ma\);)170 271 y(mpi_comm_rank\(comm_b,) g(&mb\);)170 384 y(if\(ma)j(!=)f(MPI_UNDEFINED\))242 440 y(lib_call\(comm_a\);)170 497 y(if\(mb)h(!=)f(MPI_UNDEFINED\))170 553 y({)242 610 y(lib_call\(comm_b\);)242 666 y(lib_call\(comm_b\);)170 723 y(})75 834 y FE(The)15 b(library:)75 950 y Fp(void)23 b(lib_call\(void)f(*comm\))75 1007 y({)170 1063 y(int)i(me,)f(done)h(=) f(0;)170 1120 y(mpi_comm_rank\(comm,)f(&me\);)170 1176 y(if\(me)i(==)f(0\))266 1233 y(while\(!done\))266 1289 y({)361 1345 y(mpi_recv\(...,)f(comm,)i(MPI_SRC_ANY\);)361 1402 y(...)266 1458 y(})170 1515 y(else)170 1571 y({)266 1628 y(/*)f(work)h(*/)266 1684 y(mpi_send\(...,)e(comm,)h(0\);)266 1741 y(....)170 1797 y(})170 1854 y(MPI_SYNC\(comm\);)70 b(/*)24 b(include/no)e(safety)h(for)h(safety/no)e(safety)h(*/)75 1910 y(})75 2022 y FE(The)16 b(ab)q(o)o(v)o(e)g(example)g(is)h(really)g (t)o(w)o(o)d(examples,)j(dep)q(ending)h(on)d(whether)i(or)e(not)g(y)o (ou)h(include)i(rank)75 2078 y(3)e(in)h Fp(list)p 267 2078 15 2 v 17 w(b)p FE(.)23 b(This)17 b(example)g(illustrates)g(that,) f(despite)h(con)o(texts,)f(subsequen)o(t)h(calls)g(to)f Fp(lib)p 1766 2078 V 17 w(call)75 2135 y FE(with)22 b(the)f(same)g(con) o(text)f(need)i(not)f(b)q(e)h(safe)f(from)f(one)h(another)g(\(\\bac)o (k)g(masking"\).)37 b(Safet)o(y)20 b(is)75 2191 y(realized)g(if)e(the) 37 b Fo(MPI)p 474 2191 14 2 v 16 w(SYNC)19 b FE(is)f(added.)30 b(What)17 b(this)i(demonstrates)e(is)i(that)e(libraries)j(ha)o(v)o(e)e (to)f(b)q(e)75 2247 y(written)e(carefully)l(,)i(ev)o(en)e(with)h(con)o (texts.)166 2308 y(Algorithms)i(lik)o(e)g(\\com)o(bine")g(ha)o(v)o(e)f (strong)f(enough)i(source)g(selectivit)o(y)h(so)e(that)f(they)i(are)f (in-)75 2365 y(heren)o(tly)k(OK.)g(So)g(are)g(m)o(ultiple)h(calls)g(to) e(a)g(t)o(ypical)i(tree)f(broadcast)f(algorithm)g(with)h(the)g(same)75 2421 y(ro)q(ot.)27 b(Ho)o(w)o(ev)o(er,)17 b(m)o(ultiple)j(calls)f(to)e (a)h(t)o(ypical)g(tree)g(broadcast)f(algorithm)h({)g(with)g(di\013eren) o(t)g(ro)q(ots)75 2478 y(|)h(could)i(break.)31 b(Therefore,)19 b(suc)o(h)h(algorithms)f(w)o(ould)g(ha)o(v)o(e)g(to)f(utilize)j(the)f (tag)e(to)g(k)o(eep)i(things)75 2534 y(straigh)o(t.)f(All)c(of)g(the)f (foregoing)g(is)h(a)f(discussion)i(of)e(\\collectiv)o(e)i(calls")g (implemen)o(ted)g(with)f(p)q(oin)o(t)g(to)75 2591 y(p)q(oin)o(t)k(op)q (erations.)28 b(MPI)18 b(implemen)o(tations)h(ma)o(y)f(or)f(ma)o(y)g (not)h(implemen)o(t)h(collectiv)o(e)h(calls)f(using)75 2647 y(p)q(oin)o(t-to-p)q(oin)o(t)e(op)q(erations.)22 b(These)16 b(algorithms)g(are)f(used)i(to)e(illustrate)i(the)f(issues)h (of)e(correctness)75 2704 y(and)g(safet)o(y)l(,)g(indep)q(enden)o(t)i (of)e(ho)o(w)g(MPI)g(implemen)o(ts)h(its)g(collectiv)o(e)h(calls.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 43 37 42 bop 75 -100 a Fx(3.10.)29 b(MOTIV)-5 b(A)l(TING)15 b(EXAMPLES)1056 b FE(37)75 45 y Fq(3.10.7)49 b(Inter-Communication)17 b(Examples)75 133 y Fo(Example)e(1:)k(Three-Group)e(\\Pip)q(eline")409 277 y Fp(+---------+)213 b(+---------+)h(+---------+)409 334 y(|)h(|)f(|)h(|)g(|)f(|)409 390 y(|)24 b(Group)f(0)h(|)f(<----->)g (|)h(Group)f(1)h(|)g(<----->)f(|)g(Group)h(2)f(|)409 447 y(|)215 b(|)f(|)h(|)g(|)f(|)409 503 y(+---------+)f(+---------+)h (+---------+)166 671 y FE(Groups)21 b(0)g(and)h(1)f(comm)o(unicate.)39 b(Groups)21 b(1)g(and)h(2)f(comm)o(unicate.)39 b(Therefore,)23 b(group)e(0)75 728 y(requires)e(one)f(in)o(ter-comm)o(unicator,)h (group)e(1)h(requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators,)h(and)g (group)g(2)75 784 y(requires)d(1)f(in)o(ter-comm)o(unicator.)20 b(Note)14 b(that)g(the)h(sync)o(hronous)f(in)o(ter-comm)o(unicator)h (constructor)75 841 y(\()p Fo(MPI)p 178 841 14 2 v 15 w(COMM)p 335 841 V 17 w(PEER)p 464 841 V 17 w(MAKE\(\))p FE(\))9 b(can)i(b)q(e)h(safely)e(used)i(here)f(since)h(there)e(is)i(no) e(cyclic)j(comm)o(unication.)170 953 y Fp(void)24 b(*)f(myComm;)453 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)f(*/)170 1010 y(void)i(*)f(myFirstComm;)786 b(/*)24 b(inter-communicator)d(*/) 170 1066 y(void)j(*)f(mySecondComm;)237 b(/*)24 b(second)f (inter-communicator)e(\(group)i(B)h(only\))f(*/)170 1123 y(int)h(membershipKey;)170 1179 y(int)g(subGroupLeaders[3];)170 1292 y(MPI_INIT\(\);)170 1349 y(...)170 1462 y(/*)g(User)f(code)h(must) f(generate)g(membershipKey)f(in)h(the)h(range)f([0,)h(1,)f(2])h(*/)170 1518 y(membershipKey)f(=)g(...)h(;)170 1631 y(/*)g(Build)f (intra-communicator)f(for)h(local)g(sub-group)g(and)g(get)h(group)f (leaders)g(*/)170 1687 y(/*)h(of)g(each)f(sub-group)g(\(relative)f(to)i (MPI_COMM_ALL\).)e(*/)170 1744 y(MPI_COMM_SPLITL\(MPI_COMM_ALL,)e (membershipKey,)i(3,)695 1800 y(subGroupLeaders,)g(&myComm\);)170 1913 y(/*)i(Build)f(inter-communicators.)45 b(Tags)24 b(are)f(hard-coded.)f(*/)170 1970 y(if)i(\(membershipKey)e(==)i(0\))266 2026 y({)692 b(/*)23 b(Group)h(0)f(communicates)g(with)g(group)g(1.)h (*/)266 2083 y(MPI_COMM_PEER_MAKE\(myComm)o(,)d(0,)j(MPI_COMM_ALL,)361 2139 y(subGroupLeaders[1],)e(10,)h(1,)h(&myFirstComm\);)266 2195 y(})170 2252 y(else)g(if)f(\(membershipKey)f(==)i(1\))266 2308 y({)525 b(/*)23 b(Group)h(1)f(communicates)f(with)i(groups)f(0)h (and)f(2.)h(*/)266 2365 y(MPI_COMM_PEER_MAKE\(myComm)o(,)d(0,)j (MPI_COMM_ALL,)361 2421 y(subGroupLeaders[0],)e(10,)h(1,)h (&myFirstComm\);)266 2478 y(MPI_COMM_PEER_MAKE\(myComm)o(,)d(0,)j (MPI_COMM_ALL,)361 2534 y(subGroupLeaders[2],)e(21,)h(1,)h (&mySecondComm\);)266 2591 y(})170 2647 y(else)g(if)f(\(membershipKey)f (==)i(2\))266 2704 y({)692 b(/*)23 b(Group)h(2)f(communicates)g(with)g (group)g(1.)h(*/)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 44 38 43 bop 75 -100 a FE(38)414 b Fx(SECTION)16 b(3.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)266 45 y Fp(MPI_COMM_PEER_MAKE\(myComm)o(,)21 b(0,)j(MPI_COMM_ALL,)361 102 y(subGroupLeaders[1],)e(21,)h(1,)h(&myFirstComm\);)266 158 y(})75 277 y Fo(Example)15 b(2:)k(Three-Group)e(\\Ring")290 363 y Fp(+-----------------------)o(--------)o(-------)o(-------)o (--------)o(------+)290 419 y(|)1408 b(|)290 476 y(|)95 b(+---------+)213 b(+---------+)h(+---------+)94 b(|)290 532 y(|)h(|)215 b(|)f(|)h(|)g(|)f(|)96 b(|)290 589 y(+-->)23 b(|)h(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f(|)g (Group)h(2)f(|)h(<--+)409 645 y(|)215 b(|)f(|)h(|)g(|)f(|)409 701 y(+---------+)f(+---------+)h(+---------+)166 799 y FE(Groups)18 b(0)f(and)i(1)f(comm)o(unicate.)29 b(Groups)17 b(1)h(and)h(2)e(comm)o(unicate.)29 b(Groups)18 b(0)g(and)g(2)g(com-)75 856 y(m)o(unicate.)49 b(Therefore,)26 b(eac)o(h)f(requires)h(t)o(w)o(o) d(in)o(ter-comm)o(unicators.)48 b(Note)24 b(that)g(the)h("lo)q(osely)75 912 y(sync)o(hronous")f(in)o(ter-comm)o(unicator)g(constructor)f(\()h Fo(MPI)p 1154 912 14 2 v 16 w(COMM)p 1312 912 V 16 w(PEER)p 1440 912 V 17 w(MAKE)p 1586 912 V 16 w(ST)l(ART\(\))h FE(and)75 969 y Fo(MPI)p 160 969 V 16 w(COMM)p 318 969 V 16 w(PEER)p 446 969 V 17 w(MAKE)p 592 969 V 16 w(FINISH\(\))p FE(\))c(is)h(the)g(b)q(est)g(c)o(hoice)g(here)h(due)f(to)f(the)h (cyclic)h(comm)o(uni-)75 1025 y(cation.)170 1123 y Fp(MPI_Comm)47 b(myComm;)333 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)g (*/)170 1179 y(MPI_Comm)g(myFirstComm;)667 b(/*)23 b (inter-communicators)f(*/)170 1236 y(MPI_Comm)h(mySecondComm;)170 1292 y(make_id)g(firstMakeID,)g(secondMakeID;)332 b(/*)23 b(handles)g(for)h("FINISH")f(*/)170 1349 y(int)h(membershipKey;)170 1405 y(int)g(subGroupLeaders[3];)170 1518 y(MPI_INIT\(\);)170 1574 y(...)170 1687 y(/*)g(User)f(code)h(must)f(generate)g (membershipKey)f(in)h(the)h(range)f([0,)h(1,)f(2])h(*/)170 1744 y(membershipKey)f(=)g(...)h(;)170 1857 y(/*)g(Build)f (intra-communicator)f(for)h(local)g(sub-group)g(and)g(get)h(group)f (leaders)g(*/)170 1913 y(/*)h(of)g(each)f(sub-group)g(\(relative)f(to)i (MPI_COMM_ALL\).)e(*/)170 1970 y(MPI_COMM_SPLITL\(MPI_COMM_ALL,)e (membershipKey,)i(3,)i(subGroupLeaders,)e(&myComm\);)170 2083 y(/*)i(Build)f(inter-communicators.)45 b(Tags)24 b(are)f(hard-coded.)f(*/)170 2139 y(if)i(\(membershipKey)e(==)i(0\))266 2195 y({)525 b(/*)23 b(Group)h(0)f(communicates)f(with)i(groups)f(1)h (and)f(2.)h(*/)266 2252 y(MPI_COMM_PEER_MAKE_START\()o(myComm,)c(0,)k (MPI_COMM_ALL,)e(subGroupLeaders[2],)f(20,)862 2308 y(1,)j (&firstMakeID\);)266 2365 y(MPI_COMM_PEER_MAKE_START\()o(myComm,)c(0,)k (MPI_COMM_ALL,)e(subGroupLeaders[1],)f(10,)862 2421 y(1,)j (&secondMakeID\);)266 2478 y(})170 2534 y(else)g(if)f(\(membershipKey)f (==)i(1\))266 2591 y({)525 b(/*)23 b(Group)h(1)f(communicates)f(with)i (groups)f(0)h(and)f(2.)h(*/)266 2647 y(MPI_COMM_PEER_MAKE_START\()o (myComm,)c(0,)k(MPI_COMM_ALL,)e(subGroupLeaders[0],)f(10,)862 2704 y(1,)j(&firstMakeID\);)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 45 39 44 bop 75 -100 a Fx(3.10.)29 b(MOTIV)-5 b(A)l(TING)15 b(EXAMPLES)1056 b FE(39)266 45 y Fp(MPI_COMM_PEER_MAKE_START\()o (myComm,)20 b(0,)k(MPI_COMM_ALL,)e(subGroupLeaders[2],)f(21,)862 102 y(1,)j(&secondMakeID\);)266 158 y(})170 214 y(else)g(if)f (\(membershipKey)f(==)i(2\))266 271 y({)525 b(/*)23 b(Group)h(2)f (communicates)f(with)i(groups)f(0)h(and)f(1.)h(*/)266 327 y(MPI_COMM_PEER_MAKE_START\()o(myComm,)c(0,)k(MPI_COMM_ALL,)e (subGroupLeaders[1],)f(21,)862 384 y(1,)j(&firstMakeID\);)266 440 y(MPI_COMM_PEER_MAKE_START\()o(myComm,)c(0,)k(MPI_COMM_ALL,)e (subGroupLeaders[0],)f(20,)862 497 y(1,)j(&secondMakeID\);)266 553 y(})170 666 y(/*)g(Everyone)f(has)g(the)h(same)f("FINISH")g (code...)g(*/)170 723 y(MPI_COMM_PEER_MAKE_FINISH\(fir)o(stMakeID)o(,)e (&myFirstComm\);)170 779 y(MPI_COMM_PEER_MAKE_FINISH\(sec)o(ondMakeI)o (D,)g(&mySecondComm\);)75 898 y Fo(Example)15 b(3:)k(Three-Group)e (\\Pip)q(eline")e(Using)h(Name)f(Service)409 984 y Fp(+---------+)213 b(+---------+)h(+---------+)409 1040 y(|)h(|)f(|)h(|)g(|)f(|)409 1097 y(|)24 b(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f (|)g(Group)h(2)f(|)409 1153 y(|)215 b(|)f(|)h(|)g(|)f(|)409 1210 y(+---------+)f(+---------+)h(+---------+)166 1307 y FE(Groups)21 b(0)g(and)h(1)f(comm)o(unicate.)39 b(Groups)21 b(1)g(and)h(2)f(comm)o(unicate.)39 b(Therefore,)23 b(group)e(0)75 1364 y(requires)e(one)f(in)o(ter-comm)o(unicator,)h(group)e(1)h (requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators,)h(and)g(group)g(2)75 1420 y(requires)d(1)f(in)o(ter-comm)o(unicator.)20 b(Note)14 b(that)g(the)h(sync)o(hronous)f(in)o(ter-comm)o(unicator)h(constructor) 75 1477 y(\()p Fo(MPI)p 178 1477 14 2 v 15 w(COMM)p 335 1477 V 17 w(NAME)p 481 1477 V 17 w(MAKE\(\))p FE(\))f(can)j(b)q(e)f (safely)h(used)f(here)h(since)g(there)f(is)h(no)f(cyclic)h(comm)o (unica-)75 1533 y(tion.)170 1631 y Fp(MPI_Comm)23 b(myComm;)381 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)f(*/)170 1687 y(MPI_Comm)h(myFirstComm;)714 b(/*)24 b(inter-communicator)d(*/) 170 1744 y(MPI_Comm)i(mySecondComm;)165 b(/*)24 b(second)f (inter-communicator)f(\(group)h(B)g(only\))g(*/)170 1857 y(MPI_INIT\(\);)170 1913 y(...)170 2026 y(/*)h(User)f(builds)g (intra-communicator)f(myComm)h(describing)f(the)i(local)f(sub-group)g (*/)170 2083 y(/*)h(using)f(any)h(appropriate)e(MPI)h(routine\(s\).)47 b(\(For)23 b(example,)g(myComm)g(could)g(*/)170 2139 y(/*)h(have)f(been)h(passed)f(in)g(as)h(an)f(argument)g(to)h(a)g(user)f (subroutine.\))f(*/)170 2195 y(myComm)h(=)h(...)g(;)170 2308 y(/*)g(Build)f(inter-communicators.)45 b(Group)23 b(membership)g(conditions)f(must)i(be)f(*/)170 2365 y(/*)h(provided)f (by)g(the)h(user.)f(*/)170 2421 y(if)h(\(\)) 266 2478 y({)692 b(/*)23 b(Group)h(0)f(communicates)g(with)g(group)g (1.)h(*/)266 2534 y(MPI_COMM_NAME_MAKE\(myComm)o(,)d("Connect)i(10",)g (1,)h(&myFirstComm\);)266 2591 y(})170 2647 y(else)g(if)f(\(\))266 2704 y({)525 b(/*)23 b(Group)h(1)f(communicates) f(with)i(groups)f(0)h(and)f(2.)h(*/)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 46 40 45 bop 75 -100 a FE(40)414 b Fx(SECTION)16 b(3.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)266 45 y Fp(MPI_COMM_NAME_MAKE\(myComm)o(,)21 b("Connect)i(10",)g(1,)h (&myFirstComm\);)266 102 y(MPI_COMM_NAME_MAKE\(myComm)o(,)d("Connect)i (21",)g(1,)h(&mySecondComm\);)266 158 y(})170 214 y(else)g(if)f(\(\))266 271 y({)692 b(/*)23 b(Group)h(2)f (communicates)g(with)g(group)g(1.)h(*/)266 327 y (MPI_COMM_NAME_MAKE\(myComm)o(,)d("Connect)i(21",)g(1,)h (&myFirstComm\);)266 384 y(})75 503 y Fo(Example)15 b(4:)k(Three-Group) e(\\Ring")e(Using)g(Name)g(Service)290 589 y Fp (+-----------------------)o(--------)o(-------)o(-------)o(--------)o (------+)290 645 y(|)1408 b(|)290 701 y(|)95 b(+---------+)213 b(+---------+)h(+---------+)94 b(|)290 758 y(|)h(|)215 b(|)f(|)h(|)g(|)f(|)96 b(|)290 814 y(+-->)23 b(|)h(Group)f(0)h(|)f (<----->)g(|)h(Group)f(1)h(|)g(<----->)f(|)g(Group)h(2)f(|)h(<--+)409 871 y(|)215 b(|)f(|)h(|)g(|)f(|)409 927 y(+---------+)f(+---------+)h (+---------+)166 1025 y FE(Groups)18 b(0)f(and)i(1)f(comm)o(unicate.)29 b(Groups)17 b(1)h(and)h(2)e(comm)o(unicate.)29 b(Groups)18 b(0)g(and)g(2)g(com-)75 1082 y(m)o(unicate.)49 b(Therefore,)26 b(eac)o(h)f(requires)h(t)o(w)o(o)d(in)o(ter-comm)o(unicators.)48 b(Note)24 b(that)g(the)h("lo)q(osely)75 1138 y(sync)o(hronous")20 b(in)o(ter-comm)o(unicator)h(constructor)f(\()h Fo(MPI)p 1141 1138 14 2 v 15 w(COMM)p 1298 1138 V 17 w(NAME)p 1444 1138 V 17 w(MAKE)p 1590 1138 V 16 w(ST)l(ART\(\))g FE(and)75 1194 y Fo(MPI)p 160 1194 V 16 w(COMM)p 318 1194 V 16 w(NAME)p 463 1194 V 17 w(MAKE)p 609 1194 V 16 w(FINISH\(\))p FE(\))15 b(is)h(the)g(b)q(est)g(c)o(hoice)g(here)h (due)f(to)f(the)h(cyclic)h(comm)o(unica-)75 1251 y(tion.)170 1349 y Fp(MPI_Comm)47 b(myComm;)333 b(/*)24 b(intra-communicator)d(of)j (local)f(sub-group)g(*/)170 1405 y(MPI_Comm)47 b(myFirstComm;)643 b(/*)23 b(inter-communicators)f(*/)170 1462 y(MPI_Comm)47 b(mySecondComm;)170 1518 y(make_id)23 b(firstMakeID,)g(secondMakeID;) 332 b(/*)23 b(handles)g(for)h("FINISH")f(*/)170 1631 y(MPI_INIT\(\);)170 1687 y(...)170 1800 y(/*)h(User)f(builds)g (intra-communicator)f(myComm)h(describing)f(the)i(local)f(sub-group)g (*/)170 1857 y(/*)h(using)f(any)h(appropriate)e(MPI)h(routine\(s\).)47 b(\(For)23 b(example,)g(myComm)g(could)g(*/)170 1913 y(/*)h(have)f(been)h(passed)f(in)g(as)h(an)f(argument)g(to)h(a)g(user)f (subroutine.\))f(*/)170 1970 y(myComm)h(=)h(...)g(;)170 2083 y(/*)g(Build)f(inter-communicators.)45 b(Group)23 b(membership)g(conditions)f(must)i(be)f(*/)170 2139 y(/*)h(provided)f (by)g(the)h(user.)f(*/)170 2195 y(if)h(\(\)) 266 2252 y({)525 b(/*)23 b(Group)h(0)f(communicates)f(with)i(groups)f (1)h(and)f(2.)h(*/)266 2308 y(MPI_COMM_NAME_MAKE_START\()o(myComm,)c ("Connect)j(20",)g(1,)h(&firstMakeID\);)266 2365 y (MPI_COMM_NAME_MAKE_START\()o(myComm,)c("Connect)j(10",)g(1,)h (&secondMakeID\);)266 2421 y(})170 2478 y(else)g(if)f(\(\))266 2534 y({)525 b(/*)23 b(Group)h(1)f(communicates)f (with)i(groups)f(0)h(and)f(2.)h(*/)266 2591 y (MPI_COMM_NAME_MAKE_START\()o(myComm,)c("Connect)j(10",)g(1,)h (&firstMakeID\);)266 2647 y(MPI_COMM_NAME_MAKE_START\()o(myComm,)c ("Connect)j(21",)g(1,)h(&secondMakeID\);)266 2704 y(})1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 47 41 46 bop 75 -100 a Fx(3.10.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b FE(41)170 45 y Fp(else)24 b(if)f(\(\))266 102 y({)525 b(/*)23 b(Group)h(2)f(communicates)f (with)i(groups)f(0)h(and)f(1.)h(*/)266 158 y (MPI_COMM_NAME_MAKE_START\()o(myComm,)c("Connect)j(21",)g(1,)h (&firstMakeID\);)266 214 y(MPI_COMM_NAME_MAKE_START\()o(myComm,)c ("Connect)j(20",)g(1,)h(&secondMakeID\);)266 271 y(})170 384 y(/*)g(Everyone)f(has)g(the)h(same)f("FINISH")g(code...)g(*/)170 440 y(MPI_COMM_NAME_MAKE_FINISH\(fir)o(stMakeID)o(,)e(&myFirstComm\);) 170 497 y(MPI_COMM_NAME_MAKE_FINISH\(sec)o(ondMakeI)o(D,)g (&mySecondComm\);)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 42 48 42 47 bop 75 369 a Fz(Section)35 b(4)75 590 y FG(P)m(oin)m(t)41 b(to)f(P)m(oin)m(t)h(Comm)m(unication)75 809 y Fw(1)75 1043 y Fy(4.1)59 b(Intro)r(duction)75 1170 y FE(This)17 b(c)o(hapter)e(is)i(a)f(draft)f(of)h(the)g(curren)o(t)g(prop)q(osal)g (for)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unication.)23 b(Sending)75 1226 y(and)c(receiving)h(of)e(messages)g(b)o(y)g(pro)q (cesses)h(is)g(the)g(basic)g(MPI)g(comm)o(unication)g(mec)o(hanism.)31 b(All)75 1283 y(other)20 b(comm)o(unication)h(functions)f(can)h(b)q(e)g (implemen)o(ted)g(on)f(top)g(of)g(this)g(basic)h(comm)o(unication)75 1339 y(la)o(y)o(er)15 b(\(although)g(a)g(more)g(direct)h(implemen)o (tation)g(ma)o(y)f(lead)h(to)e(greater)g(e\016ciency\).)166 1409 y(The)19 b(basic)g(p)q(oin)o(t)g(to)f(p)q(oin)o(t)h(comm)o (unication)g(op)q(erations)g(are)f FF(send)g FE(and)h FF(receiv)o(e)p FE(.)29 b(A)18 b FF(send)75 1465 y FE(op)q(eration)13 b(creates)f(and)h(sends)g(a)g(message.)18 b(The)13 b(op)q(eration)g(sp) q(eci\014es)i(a)d FF(send)i(bu\013er)e FE(in)i(the)e(sender)75 1522 y(memory)f(from)g(whic)o(h)h(the)g(message)f(data)g(is)h(tak)o (en.)18 b(In)12 b(addition,)h(the)f(send)g(op)q(eration)g(asso)q (ciates)f(an)75 1578 y FF(en)o(v)o(elop)q(e)16 b FE(with)g(the)g (message.)22 b(This)16 b(en)o(v)o(elop)q(e)h(sp)q(eci\014es)h(the)e (message)g(destination)h(and)f(con)o(tains)75 1634 y(distinguishing)h (information)d(that)f(can)i(b)q(e)g(used)f(b)o(y)g(the)g FF(receiv)o(e)g FE(op)q(eration)g(to)g(select)h(a)e(particular)75 1691 y(message.)166 1760 y(A)22 b FF(receiv)o(e)f FE(op)q(eration)h (consumes)g(a)f(message.)39 b(The)22 b(message)f(to)g(b)q(e)i(receiv)o (ed)f(is)h(selected)75 1817 y(according)13 b(to)f(the)g(v)m(alue)i(on)e (its)g(en)o(v)o(elop)q(e,)i(and)f(the)f(message)g(data)g(is)h(put)f(in) o(to)g(the)h FF(receiv)o(e)g(bu\013er)p FE(.)166 1886 y(The)f(next)g(sections)h(describ)q(e)g(the)g(basic)f(\(blo)q(c)o (king\))h(send)g(and)f(receiv)o(e)h(op)q(erations.)19 b(W)l(e)12 b(discuss)75 1943 y(send,)21 b(receiv)o(e,)g(basic)f(comm)o (unication)g(seman)o(tics,)h(t)o(yp)q(e)e(matc)o(hing)h(requiremen)o (ts,)h(t)o(yp)q(e)e(con)o(v)o(er-)75 1999 y(sion)14 b(in)h (heterogeneous)f(en)o(vironmen)o(ts,)f(and)h(more)g(general)g(comm)o (unication)g(mo)q(des.)20 b(Non)o(blo)q(c)o(king)75 2056 y(comm)o(unication)c(is)h(addressed)f(next,)f(follo)o(w)o(ed)h(b)o(y)g (c)o(hannel-lik)o(e)i(constructs)e(and)f(send-receiv)o(e)j(op-)75 2112 y(erations.)i(W)l(e)15 b(then)g(consider)h(general)f(datat)o(yp)q (es)f(that)g(allo)o(w)h(one)g(to)g(transfer)f(heterogeneous)h(and)75 2169 y(noncon)o(tiguous)f(data,)f(and)h(conclude)h(with)f(a)g (description)h(of)e(an)h(implemen)o(tation)h(of)e(MPI)g(p)q(oin)o(t)i (to)75 2225 y(p)q(oin)o(t)h(comm)o(unication)g(using)g(a)e(small)i(n)o (um)o(b)q(er)g(of)f(primitiv)o(es.)75 2443 y Fy(4.2)59 b(Basic)19 b(send)h(op)r(eration)75 2569 y FE(The)15 b(syn)o(tax)g(of)g(the)g(simplest)h(send)g(op)q(eration)f(is)h(giv)o (en)g(b)q(elo)o(w.)p 75 2661 720 2 v 127 2688 a FC(1)144 2704 y Fs(V)m(ersion)e(of)f(August)g(9,)g(1993)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 43 49 43 48 bop 75 -100 a Fx(4.2.)34 b(BASIC)16 b(SEND)f(OPERA)l(TION)1075 b FE(43)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(SEND)15 b(\(sta)o(rt,)g (count,)h(datat)o(yp)q(e,)h(dest,)f(tag,)f(comm\))117 136 y Fu(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 238 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)g(in)g(send)h(bu\013er)h (\(nonnegativ)o(e)e(in)o(te-)905 294 y(ger\))117 396 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 498 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o (teger\))117 600 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 702 y(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(\(handle\))75 968 y Fq(4.2.1)49 b(Message)16 b(data)75 1080 y FE(The)i(send)h(bu\013er)f(sp)q (eci\014ed)i(b)o(y)d(the)h Fo(MPI)p 828 1080 V 16 w(SEND)h FE(op)q(eration)f(consists)g(of)f Fo(count)i FE(successiv)o(e)h(en)o (tries)75 1136 y(of)e(the)g(t)o(yp)q(e)g(indicated)h(b)o(y)f Fo(datat)o(yp)q(e)p FE(,)i(starting)e(with)g(the)g(en)o(try)f(at)h (address)g Fo(sta)o(rt)p FE(.)28 b(Note)18 b(that)f(w)o(e)75 1193 y(sp)q(ecify)d(the)e(message)g(length)h(in)g(terms)e(of)h(n)o(um)o (b)q(er)h(of)e Ft(elements)p FE(,)h(not)f(n)o(um)o(b)q(er)i(of)f Ft(bytes)p FE(.)18 b(The)13 b(former)75 1249 y(is)j(mac)o(hine)g(indep) q(enden)o(t)h(and)f(closer)f(to)g(the)g(application)i(lev)o(el.)166 1319 y(The)g(data)f(part)g(of)g(the)g(message)h(consists)f(of)h(a)f (sequence)i(of)e Fo(count)i FE(v)m(alues,)f(eac)o(h)g(of)f(the)h(t)o (yp)q(e)75 1375 y(indicated)i(b)o(y)e Fo(datat)o(yp)q(e)p FE(.)52 b Fo(count)18 b FE(ma)o(y)e(b)q(e)i(zero,)e(in)i(whic)o(h)g (case)f(the)g(data)f(part)g(of)h(the)g(message)f(is)75 1432 y(empt)o(y)l(.)30 b(The)18 b(basic)h(datat)o(yp)q(es)f(that)g(can) h(b)q(e)g(sp)q(eci\014ed)h(for)e(message)g(data)g(v)m(alues)i(corresp)q (ond)f(to)75 1488 y(the)13 b(basic)h(datat)o(yp)q(es)f(of)g(the)g(host) g(language.)19 b(The)14 b(p)q(ossible`)g(v)m(alues)h(of)d(this)i (parameter)f(for)f(F)l(ortran)75 1545 y(and)j(the)h(corresp)q(onding)g (F)l(ortran)e(t)o(yp)q(es)h(are)g(listed)h(b)q(elo)o(w)p 473 1788 1004 2 v 472 1844 2 57 v 498 1827 a(MPI)f(datat)o(yp)q(e)p 1044 1844 V 292 w(F)l(ortran)f(datat)o(yp)q(e)p 1476 1844 V 473 1846 1004 2 v 472 1902 2 57 v 513 1885 a Fo(MPI)p 598 1885 14 2 v 16 w(INTEGER)p 1044 1902 2 57 v 270 w Fp(INTEGER)p 1476 1902 V 472 1959 V 513 1942 a Fo(MPI)p 598 1942 14 2 v 16 w(REAL)p 1044 1959 2 57 v 348 w Fp(REAL)p 1476 1959 V 472 2015 V 513 1998 a Fo(MPI)p 598 1998 14 2 v 16 w(DOUBLE)p 1044 2015 2 57 v 280 w Fp(DOUBLE)23 b(PRECISION)p 1476 2015 V 472 2071 V 513 2055 a Fo(MPI)p 598 2055 14 2 v 16 w(COMPLEX)p 1044 2071 2 57 v 246 w Fp(COMPLEX)p 1476 2071 V 472 2128 V 513 2111 a Fo(MPI)p 598 2111 14 2 v 16 w(DOUBLE)p 793 2111 V 17 w(COMPLEX)p 1044 2128 2 57 v 50 w Fp(DOUBLE)g(COMPLEX)p 1476 2128 V 472 2184 V 513 2167 a Fo(MPI)p 598 2167 14 2 v 16 w(LOGICAL)p 1044 2184 2 57 v 274 w Fp(LOGICAL)p 1476 2184 V 472 2241 V 513 2224 a Fo(MPI)p 598 2224 14 2 v 16 w(CHARA)o(CTER)p 1044 2241 2 57 v 194 w Fp(CHARACTER)p 1476 2241 V 472 2297 V 513 2280 a Fo(MPI)p 598 2280 14 2 v 16 w(BYTE)p 1044 2297 2 57 v 1476 2297 V 473 2299 1004 2 v 75 2549 a FE(The)13 b(p)q(ossible)i(v)m(alues)g(for)d(this)i(parameter)e(for)h (C)f(and)i(the)f(corresp)q(onding)h(C)f(t)o(yp)q(es)g(are)g(listed)h(b) q(elo)o(w)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 44 50 44 49 bop 75 -100 a FE(44)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)p 645 5 660 2 v 644 62 2 57 v 670 45 a FE(MPI)f(datat)o(yp)q(e)p 1031 62 V 107 w(C)g(datat)o(yp)q(e)p 1304 62 V 645 63 660 2 v 644 120 2 57 v 685 103 a Fo(MPI)p 770 103 14 2 v 16 w(SHORT)p 1031 120 2 57 v 124 w Fp(short)p 1304 120 V 644 176 V 685 159 a Fo(MPI)p 770 159 14 2 v 16 w(INT)p 1031 176 2 57 v 198 w Fp(int)p 1304 176 V 644 233 V 685 216 a Fo(MPI)p 770 216 14 2 v 16 w(LONG)p 1031 233 2 57 v 154 w Fp(long)p 1304 233 V 644 289 V 685 272 a Fo(MPI)p 770 272 14 2 v 16 w(UNSIGNED)p 1031 289 2 57 v 51 w Fp(unsigned)p 1304 289 V 644 346 V 685 329 a Fo(MPI)p 770 329 14 2 v 16 w(FLO)o(A)l(T)p 1031 346 2 57 v 134 w Fp(float)p 1304 346 V 644 402 V 685 385 a Fo(MPI)p 770 385 14 2 v 16 w(DOUBLE)p 1031 402 2 57 v 95 w Fp(double)p 1304 402 V 644 459 V 685 442 a Fo(MPI)p 770 442 14 2 v 16 w(CHAR)p 1031 459 2 57 v 154 w Fp(char)p 1304 459 V 644 515 V 685 498 a Fo(MPI)p 770 498 14 2 v 16 w(BYTE)p 1031 515 2 57 v 1304 515 V 645 517 660 2 v 166 695 a FE(The)g(datat)o(yp)q(e)28 b Fo(MPI)p 548 695 14 2 v 16 w(BYTE)15 b FE(do)q(es)g(not)f(corresp)q(ond)h(to)e(a)i (F)l(ortran)e(or)h(C)g(datat)o(yp)q(e.)19 b(A)c(v)m(alue)g(of)75 752 y(t)o(yp)q(e)33 b Fo(MPI)p 280 752 V 16 w(BYTE)17 b FE(consists)g(of)f(8)g(binary)i(digits.)24 b(A)17 b(b)o(yte)f(is)h (unin)o(terpreted)h(and)f(is)g(di\013eren)o(t)g(from)75 808 y(a)g(c)o(haracter.)25 b(Di\013eren)o(t)17 b(mac)o(hines)g(ma)o(y)g (ha)o(v)o(e)g(di\013eren)o(t)g(represen)o(tations)g(for)g(c)o (haracters,)f(or)h(ma)o(y)75 865 y(use)g(more)g(than)g(one)g(b)o(yte)g (to)g(represen)o(t)g(c)o(haracters.)25 b(On)18 b(the)f(other)g(hand,)g (a)g(b)o(yte)g(has)g(the)g(same)75 921 y(binary)f(v)m(alue)g(on)f(all)h (mac)o(hines.)166 1056 y Fv(Discussion:)166 1108 y Fu(Need)f(to)f (decide)g(whether)i(w)o(e)e(w)o(an)o(t)f(\(for)h(C\))166 1159 y Fn(unsigned)20 b(char)p Fu(,)13 b Fn(unsigned)20 b(short)p Fu(,)12 b Fn(unsigned)20 b(long)p Fu(,)13 b Fn(long)21 b(double)166 1211 y(DOUBLE)f(COMPLEX)13 b Fu(ma)o(y)f(not)i(b)q(e)g(standard)g(F)m(ortran)g(77)166 1263 y Fr(MPI)p 243 1263 13 2 v 14 w(DOUBLE)f Fu(is)h(no)o(w)f(used)i (b)q(oth)f(for)g(F)m(ortran)f(and)h(C.)f(Should)h(w)o(e)g(use)h (di\013eren)o(t)f(names?)75 1479 y Fq(4.2.2)49 b(Message)16 b(envelop)q(e)75 1568 y FE(In)g(addition)g(to)e(the)h(data)g(part,)f (messages)g(con)o(tain)h(information)h(that)e(can)h(b)q(e)h(used)f(to)g (distinguish)75 1625 y(messages)h(and)g(selectiv)o(ely)i(receiv)o(e)f (them.)23 b(This)17 b(information)f(is)h(con)o(tained)g(in)g(a)f (\014xed)h(n)o(um)o(b)q(er)f(of)75 1681 y(\014xed-format)f(\014elds,)h (whic)o(h)g(w)o(e)f(collectiv)o(ely)i(call)g(the)e FF(message)i(en)o(v) o(elop)q(e)p FE(.)i(These)d(\014elds)g(are)75 1783 y FF(source)75 1884 y(destination)75 1986 y(tag)75 2088 y(con)o(text)166 2190 y FE(The)c(message)g(source)g(is)h(implicitly)i (determined)f(b)o(y)e(the)g(iden)o(tit)o(y)h(of)f(the)g(message)g (sender.)20 b(The)75 2246 y(other)15 b(\014elds)h(are)f(sp)q(eci\014ed) i(b)o(y)f(parameters)e(in)i(the)f(send)h(op)q(eration.)166 2304 y(The)j(in)o(teger-v)m(alued)h(message)e(tag)g(is)h(sp)q (eci\014ed)i(b)o(y)d(the)37 b Fo(tag)19 b FE(parameter.)29 b(This)19 b(in)o(teger)g(can)75 2361 y(b)q(e)e(used)g(b)o(y)g(the)f (program)f(to)h(distinguish)j(di\013eren)o(t)d(t)o(yp)q(es)h(of)f (messages.)23 b(The)17 b(range)f(of)g(v)m(alid)i(tag)75 2417 y(v)m(alues)e(is)g(implemen)o(tation)g(dep)q(enden)o(t)g(and)f (can)g(b)q(e)h(queried)g(using)g(the)f Fo(T)l(agRange)g FE(en)o(vironmen)o(tal)75 2474 y(inquiry)i(function,)e(as)g(describ)q (ed)i(in)f(Chapter)f(8.)166 2532 y(The)k(con)o(text)g(of)g(the)g (message)g(sen)o(t)g(is)h(sp)q(eci\014ed)h(b)o(y)f(the)38 b Fo(comm)18 b FE(parameter.)31 b(the)19 b(message)75 2589 y(carries)c(the)h(con)o(text)e(asso)q(ciated)i(with)f(this)h(comm) o(unicator)f(\(see)g(Chapter)g(3\).)166 2647 y(The)d(message)f (destination)i(is)f(sp)q(eci\014ed)i(b)o(y)e(the)23 b Fo(dest)14 b FE(parameter)d(as)g(a)g(rank)h(within)h(the)f(pro)q(cess) 75 2704 y(group)18 b(asso)q(ciated)h(with)g(that)f(same)h(comm)o (unicator.)30 b(The)19 b(range)f(of)h(v)m(alid)h(v)m(alues)g(is)f Fp(0,)24 b(...)47 b(,)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 45 51 45 50 bop 75 -100 a Fx(4.3.)34 b(BASIC)16 b(RECEIVE)g(OPERA)l(TION)990 b FE(45)75 45 y Fp(n-1)p FE(,)18 b(where)g Fp(n)g FE(is)h(the)f(n)o(um) o(b)q(er)g(of)g(pro)q(cesses)g(in)h(this)g(group.)28 b(Th)o(us,)19 b(p)q(oin)o(t-to-p)q(oin)o(t)f(comm)o(unica-)75 102 y(tions)e(do)g(not)f(use)i(absolute)f(addresses,)g(but)g(only)g (relativ)o(e)h(ranks)e(within)j(a)d(group.)22 b(This)16 b(pro)o(vides)75 158 y(imp)q(ortan)o(t)f(mo)q(dularit)o(y)l(.)166 215 y(The)j(message)f(en)o(v)o(elop)q(e)i(w)o(ould)f(normally)g(b)q(e)h (enco)q(ded)g(b)o(y)e(a)g(\014xed-length)j(message)d(header.)75 272 y(Ho)o(w)o(ev)o(er,)c(the)i(actual)f(mec)o(hanism)h(used)g(to)f (asso)q(ciate)g(an)g(en)o(v)o(elop)q(e)i(with)e(a)g(message)g(is)h (implemen-)75 328 y(tation)d(dep)q(enden)o(t:)20 b(some)12 b(of)g(the)g(information)g(\(e.g.,)g(source)g(or)g(destination\))g(ma)o (y)g(b)q(e)h(implicit,)i(and)75 385 y(need)f(not)g(b)q(e)g(explicitly)i (carried)e(b)o(y)g(messages;)f(pro)q(cesses)h(ma)o(y)f(b)q(e)h(iden)o (ti\014ed)i(b)o(y)d(relativ)o(e)h(ranks,)f(or)75 441 y(absolute)j(ids;)f(etc.)75 589 y Fy(4.3)59 b(Basic)19 b(receive)g(op)r(eration)75 692 y FE(The)c(syn)o(tax)g(of)g(the)g (simplest)h(receiv)o(e)g(op)q(eration)g(is)f(giv)o(en)h(b)q(elo)o(w.)75 796 y Fo(MPI)p 160 796 14 2 v 16 w(RECV)g(\(sta)o(rt,)f(count,)h(datat) o(yp)q(e,)g(source,)f(tag,)g(comm,)e(status\))117 874 y Fu(OUT)108 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 951 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1027 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(state\))117 1104 y(IN)155 b Fo(source)468 b Fu(rank)14 b(of)f(source)j(\(in)o(teger\))117 1181 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 1257 y(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(\(handle\))117 1334 y(OUT)108 b Fo(status)476 b Fu(status)15 b(ob)r(ject)166 1459 y FE(The)f(receiv)o(e)h(bu\013er)f(consists)g(of)f(the)h(storage)f (con)o(taining)29 b Fo(count)15 b FE(consecutiv)o(e)g(elemen)o(ts)f(of) g(the)75 1516 y(t)o(yp)q(e)j(sp)q(eci\014ed)i(b)o(y)33 b Fo(datat)o(yp)q(e)p FE(,)18 b(starting)e(at)g(address)34 b Fo(sta)o(rt)p FE(.)24 b(The)17 b(length)h(of)e(the)h(receiv)o(ed)g (message)75 1572 y(m)o(ust)e(b)q(e)i(less)g(then)f(or)f(equal)i(to)e (the)h(length)h(of)f(the)g(receiv)o(e)g(bu\013er.)23 b(I.e.,)15 b(all)i(incoming)g(data)f(m)o(ust)75 1629 y(\014t,)23 b(without)e(truncation,)i(in)o(to)e(the)h(receiv)o(e)g (bu\013er.)39 b(The)22 b Fo(MPI)p 1262 1629 V 16 w(PROBE)g FE(function)h(describ)q(ed)g(in)75 1685 y(Section)16 b(4.9)e(can)i(b)q(e)g(used)f(to)g(receiv)o(e)h(messages)f(of)f(unkno)o (wn)i(length.)166 1742 y(The)j(selection)h(of)f(a)f(message)h(b)o(y)f (a)h(receiv)o(e)h(op)q(eration)f(is)g(done)g(uniquely)i(according)f(to) e(the)75 1799 y(v)m(alue)i(of)f(the)g(message)g(en)o(v)o(elop)q(e.)32 b(The)19 b(receiv)o(e)h(op)q(eration)g(sp)q(eci\014es)h(an)d FF(en)o(v)o(elop)q(e)k(pattern)p FE(;)f(a)75 1855 y(message)13 b(can)g(b)q(e)g(receiv)o(ed)h(b)o(y)f(that)g(receiv)o(e)h(op)q(eration) f(only)g(if)h(its)f(en)o(v)o(elop)q(e)h(matc)o(hes)f(that)f(pattern.)75 1912 y(A)k(pattern)g(sp)q(eci\014es)h(v)m(alues)h(for)d(the)32 b Fo(source)p FE(,)g Fo(tag)16 b FE(and)32 b Fo(context)18 b FE(\014elds)f(of)f(the)g(message)f(en)o(v)o(elop)q(e.)75 1968 y(The)g(receiv)o(er)h(ma)o(y)e(sp)q(ecify)i(a)e(wildcard)j Fr(MPI)p 882 1968 13 2 v 14 w(ANY)p 981 1968 V 15 w(SOURCE)c FE(v)m(alue)j(for)29 b Fo(source)p FE(,)15 b(and/or)f(a)h(wildcard)75 2025 y Fr(MPI)p 152 2025 V 14 w(ANY)p 251 2025 V 15 w(T)m(A)o(G)21 b FE(v)m(alue)j(for)44 b Fo(tag)p FE(,)24 b(indicating)g(that)e(an)o(y) g(source)h(and/or)f(tag)f(are)h(acceptable.)43 b(It)75 2081 y(cannot)15 b(sp)q(ecify)i(a)f(wildcard)h(v)m(alue)f(for)31 b Fo(context)p FE(.)23 b(Th)o(us,)15 b(a)g(message)g(can)h(b)q(e)g (receiv)o(ed)h(b)o(y)f(a)f(receiv)o(e)75 2138 y(op)q(eration)23 b(only)f(if)h(it)g(is)g(addressed)f(to)g(the)h(receiving)g(pro)q(cess,) h(has)f(a)f(matc)o(hing)g(con)o(text,)h(has)75 2194 y(matc)o(hing)18 b(source)g(unless)h(source=)p Fr(MPI)p 791 2194 V 15 w(ANY)p 891 2194 V 14 w(SOURCE)d FE(in)j(the)f(pattern,)g(and)g(has)f (a)h(matc)o(hing)g(tag)75 2250 y(unless)e(tag=)e Fr(MPI)p 399 2250 V 14 w(ANY)p 498 2250 V 15 w(T)m(A)o(G)g FE(in)i(the)f (pattern.)166 2308 y(The)22 b(message)f(tag)g(is)h(sp)q(eci\014ed)i(b)o (y)e(the)43 b Fo(tag)22 b FE(parameter)f(of)g(the)h(receiv)o(e)h(op)q (eration.)40 b(The)75 2364 y(message)26 b Fo(context)15 b FE(is)f(the)g(con)o(text)f(attac)o(hed)g(with)g(the)h(comm)o (unicator)f(sp)q(eci\014ed)j(b)o(y)d(the)h(parameter)75 2421 y Fo(comm)p FE(.)j(The)12 b(message)g(source,)g(if)h(di\013eren)o (t)f(from)22 b Fr(MPI)p 1040 2421 V 14 w(ANY)p 1139 2421 V 15 w(SOURCE)p FE(,)10 b(is)i(sp)q(eci\014ed)j(as)c(a)h(rank)g(within) 75 2477 y(the)h(pro)q(cess)h(group)f(asso)q(ciated)g(with)h(that)e (same)h(comm)o(unicator.)19 b(Th)o(us,)13 b(the)g(range)g(of)g(v)m (alid)i(v)m(alues)75 2533 y(for)20 b(the)43 b Fo(source)21 b FE(parameter)g(is)g FA(f)j Fp(0,)g(...)47 b(,)24 b(n-1)f FA(g)h([f)f Fp(MPI)p 1250 2533 15 2 v 17 w(ANY)p 1339 2533 V 17 w(SOURCE)g FA(g)p FE(,)f(where)f Fp(n)g FE(is)h(the)75 2590 y(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(in)g(this)g(group.)166 2647 y(Note)i(the)i(asymmetry)d(b)q(et)o(w)o(een)j(send)f(and)g(receiv) o(e)h(op)q(erations:)28 b(A)19 b(receiv)o(e)h(op)q(eration)f(ma)o(y)75 2704 y(accept)13 b(messages)f(from)g(an)h(arbitrary)f(sender;)i(on)f (the)g(other)f(hand,)i(a)e(send)h(op)q(eration)g(m)o(ust)g(sp)q(ecify) -32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 46 52 46 51 bop 75 -100 a FE(46)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(a)f(unique)j (receiv)o(er.)k(This)16 b(matc)o(hes)f(a)h(\\push")f(comm)o(unication)i (mec)o(hanism,)f(where)g(data)f(transfer)75 102 y(is)i(e\013ected)h(b)o (y)f(the)g(sender)g(\(rather)f(than)h(a)g(\\pull")h(mec)o(hanism,)g (where)f(data)f(transfer)g(is)i(e\013ected)75 158 y(b)o(y)d(the)g (receiv)o(er\).)166 214 y(Source)h(=)f(destination)h(is)g(allo)o(w)o (ed:)k(a)15 b(pro)q(cess)g(can)h(send)g(a)e(message)h(to)g(itself.)75 335 y Fq(4.3.1)49 b(Return)16 b(status)75 504 y Fv(Discussion:)f(This)g (is)g(a)h(c)o(hange)f(from)g(previous)f(draft)166 643 y FE(The)f(source)g(or)f(the)h(tag)f(of)h(a)f(receiv)o(ed)i(message)f (ma)o(y)f(not)g(b)q(e)i(kno)o(wn)e(if)i(wildcard)g(v)m(alues)g(where)75 699 y(used)j(in)g(the)g(receiv)o(e)g(op)q(eration.)24 b(Also,)16 b(the)h(actual)f(length)i(of)e(the)g(message)g(receiv)o(ed)i (ma)o(y)d(not)h(b)q(e)75 756 y(kno)o(wn.)22 b(Th)o(us,)15 b(this)i(information)f(needs)g(to)g(b)q(e)g(returned)g(b)o(y)g(the)g (receiv)o(e)h(op)q(eration.)22 b(F)l(or)15 b(reasons)75 812 y(that)i(will)j(b)q(ecome)f(clear)f(later,)g(it)g(is)h(preferable)g (to)e(return)h(this)g(information)h(in)f(a)g(separate)35 b Fo(out)75 869 y FE(v)m(ariable,)23 b(rather)d(then)i(using)f(the)42 b Fo(count)p FE(,)j Fo(source)21 b FE(and)42 b Fo(tag)21 b FE(parameters)f(b)q(oth)h(for)f(input)i(and)75 925 y(output.)j(The)17 b(information)g(is)h(returned)f(b)o(y)g(the)34 b Fo(status)19 b FE(parameter)d(of)h(the)g Fo(MPI)p 1554 925 14 2 v 16 w(RECV)g FE(function.)75 981 y(This)h(is)f(a)g(parameter) f(of)g(a)h(sp)q(ecial)h(MPI-de\014ned)h(t)o(yp)q(e.)25 b(The)17 b(status)f(v)m(ariable)j(can)e(b)q(e)g(\\deco)q(ded")75 1038 y(to)e(retriev)o(e)g(the)30 b Fo(count)p FE(,)i Fo(source)15 b FE(and)31 b Fo(tag)15 b FE(\014elds,)h(using)g(the)f (query)h(functions)g(listed)g(b)q(elo)o(w.)75 1142 y Fo(MPI)p 160 1142 V 16 w(GET)p 264 1142 V 17 w(SOURCE\(status,)i (source\))117 1219 y Fu(IN)155 b Fo(status)476 b Fu(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 1293 y(OUT)108 b Fo(source)468 b Fu(source)15 b(rank)f(\(in)o(teger\)) 166 1417 y FE(Returns)e(the)f(rank)h(of)f(the)g(message)g(source)h (\(in)g(the)f(group)g(asso)q(ciated)h(with)g(the)f(comm)o(unicator)75 1474 y(that)j(w)o(as)h(used)h(to)e(receiv)o(e\).)75 1577 y Fo(MPI)p 160 1577 V 16 w(GET)p 264 1577 V 17 w(T)l(A)o(G\(status,)j (tag\))117 1655 y Fu(IN)155 b Fo(status)476 b Fu(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 1729 y(OUT)108 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))166 1853 y FE(Returns)j(the)f(tag)f(of)h(receiv)o(ed)h(message.)75 1957 y Fo(MPI)p 160 1957 V 16 w(GET)p 264 1957 V 17 w(COUNT\(status,)h (count\))117 2034 y Fu(IN)155 b Fo(status)476 b Fu(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 2108 y(OUT)108 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(elemen)o(ts)f(\(in)o(teger\))166 2232 y FE(Returns)h(the)f(n)o(um)o(b)q(er)h(of)f(elemen)o(ts)h(receiv)o (ed.)21 b(\(Here,)14 b(again,)h(w)o(e)f(coun)o(t)g Ft(elements)p FE(,)f(not)h Ft(bytes)p FE(.\))166 2289 y(The)20 b(information)f (returned)h(b)o(y)g(these)f(query)h(functions)g(is)g(the)g(information) g(last)f(stored)g(in)75 2345 y(the)28 b Fo(status)16 b FE(v)m(ariable)f(b)o(y)f(a)g(receiv)o(e)h(function.)20 b(It)14 b(is)g(erroneous)g(to)f(call)i(these)g(query)f(functions)g(if)h (the)75 2402 y Fo(status)i FE(v)m(ariable)g(w)o(as)d(nev)o(er)h(set)g (b)o(y)h(a)e(receiv)o(e.)166 2458 y(Note)h(that)g(it)h(is)g(not)f (mandatory)g(to)g(query)g(the)h(return)g(status)e(after)h(a)g(receiv)o (e.)22 b(The)16 b(receiv)o(er)75 2515 y(will)h(use)e(these)h(calls,)g (or)e(some)h(of)g(them,)g(only)h(when)f(the)h(information)f(they)g (return)g(is)h(needed.)166 2571 y(information)f(returned)h(in)g(it)f(b) o(y)g(the)h(last)f(receiv)o(e)h(op)q(eration)f(that)g(up)q(dated)166 2704 y Fv(Discussion:)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 47 53 47 52 bop 75 -100 a Fx(4.4.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b FE(47)166 45 y Fu(The)16 b(use)g(of)e(a)h (separate)i(status)f(parameter)e(prev)o(en)o(ts)j(errors)f(that)g(are)f (often)h(attac)o(hed)f(with)30 b(INOUT)75 95 y(parameters)14 b(\(e.g.,)f(passing)g(the)28 b Fr(MPI)p 701 95 13 2 v 15 w(ANY)p 801 95 V 14 w(T)m(A)o(G)13 b Fu(constan)o(t)h(as)g(the)h (actual)e(argumen)o(t)g(for)27 b Fr(tag)p Fu(\).)18 b(The)c(use)h(of)75 145 y(an)h(explicit)f(user)i(ob)r(ject)g(b)q(ecomes)f(imp)q(ortan)o(t)f (for)g(non)o(blo)q(c)o(king)g(comm)o(unication)e(as)j(it)f(allo)o(ws)g (the)i(receiv)o(e)75 195 y(op)q(eration)c(to)g(b)q(e)h(stateless)h (and,)d(hence,)i(reen)o(tran)o(t.)20 b(This)13 b(prev)o(en)o(ts)h (confusions)f(in)g(the)h(case)g(where)g(m)o(ultiple)75 244 y(receiv)o(es)i(can)e(b)q(e)g(p)q(osted)h(b)o(y)f(a)f(pro)q(cess.) 166 461 y Fv(Implemen)o(tati)o(on)f(note:)166 518 y Fu(One)j(exp)q (ects)g(deco)q(de)h(functions)e(to)f(b)q(e)i(in-lined)e(in)g(man)o(y)f (implemen)o(tations.)166 658 y FE(All)20 b(send)g(and)f(receiv)o(e)h (op)q(erations)f(use)g(the)38 b Fo(sta)o(rt)p FE(,)i Fo(count)p FE(,)21 b Fo(datat)o(yp)q(e)p FE(,)41 b Fo(source)p FE(,)f Fo(dest)p FE(,)g Fo(tag)p FE(,)75 715 y Fo(comm)17 b FE(and)37 b Fo(status)21 b FE(parameters)c(in)j(the)f(same)f(w)o(a)o (y)f(as)i(the)f(basic)38 b Fo(MPI)p 1402 715 14 2 v 16 w(SEND)19 b FE(and)37 b Fo(MPI)p 1746 715 V 16 w(RECV)75 771 y FE(op)q(erations)15 b(describ)q(ed)i(in)g(this)e(section.)75 921 y Fy(4.4)59 b(Semantics)18 b(of)i(p)r(oint-to-p)r(oint)e (communication)75 1025 y FE(A)g(v)m(alid)h(MPI)e(implemen)o(tation)i (guaran)o(tees)e(certain)h(general)g(prop)q(erties)g(of)f(p)q(oin)o (t-to-p)q(oin)o(t)h(com-)75 1081 y(m)o(unication,)e(whic)o(h)g(are)f (describ)q(ed)i(in)f(this)f(section.)75 1208 y Fo(Order)46 b FE(Messages)14 b(are)h Ft(non-overtaking)p FE(,)g(within)h(eac)o(h)f (con)o(text:)20 b(if)c(t)o(w)o(o)d(messages)i(are)g(successiv)o(ely)75 1264 y(sen)o(t)10 b(from)g(the)g(same)g(source,)h(to)e(the)i(same)f (destination,)h(with)g(the)f(same)g(con)o(text,)h(then)f(the)g (messages)75 1320 y(are)17 b(receiv)o(ed)i(in)f(the)f(order)g(they)h(w) o(ere)f(sen)o(t.)26 b(I.e.,)17 b(if)h(the)f(receiv)o(er)i(p)q(osts)e(t) o(w)o(o)f(successiv)o(e)i(receiv)o(es)75 1377 y(that)g(b)q(oth)g(matc)o (h)g(either)i(message,)e(then)h(the)f(\014rst)h(receiv)o(e)g(will)h (receiv)o(e)g(the)e(\014rst)g(message,)h(and)75 1433 y(the)e(second)h(receiv)o(e)h(will)g(receiv)o(e)f(the)f(second)h (message.)26 b(This)18 b(requiremen)o(t)g(facilitates)g(matc)o(hing)75 1490 y(of)e(sends)h(to)f(receiv)o(es.)25 b(It)16 b(guaran)o(tees)g (that)g(message-passing)g(co)q(de)h(is)g(deterministic,)i(if)e(pro)q (cesses)75 1546 y(are)e(single-threaded)i(and)e(wildcard)31 b Fr(MPI)p 831 1546 13 2 v 14 w(ANY)p 930 1546 V 15 w(SOURCE)13 b FE(is)j(not)f(used)g(in)h(receiv)o(es.)166 1604 y(If)h(a)g(pro)q (cess)h(has)f(a)g(single)h(thread)f(of)g(execution,)i(then)e(an)o(y)g (t)o(w)o(o)f(comm)o(unications)i(executed)75 1660 y(b)o(y)e(this)h(pro) q(cess)g(are)f(ordered.)23 b(On)17 b(the)f(other)g(hand,)h(if)g(the)f (pro)q(cess)h(is)g(m)o(ulti-threaded,)g(then)g(the)75 1717 y(seman)o(tics)e(of)f(thread)h(execution)h(ma)o(y)e(not)g (de\014ne)i(a)e(relativ)o(e)i(order)e(b)q(et)o(w)o(een)h(t)o(w)o(o)e (send)j(op)q(erations)75 1773 y(executed)g(b)o(y)f(t)o(w)o(o)f (distinct)j(threads:)i(the)d(op)q(erations)f(are)g(logically)i (concurren)o(t,)e(ev)o(en)h(though)f(one)75 1830 y(ph)o(ysically)22 b(precedes)e(the)g(other.)33 b(In)20 b(suc)o(h)g(a)f(case,)i(the)e(t)o (w)o(o)g(messages)g(sen)o(t)g(can)h(b)q(e)g(receiv)o(ed)h(in)75 1886 y(an)o(y)f(order.)37 b(Similarly)l(,)24 b(if)d(t)o(w)o(o)e(receiv) o(e)j(op)q(erations)f(that)f(are)g(logically)j(concurren)o(t)e(receiv)o (e)g(t)o(w)o(o)75 1943 y(successiv)o(ely)i(sen)o(t)d(messages,)h(then)g (the)g(t)o(w)o(o)f(messages)g(can)h(matc)o(h)f(the)h(t)o(w)o(o)e (receiv)o(es)j(in)f(either)75 1999 y(order.)75 2125 y Fo(Progress)45 b FE(If)13 b(a)g(pair)g(of)f(matc)o(hing)h(send)h(and)f (receiv)o(es)g(ha)o(v)o(e)g(b)q(een)h(initiated)g(on)f(t)o(w)o(o)e(pro) q(cesses,)j(then)75 2182 y(at)k(least)h(one)h(of)e(these)h(t)o(w)o(o)f (op)q(erations)h(will)i(complete,)f(indep)q(enden)o(tly)i(of)c(other)h (actions)g(in)h(the)75 2238 y(system:)e(the)13 b(send)h(op)q(eration)f (will)h(complete,)g(unless)g(the)f(receiv)o(e)h(is)f(satis\014ed)h(b)o (y)e(another)h(message,)75 2295 y(and)h(completes;)g(the)g(receiv)o(e)h (op)q(eration)f(will)h(complete,)g(unless)g(the)f(message)f(sen)o(t)g (is)i(consumed)f(b)o(y)75 2351 y(another)h(matc)o(hing)g(receiv)o(e)h (that)f(w)o(as)f(p)q(osted)h(at)g(the)g(same)g(destination)h(pro)q (cess.)75 2478 y Fo(F)o(airness)46 b FE(MPI)15 b(mak)o(es)g(no)g (guaran)o(tee)g(of)g Ft(fairness)f FE(in)i(the)g(handling)h(of)e(comm)o (unication.)21 b(Supp)q(ose)75 2534 y(that)15 b(a)g(send)h(w)o(as)f(p)q (osted.)21 b(Then)15 b(it)h(is)g(p)q(ossible)h(that)e(the)h (destination)g(rep)q(eatedly)h(p)q(osts)e(a)g(receiv)o(e)75 2591 y(that)e(matc)o(hes)h(this)g(send,)h(y)o(et)e(the)h(message)g(is)g (nev)o(er)g(receiv)o(ed,)h(b)q(ecause)g(it)f(is)h(eac)o(h)f(time)g(o)o (v)o(ertak)o(en)75 2647 y(b)o(y)e(another)h(message,)f(sen)o(t)g(from)g (another)g(source.)19 b(Similarly)l(,)c(supp)q(ose)e(that)f(a)g(receiv) o(e)h(w)o(as)f(p)q(osted.)75 2704 y(Then)18 b(it)f(is)h(p)q(ossible)h (that)e(messages)g(that)f(matc)o(h)h(this)h(receiv)o(e)g(are)f(rep)q (eatedly)h(receiv)o(ed,)h(y)o(et)e(the)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 48 54 48 53 bop 75 -100 a FE(48)635 b Fx(SECTION)16 b(4.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(receiv)o(e)h(is)g (nev)o(er)g(satis\014ed,)f(b)q(ecause)i(it)e(is)h(o)o(v)o(ertak)o(en)e (b)o(y)h(other)g(receiv)o(es)h(p)q(osted)g(at)f(this)g(no)q(de)h(\(b)o (y)75 102 y(other)12 b(executing)h(threads\).)19 b(It)12 b(is)h(the)f(programmer's)e(resp)q(onsibili)q(t)o(y)k(to)e(prev)o(en)o (t)g(starv)m(ation)g(in)h(suc)o(h)75 158 y(situations.)166 291 y Fv(Discussion:)166 342 y(Need)i(to)h(b)q(e)f(discussed)f(b)o(y)h (MPIF)166 392 y Fu(Alternativ)o(e)f(p)q(ositions:)166 443 y(1.)j(Sends)e(are)f(handled)f(fairly;)e(e.g.)18 b(a)13 b(p)q(osted)h(send)h(cannot)e(b)q(e)h(rep)q(eatedly)h(o)o(v)o (ertak)o(en)e(b)o(y)g(other)h(sends.)75 492 y(An)h(example)g(of)f(an)h (implemen)o(tation)d(that)k(violates)e(this:)21 b(Assume)16 b(that)f(arriving)f(messages)i(are)g(k)o(ept)f(in)g(a)75 542 y(hash)i(table)g(organized)f(b)o(y)h(source,)h(and)f(that)g(a)f (receiv)o(e)i(with)f(source=don)o(tcare)i(is)d(handled)h(b)o(y)f(searc) o(hing)75 592 y(this)d(table)g(sequen)o(tially)m(.)j(Then)e(a)e (message)h(p)q(osted)g(at)g(the)g(end)h(of)e(this)h(table)g(ma)o(y)d(b) q(e)k(rep)q(eatedly)g(o)o(v)o(ertak)o(en)75 642 y(b)o(y)g(messages)g (that)g(get)g(p)q(osted)h(at)e(the)i(head)f(of)f(the)i(table.)j(\(Easy) c(\014x:)k(rotate)c(starting)g(p)q(oin)o(t)f(of)h(searc)o(h.\))166 692 y(2.)22 b(Receiv)o(es)16 b(are)g(handled)f(fairly;)f(e.g.)22 b(a)15 b(p)q(osted)h(receiv)o(e)g(cannot)g(b)q(e)f(rep)q(eatedly)i(o)o (v)o(ertak)o(en)e(b)o(y)g(other)75 742 y(receiv)o(es.)24 b(An)15 b(example)f(of)h(an)g(implemen)o(tatio)o(n)e(that)i(violates)g (this:)21 b(assume)15 b(that)g(there)h(are)g(t)o(w)o(o)f(di\013eren)o (t)75 792 y(p)q(osting)k(tables)g(for)g(receiv)o(es)h(with)f(a)g(sp)q (eci\014c)h(source)g(and)f(for)g(receiv)o(es)h(with)f(source=don)o (tcare;)k(assume)75 842 y(further)c(that)g(these)g(t)o(w)o(o)f (structures)j(are)e(searc)o(hed)h(in)e(a)g(\014xed)h(order)g(when)g(a)f (message)g(arriv)o(es.)31 b(Then)19 b(a)75 892 y(receiv)o(e)d(with)f (source=don)o(tcare)i(ma)o(y)c(b)q(e)i(rep)q(eatedly)h(o)o(v)o(ertak)o (en)f(b)o(y)f(receiv)o(es)j(with)d(source)i(sp)q(eci\014ed.)23 b(\(Fix:)75 941 y(alternate)15 b(searc)o(h)g(order;)g(but)f(this)h(has) f(p)q(erformance)g(problems,)f(if)h(w)o(e)g(assume)g(that)h(one)f(t)o (yp)q(e)h(of)f(receiv)o(e)h(is)75 991 y(m)o(uc)o(h)e(more)g(frequen)o (t)h(than)g(the)h(other)f(t)o(yp)q(e.\))166 1042 y(3.)k(Both)c(send)h (and)e(receiv)o(es)j(are)e(handled)g(fairly)m(.)166 1099 y(Opinions?)75 1305 y Fo(Resource)g(limitations)45 b FE(The)13 b(curren)o(t)f(practice)h(for)f(man)o(y)g(commercial)i (message-passing)e(libraries)i(is)75 1362 y(that)f(\(short\))f (messages)h(are)g(bu\013ered)g(b)o(y)h(the)f(system,)g(th)o(us)g(allo)o (wing)h(blo)q(c)o(king)h(send)f(op)q(erations)g(to)75 1418 y(complete)d(ahead)f(of)g(the)h(matc)o(hing)f(receiv)o(es.)19 b(It)11 b(is)f(exp)q(ected)i(that)e(man)o(y)f(MPI)i(implemen)o(tations) g(will)75 1475 y(follo)o(w)j(this)g(practice,)g(and)f(pro)o(vide)h(the) g(same)f(lev)o(el)i(of)e(bu\013ering)h(that)f(is)h(a)o(v)m(ailable)h (on)f(the)f(libraries)75 1531 y(they)21 b(replace.)37 b(Ho)o(w)o(ev)o(er,)20 b(message)g(bu\013ering)h(is)h(not)e(a)g(univ)o (ersal)i(practice.)36 b(Ev)o(en)21 b(on)f(systems)75 1588 y(where)e(bu\013ering)g(o)q(ccur,)g(the)f(amoun)o(t)g(of)g (bu\013er)h(space)f(a)o(v)m(ailable)j(and)d(the)h(w)o(a)o(y)e(it)i(is)g (allo)q(cated)g(is)75 1644 y(b)q(ound)e(to)f(b)q(e)h(implemen)o(tation) g(dep)q(enden)o(t.)166 1701 y(Therefore,)24 b(message)e(bu\013ering)h (is)g(not)f(mandated)g(b)o(y)g(MPI)h(and)f(is)h(seen)g(as)f(a)g(qualit) o(y)h(of)75 1758 y(implemen)o(tation)c(issue.)28 b(A)18 b(v)m(alid)h(MPI)f(implemen)o(tation)h(of)35 b Fo(MPI)p 1291 1758 14 2 v 16 w(SEND)18 b FE(is)g(to)f(blo)q(c)o(k)i(the)e (sender)75 1814 y(un)o(til)23 b(a)f(matc)o(hing)h(receiv)o(e)g(has)f(b) q(een)h(initiated.)43 b(In)23 b(general,)h(the)e(programmer)f(can)h (mak)o(e)g(no)75 1871 y(assumptions)f(on)g(the)g(a)o(v)m(ailabilit)o(y) j(of)c(bu\013er)h(space,)i(and)e(ho)o(w)f(this)i(space)f(is)h(allo)q (cated.)38 b(Th)o(us,)75 1927 y(p)q(ortable)14 b(\(safe\))e(MPI)h(co)q (de)h(should)g(w)o(ork)f(under)h(the)f(assumption)g(that)g(an)g (arbitrary)g(subset)g(of)g(the)75 1983 y(send)18 b(op)q(erations)f(are) g(going)g(to)g(return)g(b)q(efore)h(a)f(matc)o(hing)g(receiv)o(e)h(is)g (p)q(osted,)g(and)f(the)g(rest)g(will)75 2040 y(blo)q(c)o(k)f(un)o(til) g(a)f(matc)o(hing)g(receiv)o(e)i(is)e(p)q(osted.)166 2097 y(MPI)g(implemen)o(tation)i(will)h(pro)o(vide)e(information)g(on)f (the)h(amoun)o(t)f(of)g(a)o(v)m(ailable)i(bu\013er)f(space)75 2153 y(and)f(on)h(the)f(bu\013ering)h(p)q(olicy)g(via)g(the)f(en)o (vironmen)o(tal)h(enquiries)h(describ)q(ed)g(in)f(Chapter)f(8.)166 2211 y(Examples)h(\(in)o(v)o(olving)g(t)o(w)o(o)e(pro)q(cessors)h(with) g(ranks)g(0)g(and)g(1\))166 2268 y(The)g(follo)o(wing)h(program)e(is)i (safe,)f(and)g(should)h(alw)o(a)o(ys)f(succeed.)75 2421 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2478 y(IF)h(\(rank.EQ.0\))123 2534 y(THEN)170 2591 y(CALL)g (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 2647 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(ierr\))123 2704 y(ELSE)95 b(!)23 b(rank.EQ.1)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 49 55 49 54 bop 75 -100 a Fx(4.5.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) 1105 b FE(49)170 45 y Fp(CALL)24 b(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))170 102 y(CALL)h (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g (context\))75 158 y(END)g(IF)166 252 y FE(The)15 b(follo)o(wing)h (program)e(is)i(erroneous,)f(and)g(should)h(alw)o(a)o(ys)f(fail.)75 402 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 459 y(IF)h(\(rank.EQ.0\))123 515 y(THEN)170 572 y(CALL)g (MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 628 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(ierr\))123 685 y(ELSE)95 b(!)23 b(rank.EQ.1)170 741 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 798 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 854 y(END)g(IF)166 948 y FE(The)15 b(receiv)o(e)g(op)q(eration)g(of)f(the)h(\014rst)f(pro) q(cess)h(m)o(ust)f(complete)i(b)q(efore)f(its)f(send,)h(and)g(can)g (com-)75 1004 y(plete)j(only)g(if)g(the)f(matc)o(hing)g(send)h(of)f (the)g(second)h(pro)q(cessor)f(is)h(executed;)g(the)g(receiv)o(e)g(op)q (eration)75 1061 y(of)c(the)h(second)g(pro)q(cess)g(m)o(ust)f(complete) h(b)q(efore)g(its)f(send)i(and)e(can)h(complete)g(only)g(if)g(the)g (matc)o(hing)75 1117 y(send)h(of)f(the)g(\014rst)g(pro)q(cess)g(is)h (executed.)21 b(This)15 b(program)f(will)j(deadlo)q(c)o(k.)166 1174 y(The)10 b(follo)o(wing)h(program)f(is)g(unsafe,)h(and)g(ma)o(y)e (succeed)j(or)e(fail,)h(dep)q(ending)i(on)d(implemen)o(tation.)75 1337 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1393 y(IF)h(\(rank.EQ.0\))123 1450 y(THEN)170 1506 y(CALL)g (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 1562 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(ierr\))123 1619 y(ELSE)95 b(!)23 b(rank.EQ.1)170 1675 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 1732 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 1788 y(END)g(IF)166 1951 y FE(The)f(message)g(sen)o(t)f(b)o(y)h(eac)o(h)g(pro)q(cess)g(has) g(to)g(b)q(e)g(copied)h(out)f(b)q(efore)g(the)g(send)h(op)q(eration)75 2008 y(returns)15 b(and)h(the)f(receiv)o(e)h(op)q(eration)g(starts.)i (F)l(or)d(the)g(program)g(to)f(complete,)i(it)f(is)h(necessary)g(that) 75 2064 y(at)h(least)g(one)h(of)f(the)g(t)o(w)o(o)f(messages)h(sen)o(t) g(is)h(bu\013ered.)27 b(Th)o(us,)17 b(this)h(program)e(can)i(succeed)h (only)e(if)75 2120 y(the)h(comm)o(unication)g(system)g(has)f (su\016cien)o(t)i(bu\013er)e(space)h(to)g(bu\013er)35 b Fo(count)19 b FE(w)o(ords)e(of)g(data,)h(and)75 2177 y(the)d(MPI)g(implemen)o(tation)i(bu\013ers)e(messages.)75 2320 y Fy(4.5)59 b(Data)19 b(T)-5 b(yp)r(e)19 b(Matching)75 2422 y FE(One)d(can)f(think)h(of)f(message)g(transmission)g(as)g (consisting)h(of)f(three)g(phases:)131 2516 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i(send)f(bu\013er)h(and)f(a)g(message)g (is)g(assem)o(bled)131 2610 y(2.)22 b(A)15 b(message)g(is)g (transferred)g(from)f(sender)i(to)f(receiv)o(er)131 2704 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h(incoming)i(message)e (and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o(e)h(bu\013er)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 50 56 50 55 bop 75 -100 a FE(50)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FE(T)o(yp)q(e)k (matc)o(hing)f(has)h(to)e(b)q(e)j(observ)o(ed)e(at)g(eac)o(h)g(of)h (these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f(eac)o(h)75 102 y(v)m(ariable)i(in)f(the)g(sender)g(bu\013er)g(has)g(to)f(matc)o(h) g(the)g(t)o(yp)q(e)h(sp)q(eci\014ed)i(for)d(that)g(en)o(try)g(b)o(y)h (the)f(send)75 158 y(op)q(eration;)d(the)f(t)o(yp)q(e)h(sp)q(eci\014ed) i(b)o(y)e(the)f(send)i(op)q(eration)e(has)h(to)f(matc)o(h)g(the)h(t)o (yp)q(e)f(sp)q(eci\014ed)j(b)o(y)e(the)75 214 y(receiv)o(e)g(op)q (eration;)e(and)h(the)g(t)o(yp)q(e)f(of)h(eac)o(h)f(v)m(ariable)i(in)g (the)e(receiv)o(e)i(bu\013er)f(has)f(to)g(matc)o(h)g(the)h(t)o(yp)q(e) 75 271 y(sp)q(eci\014ed)f(for)e(that)g(en)o(try)g(b)o(y)g(the)g(receiv) o(e)h(op)q(eration.)20 b(A)12 b(program)f(that)h(fails)h(to)e(observ)o (e)h(these)h(three)75 327 y(rules)j(is)g(erroneous.)166 384 y(T)l(o)f(de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l (,)j(w)o(e)d(need)i(to)e(deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 441 y(t)o(yp)q(es)e(of)g(the)h(host)f (language)g(with)h(t)o(yp)q(es)g(sp)q(eci\014ed)h(in)f(comm)o (unication)h(op)q(erations;)e(and)h(matc)o(hing)75 497 y(of)h(t)o(yp)q(es)g(at)g(sender)g(and)h(receiv)o(er.)166 554 y(A)h(t)o(yp)q(e)f(sp)q(eci\014ed)j(for)d(an)h(en)o(try)f(b)o(y)h (a)f(send)h(op)q(eration)g(matc)o(hes)f(the)h(t)o(yp)q(e)g(sp)q (eci\014ed)i(for)d(that)75 610 y(en)o(try)11 b(b)o(y)h(a)f(receiv)o(e)h (op)q(eration)g(if)g(b)q(oth)f(op)q(erations)h(use)g(iden)o(tical)h (names:)36 b Fo(MPI)p 1500 610 14 2 v 16 w(INTEGER)12 b FE(matc)o(hes)75 667 y Fo(MPI)p 160 667 V 16 w(INTEGER)p FE(,)30 b Fo(MPI)p 490 667 V 16 w(REAL)16 b FE(matc)o(hes)30 b Fo(MPI)p 906 667 V 16 w(REAL)p FE(,)15 b(and)g(so)g(on.)166 723 y(The)k(t)o(yp)q(e)g(of)f(a)g(v)m(ariable)i(in)g(a)e(host)h (program)e(matc)o(hes)h(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)i(in)e(the)g (comm)o(u-)75 780 y(nication)g(op)q(eration)g(if)f(the)h(datat)o(yp)q (e)f(name)g(used)h(b)o(y)f(that)f(op)q(eration)i(corresp)q(onds)f(to)g (the)g(basic)75 836 y(t)o(yp)q(e)i(of)g(the)g(host)g(program)f(v)m (ariable:)31 b(an)20 b(en)o(try)f(with)i(t)o(yp)q(e)f(name)40 b Fo(MPI)p 1449 836 V 16 w(INTEGER)21 b FE(matc)o(hes)e(a)75 893 y(F)l(ortran)13 b(v)m(ariable)i(of)e(t)o(yp)q(e)38 b Fp(INTEGER)p FE(,)12 b(an)i(en)o(try)f(with)h(t)o(yp)q(e)g(name)27 b Fo(MPI)p 1376 893 V 16 w(REAL)14 b FE(matc)o(hes)g(a)f(F)l(ortran)75 949 y(v)m(ariable)18 b(of)d(t)o(yp)q(e)40 b Fp(REAL)p FE(,)15 b(and)i(so)f(on.)22 b(There)17 b(is)f(one)g(exception)i(to)d (this)i(last)f(rule:)22 b(An)17 b(en)o(try)f(with)75 1005 y(t)o(yp)q(e)i(name)37 b Fo(MPI)p 408 1005 V 15 w(BYTE)19 b FE(can)f(b)q(e)h(used)g(to)e(matc)o(h)h(an)o(y)g(b)o(yte)g (of)g(storage)e(\(on)i(a)g(b)o(yte-addressable)75 1062 y(mac)o(hine\),)d(irresp)q(ectiv)o(e)i(of)d(the)h(datat)o(yp)q(e)g(of)f (the)h(v)m(ariable)i(that)d(con)o(tains)h(this)g(b)o(yte.)20 b(The)15 b(v)m(alue)h(of)75 1118 y(the)f(message)g(en)o(try)g(will)i(b) q(e)f(the)f(binary)h(v)m(alue)g(of)f(the)g(corresp)q(onding)h(b)o(yte)f (in)h(memory)l(.)166 1175 y(W)l(e)f(th)o(us)g(ha)o(v)o(e)g(t)o(w)o(o)f (cases:)143 1270 y FA(\017)23 b FE(Comm)o(unication)11 b(of)f(t)o(yp)q(ed)i(v)m(alues)g(\(e.g.,)e(with)h(datat)o(yp)q(e)g (di\013eren)o(t)g(from)21 b Fo(MPI)p 1586 1270 V 16 w(BYTE)p FE(\),)11 b(where)189 1326 y(the)k(datat)o(yp)q(es)f(of)g(the)h (corresp)q(onding)h(en)o(tries)f(in)h(the)f(sender)g(program,)e(in)j (the)f(send)g(call,)h(in)189 1383 y(the)f(receiv)o(e)h(call)g(and)g(in) g(the)f(receiv)o(er)h(program)e(should)i(all)g(matc)o(h.)143 1477 y FA(\017)23 b FE(Comm)o(unication)15 b(of)f(un)o(t)o(yp)q(ed)h(v) m(alues)h(\(e.g.,)d(of)h(datat)o(yp)q(e)29 b Fo(MPI)p 1334 1477 V 16 w(BYTE)p FE(\),)15 b(where)g(b)q(oth)f(sender)189 1534 y(and)h(receiv)o(er)g(use)h(the)f(datat)o(yp)q(e)29 b Fo(MPI)p 890 1534 V 16 w(BYTE)p FE(.)15 b(In)g(this)h(case,)e(there)h (are)g(no)g(requiremen)o(ts)g(on)189 1590 y(the)f(t)o(yp)q(es)h(of)f (the)g(corresp)q(onding)h(en)o(tries)g(in)h(the)e(sender)h(and)g(the)f (receiv)o(er)h(programs,)e(nor)h(is)189 1647 y(it)h(required)h(that)f (they)g(b)q(e)h(the)f(same.)166 1741 y(Examples:)166 1798 y(1st)f(program:)75 1893 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f (rank,)h(ierr\))75 1949 y(IF\(rank.EQ.0\))123 2006 y(THEN)170 2062 y(CALL)h(MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f (ierr\))123 2119 y(ELSE)170 2175 y(CALL)h(MPI_RECV\(a\(1\),)e(15,)h (MPI_REAL,)g(0,)g(tag,)h(comm,)f(ierr\))75 2232 y(END)g(IF)166 2326 y FE(This)c(co)q(de)f(is)h(correct)e(if)i(b)q(oth)f(sender)h(and)f (receiv)o(er)h(programs)d(ha)o(v)o(e)i(allo)q(cated)h(consecutiv)o(e)75 2383 y(storage)14 b(for)h(ten)g(real)g(n)o(um)o(b)q(ers,)g(starting)g (from)f Fp(a\(1\))p FE(.)20 b(2nd)15 b(program:)75 2478 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2534 y(IF\(rank.EQ.0\))123 2591 y(THEN)170 2647 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 2704 y(ELSE)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 51 57 51 56 bop 75 -100 a Fx(4.6.)34 b(D)o(A)l(T)l(A)15 b(CONVERSION)1197 b FE(51)170 45 y Fp(CALL)24 b(MPI_RECV\(a\(1\),)e(40,)h(MPI_BYTE,)g(0,) g(tag,)h(comm,)f(ierr\))75 102 y(END)g(IF)166 191 y FE(This)d(co)q(de)g (is)f(erroneous,)h(since)g(sender)g(and)g(receiv)o(er)g(do)f(not)f(pro) o(vide)i(matc)o(hing)f(datat)o(yp)q(e)75 247 y(parameters.)g(3rd)c (program:)75 337 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\)) 75 393 y(IF\(rank.EQ.0\))123 450 y(THEN)170 506 y(CALL)h (MPI_SEND\(a\(1\),)e(40,)h(MPI_BYTE,)g(1,)g(tag,)h(comm,)f(ierr\))123 563 y(ELSE)170 619 y(CALL)h(MPI_RECV\(a\(1\),)e(60,)h(MPI_BYTE,)g(0,)g (tag,)h(comm,)f(ierr\))75 676 y(END)g(IF)166 765 y FE(This)16 b(co)q(de)f(is)h(correct,)f(irresp)q(ectiv)o(e)h(of)f(the)g(t)o(yp)q(e) h(of)e Fp(a\(1\))h FE(and)g(follo)o(wing)h(v)m(ariables)h(in)f(store.) 166 897 y Fv(Discussion:)166 947 y Fu(The)f(b)q(eha)o(vior)f(of)g (erroneous)i(co)q(de)g(is)e(implemen)o(tation-dep)q(enden)o(t.)k(It)d (is)g(desirable)g(but)g(not)f(required)75 997 y(that)i(errors)i(are)e (rep)q(orted)i(at)e(run-time.)24 b(An)16 b(MPI)g(implemen)o(tation)d (ma)o(y)h(decide)j(to)f(a)o(v)o(oid)f(t)o(yp)q(e-c)o(hec)o(king)75 1047 y(for)d(the)h(sak)o(e)f(of)g(e\016ciency)m(,)h(in)f(whic)o(h)g (case)h(none)g(of)e(the)i(three)h(programs)d(will)g(rep)q(ort)i(an)f (error.)19 b(On)12 b(a)g(system)75 1097 y(where)f(real)g(n)o(um)o(b)q (ers)f(o)q(ccup)o(y)h(four)f(b)o(ytes)h(and)f(b)q(oth)g(sender)i(and)e (receiv)o(er)i(execute)g(in)e(the)h(same)f(en)o(vironmen)o(t,)75 1146 y(it)k(is)f(lik)o(ely)g(that)h(all)e(three)k(co)q(des)f(will)d(ha) o(v)o(e)i(the)g(same)f(b)q(eha)o(vior.)75 1372 y Fy(4.6)59 b(Data)19 b(conversion)75 1473 y FE(One)d(of)f(the)g(goals)g(of)g(MPI)g (is)h(to)e(supp)q(ort)i(parallel)g(computations)f(across)g (heterogeneous)g(en)o(viron-)75 1529 y(men)o(ts.)20 b(Comm)o(unication) 15 b(in)h(a)f(heterogeneous)g(en)o(vironmen)o(t)h(ma)o(y)e(require)i (data)f(con)o(v)o(ersions.)166 1586 y(W)l(e)g(use)h(the)f(follo)o(wing) h(terminology:)75 1675 y FF(t)o(yp)q(e)h(con)o(v)o(ersion)22 b FE(c)o(hanges)14 b(the)h(datat)o(yp)q(e)f(of)f(a)i(v)m(alue,)g(e.g.)k (b)o(y)14 b(rounding)i(a)28 b Fo(REAL)15 b FE(to)e(an)29 b Fo(INTE-)189 1732 y(GER)p FE(.)75 1824 y FF(represen)o(tation)17 b(con)o(v)o(ersion)22 b FE(c)o(hanges)16 b(the)h(binary)g(represen)o (tation)f(of)g(a)g(v)m(alue,)h(e.g.)23 b(from)16 b(Hex)189 1880 y(\015oating)f(p)q(oin)o(t)g(to)g(IEEE)g(\015oating)h(p)q(oin)o (t.)166 1970 y(Represen)o(tation)d(con)o(v)o(ersion)g(is)h(needed)g (when)f(data)f(is)i(mo)o(v)o(ed)e(across)g(di\013eren)o(t)h(en)o (vironmen)o(ts,)75 2026 y(e.g.)19 b(di\013eren)o(t)d(mac)o(hines)g (that)e(use)i(di\013eren)o(t)f(binary)h(enco)q(dings)h(for)d(the)i (same)e(datat)o(yp)q(e,)h(or)f(co)q(des)75 2083 y(compiled)k(with)e (di\013eren)o(t)h(compiler)g(options)f(for)g(datat)o(yp)q(e)f(represen) o(tation.)23 b(Represen)o(tation)17 b(con-)75 2139 y(v)o(ersion)12 b(preserv)o(es,)g(to)f(the)h(greatest)f(p)q(ossible)i(exten)o(t,)f(the) g FF(v)m(alue)g FE(represen)o(ted.)19 b(Ho)o(w)o(ev)o(er,)11 b(rounding)75 2195 y(errors)k(and)g(o)o(v)o(er\015o)o(w)f(and)i (under\015o)o(w)g(exceptions)g(ma)o(y)f(o)q(ccur)h(during)g(\015oating) f(p)q(oin)o(t)h(con)o(v)o(ersions;)75 2252 y(con)o(v)o(ersion)k(of)f (in)o(tegers)g(ma)o(y)g(also)h(lead)g(to)f(exceptions)i(when)f(a)f(v)m (alue)i(that)d(can)i(represen)o(ted)g(in)75 2308 y(one)h(system)g (cannot)g(b)q(e)g(represen)o(ted)h(in)g(the)f(other)f(system.)37 b(MPI)21 b(do)q(es)h(not)e(sp)q(ecify)j(rules)f(for)75 2365 y(represen)o(tation)15 b(con)o(v)o(ersion.)166 2421 y(The)e(t)o(yp)q(e)g(matc)o(hing)g(rules)g(imply)h(that)e(MPI)h(comm)o (unication)h(nev)o(er)f(en)o(tails)g(t)o(yp)q(e)g(con)o(v)o(ersion.)75 2478 y(On)20 b(the)f(other)f(hand,)i(MPI)f(requires)h(that)e(a)h (represen)o(tation)g(con)o(v)o(ersion)g(b)q(e)h(p)q(erformed)f(when)g (a)75 2534 y(t)o(yp)q(ed)i(v)m(alue)h(is)f(transferred)f(across)g(en)o (vironmen)o(ts)h(that)f(use)h(di\013eren)o(t)f(represen)o(tations)h (for)f(the)75 2591 y(datat)o(yp)q(e)15 b(of)h(this)g(v)m(alue.)24 b(An)16 b(exception)h(o)q(ccurring)g(during)g(represen)o(tation)f(con)o (v)o(ersion)g(results)g(in)75 2647 y(a)f(failure)h(of)f(the)g(comm)o (unication;)g(an)h(error)e(o)q(ccurs)h(either)h(in)g(the)f(send)h(op)q (eration,)f(or)g(the)g(receiv)o(e)75 2704 y(op)q(eration,)g(or)g(b)q (oth.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 52 58 52 57 bop 75 -100 a FE(52)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FE(If)k(a)g(v)m (alue)i(sen)o(t)e(in)h(a)e(message)h(is)h(un)o(t)o(yp)q(ed)g(\(i.e.,)f (of)g(t)o(yp)q(e)41 b Fo(MPI)p 1374 45 14 2 v 15 w(BYTE)p FE(\),)21 b(then)f(the)g(binary)75 102 y(represen)o(tation)15 b(of)f(the)h(b)o(yte)g(stored)f(at)g(the)h(receiv)o(er)h(is)f(iden)o (tical)i(to)d(the)h(binary)h(represen)o(tation)e(of)75 158 y(the)f(b)o(yte)g(loaded)h(at)e(the)h(sender.)20 b(This)14 b(holds)f(true,)g(whether)g(sender)h(and)f(receiv)o(er)h(run) f(in)h(the)f(same)75 214 y(or)i(in)h(distinct)g(en)o(vironmen)o(ts;)f (no)g(represen)o(tation)h(con)o(v)o(ersion)f(is)h(required.)166 288 y(Note)d(that)g(no)g(con)o(v)o(ersion)h(ev)o(er)f(o)q(ccurs)h(when) g(an)f(MPI)g(program)g(executes)h(in)g(a)f(homogeneous)75 344 y(system,)18 b(where)g(all)h(pro)q(cesses)g(run)f(in)h(the)f(same)g (en)o(vironmen)o(t.)29 b(Also)18 b(note)g(the)g(di\013eren)o(t)g(b)q (eha)o(v-)75 401 y(ior)g(of)37 b Fo(MPI)p 305 401 V 15 w(BYTE)19 b FE(and)g(of)36 b Fo(MPI)p 704 401 V 16 w(CHARA)o(CTER)p FE(.)19 b(A)g(bu\013er)f(descriptor)h(en)o(try)f(with)h(datat)o(yp)q(e) e(of)75 457 y Fo(MPI)p 160 457 V 16 w(CHARA)o(CTER)j FE(can)f(only)g(matc)o(h)f(a)g(F)l(ortran)g(v)m(ariable)i(of)e(t)o(yp)q (e)42 b Fp(CHARACTER)p FE(;)17 b(and)i(represen-)75 514 y(tation)c(con)o(v)o(ersion)h(ma)o(y)f(o)q(ccur)i(when)f(v)m(alues)h (of)e(t)o(yp)q(e)32 b Fo(MPI)p 1165 514 V 16 w(CHARA)o(CTER)17 b FE(are)e(transferred.,)g(e.g.,)75 570 y(from)f(an)i(EBCDIC)f(enco)q (ding)h(to)f(an)g(ASCI)q(I)h(enco)q(ding.)166 643 y(Consider)g(the)f (previous)h(three)f(examples.)166 717 y(The)c(\014rst)f(program)f(is)i (correct,)g(assuming)g(b)q(oth)f(sender)i(and)e(receiv)o(er)i(declared) f(ten)g(consecutiv)o(e)75 773 y(real)19 b(v)m(ariables)h(in)g(storage)e (starting)g(at)g Fp(a\(1\))p FE(.)30 b(If)19 b(the)g(sender)h(and)f (receiv)o(er)g(execute)h(in)g(di\013eren)o(t)75 830 y(en)o(vironmen)o (ts,)12 b(then)f(the)g(ten)f(real)i(v)m(alues)f(that)g(are)f(fetc)o (hed)h(from)f(the)h(send)h(bu\013er)e(will)j(b)q(e)e(con)o(v)o(erted)75 886 y(to)g(the)h(represen)o(tation)f(for)g(reals)h(on)f(the)h(receiv)o (er)g(site)g(b)q(efore)g(they)g(are)f(stored)g(in)h(the)g(receiv)o(e)g (bu\013er.)75 942 y(While)19 b(the)e(n)o(um)o(b)q(er)h(of)f(real)g (elemen)o(ts)h(fetc)o(hed)g(from)f(the)g(send)h(bu\013er)f(equal)i(the) e(n)o(um)o(b)q(er)h(of)f(real)75 999 y(elemen)o(ts)d(stored)e(in)i(the) f(receiv)o(e)h(bu\013er,)f(the)g(n)o(um)o(b)q(er)g(of)g(b)o(ytes)g (stored)f(need)i(not)f(equal)h(the)f(n)o(um)o(b)q(er)75 1055 y(of)f(b)o(ytes)g(loaded:)19 b(e.g.)g(the)12 b(sender)h(ma)o(y)e (use)i(a)f(four)g(b)o(yte)g(represen)o(tation)h(and)f(the)g(receiv)o (er)i(an)e(eigh)o(t)75 1112 y(b)o(yte)17 b(represen)o(tation)h(for)f (reals.)27 b(If)18 b(the)g(send)g(or)f(receiv)o(e)h(bu\013er)g(do)f (not)h(con)o(tain)f(ten)h(consecutiv)o(e)75 1168 y(real)e(v)m (ariables,)g(then)f(the)g(program)g(is)g(erroneous,)g(and)g(its)h(b)q (eha)o(vior)f(is)h(unde\014ned.)166 1242 y(The)f(second)h(program)e(is) i(erroneous,)f(and)g(its)g(b)q(eha)o(vior)h(is)g(unde\014ned.)166 1315 y(The)e(third)g(program)e(is)i(correct.)19 b(The)13 b(exact)g(same)g(sequence)i(of)e(fort)o(y)f(b)o(ytes)h(that)g(w)o(ere)g (loaded)75 1371 y(from)18 b(the)h(send)h(bu\013er)f(will)i(b)q(e)e (stored)g(in)h(the)f(receiv)o(e)h(bu\013er,)f(ev)o(en)h(if)f(sender)h (and)f(receiv)o(er)h(run)75 1428 y(in)e(a)g(di\013eren)o(t)f(en)o (vironmen)o(t.)28 b(The)18 b(message)f(sen)o(t)g(has)h(exactly)f(the)h (same)f(length)i(\(in)f(b)o(ytes\))f(and)75 1484 y(the)f(same)f(binary) h(represen)o(tation)f(as)g(the)h(message)f(receiv)o(ed.)22 b(If)16 b(the)f(v)m(ariables)i(in)f(the)g(send)g(bu\013er)75 1541 y(are)i(of)h(di\013eren)o(t)g(t)o(yp)q(es)g(as)f(the)h(v)m (ariables)h(in)g(the)f(receiv)o(e)g(bu\013er,)h(or)e(they)h(are)f(of)h (the)f(same)h(t)o(yp)q(e)75 1597 y(but)e(di\013eren)o(t)g(data)g (represen)o(tations)f(are)h(used,)h(then)f(the)g(bits)h(stored)e(in)i (the)f(receiv)o(e)h(bu\013er)f(ma)o(y)75 1654 y(enco)q(de)f(v)m(alues)h (that)d(are)h(di\013eren)o(t)g(from)g(the)g(v)m(alues)h(they)g(enco)q (ded)g(in)g(the)g(send)f(bu\013er.)166 1727 y(Data)f(represen)o(tation) i(con)o(v)o(ersion)f(also)h(applies)h(to)e(the)g(en)o(v)o(elop)q(e)i (of)e(a)g(message:)20 b(source,)15 b(des-)75 1783 y(tination)h(and)f (tag)f(are)h(all)h(in)o(tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i (con)o(v)o(erted.)166 1857 y(The)k(curren)o(t)g(draft)f(do)q(es)h(not)g (pro)o(vide)g(a)g(mec)o(hanism)h(for)e(in)o(ter-language)i(comm)o (unication:)75 1913 y(messages)d(sen)o(t)h(b)o(y)f(F)l(ortran)g(calls)i (should)f(b)q(e)h(receiv)o(ed)g(b)o(y)e(F)l(ortran)g(calls)h(and)g (messages)g(sen)o(t)f(b)o(y)75 1970 y(C)e(calls)g(should)h(b)q(e)g (receiv)o(ed)g(b)o(y)e(C)h(calls)h(\(this)f(follo)o(ws)f(from)h(the)f (requiremen)o(ts)i(of)e(t)o(yp)q(e)h(matc)o(hing)75 2026 y(and)g(the)h(fact)f(that)f(F)l(ortran)g(and)i(C)f(datat)o(yp)q(es)g (are)g(distinct\).)24 b(If)16 b(in)o(ter-language)h(comm)o(unication)75 2082 y(is)g(needed)i(then)e(one)g(needs)h(to)e(in)o(v)o(ok)o(e)h(C)f (comm)o(unication)i(routines)f(from)f(a)h(F)l(ortran)f(program)f(or)75 2139 y(F)l(ortran)f(comm)o(unication)i(routines)g(from)e(a)h(C)g (program.)166 2288 y Fv(Implemen)o(tati)o(on)d(note:)166 2355 y Fu(The)k(curren)o(t)i(de\014nition)e(do)q(es)h(not)f(require)h (messages)f(to)g(carry)h(data)f(t)o(yp)q(e)g(information.)22 b(A)16 b(message)75 2405 y(can)c(b)q(e)h(comp)q(osed)f(and)g(sen)o(t)h (using)f(only)g(the)h(information)c(pro)o(vided)j(b)o(y)g(the)h(send)g (call,)e(and)h(can)h(b)q(e)g(receiv)o(ed)75 2455 y(and)k(stored)g (using)g(only)f(the)h(information)d(pro)o(vided)j(b)o(y)g(the)g(receiv) o(e)h(call.)26 b(If)17 b(messages)g(are)g(sen)o(t)h(b)q(et)o(w)o(een)75 2504 y(di\013eren)o(t)c(mac)o(hines)f(then)g(one)h(can)f(either)h(use)g (a)f(\\univ)o(ersal")g(data)g(enco)q(ding)g(for)g(messages,)g(use)h (kno)o(wledge)75 2554 y(of)g(the)i(receiv)o(er)h(en)o(vironmen)o(t)d (in)g(order)i(to)f(con)o(v)o(ert)h(data)e(at)h(the)h(sender,)h(or)e (use)g(kno)o(wledge)g(of)g(the)g(sender)75 2604 y(en)o(vironmen)o(t)j (in)g(order)i(to)f(con)o(v)o(ert)g(data)g(at)g(the)g(receiv)o(er.)35 b(In)19 b(either)g(case)h(the)g(datat)o(yp)q(e)f(parameter)f(in)75 2654 y(the)f(lo)q(cal)f(call)g(can)g(b)q(e)h(used)h(to)e(deriv)o(e)h (the)g(t)o(yp)q(es)h(of)e(the)h(v)n(alues)f(transferred.)28 b(Ho)o(w)o(ev)o(er,)17 b(additional)d(t)o(yp)q(e)75 2704 y(information)d(carried)k(b)o(y)e(messages)h(can)g(pro)o(vide)g(b)q (etter)h(error)g(detection.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 53 59 53 58 bop 75 -100 a Fx(4.7.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b FE(53)166 45 y Fv(Discussion:)166 99 y Fu(MPI)14 b(do)q(es)h(not)f(handle)g(in)o(ter-language)g(comm)o (unicatio)o(n)e(b)q(ecause)k(there)f(seem)f(to)g(b)q(e)h(no)f(agreed)h (stan-)75 149 y(dards)20 b(for)f(in)o(ter-language)f(t)o(yp)q(e)i(con)o (v)o(ersions.)34 b(The)20 b(curren)o(t)h(\(non\)solution)d(means)g (that)i(in)o(ter-language)75 199 y(comm)o(unication)10 b(relies)15 b(on)f(the)g(v)o(endor)g(sp)q(eci\014c)i(rules)e(for)g(in)o (ter-language)f(calls)g(at)h(eac)o(h)g(site.)166 253 y(It)i(ma)o(y)e(b)q(e)i(desirable)h(to)e(pro)o(vide)h(information)d(on) j(the)g(corresp)q(ondence)j(b)q(et)o(w)o(een)e(MPI)g(C)e(datat)o(yp)q (es)75 302 y(\(suc)o(h)i(as)38 b Fn(MPI)p 331 302 14 2 v 15 w(FLOAT)p Fu(\))15 b(and)h(MPI)g(F)m(ortran)g(datat)o(yp)q(es)g (\(suc)o(h)h(as)38 b Fn(MPI)p 1265 302 V 15 w(REAL)p Fu(\))16 b(with)f(suitable)h(en)o(vironmen)o(t)75 352 y(query)e(functions.)75 602 y Fy(4.7)59 b(Communication)18 b(Mo)r(des)75 711 y FE(The)12 b(basic)h(send)g(op)q(eration)g(describ)q (ed)h(in)f(Section)g(4.2)e(used)i(the)f FF(standard)h FE(comm)o(unication)g(mo)q(de.)75 768 y(In)h(suc)o(h)g(a)f(comm)o (unication)h(mo)q(de,)g(a)f(send)h(op)q(eration)g(can)f(b)q(e)h (started)f(whether)h(or)f(not)g(a)g(matc)o(hing)75 824 y(receiv)o(e)g(w)o(as)e(p)q(osted.)20 b(The)12 b(completion)h(of)f(the) g(send)h(op)q(eration)f(indicates)i(that)d(the)i(message)e(and)i(its)75 881 y(en)o(v)o(elop)q(e)j(ha)o(v)o(e)e(b)q(een)i(safely)f(stored)f(a)o (w)o(a)o(y)g(and)g(that)h(the)f(sender)i(is)f(free)g(to)f(access)h(and) g(mo)q(dify)g(the)75 937 y(sender)g(bu\013er.)k(Th)o(us,)14 b(the)g(op)q(eration)h(is)f FF(blo)q(c)o(king)p FE(:)21 b(it)14 b(do)q(es)h(not)e(return)i(un)o(til)g(the)f(send)h(op)q (eration)75 993 y(has)g(completed)h FF(lo)q(cally)p FE(,)h(on)f(the)f (sender)h(side.)166 1054 y(The)f(completion)i(of)e(a)g(send)h(op)q (eration)f(giv)o(es)h(no)f(indication)i(that)e(the)g(message)g(w)o(as)g (receiv)o(ed)75 1110 y(on)g(the)h(receiv)o(er)g(side.)21 b(A)15 b(blo)q(c)o(king)i(send)f(ma)o(y)e(b)q(e)i(implemen)o(ted)h(so)e (that)g(a)g(blo)q(c)o(king)h(send)g(returns)75 1167 y(only)g(after)f(a) g(matc)o(hing)g(receiv)o(e)h(has)g(b)q(een)g(executed)g(on)g(the)f (receiv)o(er)h(side.)22 b(This)16 b(a)o(v)o(oids)f(the)g(need)75 1223 y(to)i(bu\013er)g(message)g(data)g(out)g(of)g(sender)h(or)f (receiv)o(er)h(memory)l(.)26 b(In)18 b(this)g(case)f(the)h(send)g(op)q (eration)75 1280 y(completes)j(only)f(after)f(the)h(matc)o(hing)g (receiv)o(e)h(has)f(started)f(executing.)35 b(On)20 b(the)g(other)g (hand,)h(it)75 1336 y(is)g(also)f(p)q(ossible)i(for)e(MPI)g(to)g (bu\013er)g(messages,)h(so)f(as)g(to)f(allo)o(w)i(the)f(sender)h(to)f (pro)q(ceed)h(ahead)75 1393 y(of)e(the)h(receiv)o(er.)35 b(In)20 b(suc)o(h)g(a)f(case)h(the)g(send)g(op)q(eration)g(ma)o(y)g (complete)g(successfully)i(b)q(efore)e(the)75 1449 y(message)c(w)o(as)g (receiv)o(ed.)26 b(Th)o(us,)16 b(the)h(basic)h(send)f(op)q(eration)g (describ)q(ed)h(in)g(the)f(previous)g(section)g(is)75 1506 y FF(async)o(hronous)p FE(,)12 b(since)i(its)f(return)g(do)q(es)h (not)f(imply)h(a)f(sync)o(hronization)h(with)f(the)g(\(remote\))f (receiv)o(e)75 1562 y(op)q(eration,)j(and)g(do)q(es)h(not)f(imply)h FF(global)h FE(termination)f(of)f(the)g(comm)o(unication.)166 1623 y(There)g(are)g(t)o(w)o(o)f(additional)j(comm)o(unication)f(mo)q (des:)166 1683 y(A)h(send)g(that)f(uses)i(the)f FF(ready)f FE(comm)o(unication)i(mo)q(de)f(ma)o(y)f(b)q(e)h(started)g(only)g(if)g (a)g(matc)o(hing)75 1740 y(receiv)o(e)d(is)f(already)g(p)q(osted;)g (otherwise)g(the)g(op)q(eration)g(is)g(erroneous)f(and)h(its)g(outcome) f(is)h(unde\014ned.)75 1796 y(In)19 b(some)f(systems,)h(this)f(allo)o (ws)h(the)f(remo)o(v)m(al)h(of)f(a)g(hand-shak)o(e)h(op)q(eration)g (that)e(is)i(otherwise)g(re-)75 1853 y(quired,)g(and)f(results)g(in)g (impro)o(v)o(ed)g(p)q(erformance.)27 b(The)18 b(completion)h(of)e(the)h (send)g(op)q(eration)g(do)q(es)75 1909 y(not)e(dep)q(end)i(on)e(the)g (status)f(of)h(a)g(matc)o(hing)g(receiv)o(e,)h(and)f(merely)h (indicates)h(that)d(the)h(send)h(bu\013er)75 1966 y(can)i(b)q(e)g (reused.)30 b(A)19 b(send)g(op)q(eration)g(that)f(uses)g(the)h(ready)f (mo)q(de)h(has)g(the)f(same)g(seman)o(tics)h(as)f(a)75 2022 y(standard)g(send)h(op)q(eration.)29 b(It)18 b(is)h(merely)g(that) f(the)g(sender)h(pro)o(vides)f(additional)i(information)f(to)75 2078 y(the)c(comm)o(unication)h(subsystem,)f(when)h(using)g(the)f (ready)g(mo)q(de.)166 2139 y(A)j(send)g(that)f(uses)g(the)h FF(sync)o(hronous)e FE(mo)q(de)i(can)g(b)q(e)g(started)f(whether)g(or)g (not)h(a)f(matc)o(hing)75 2195 y(receiv)o(e)12 b(w)o(as)e(p)q(osted.)19 b(Ho)o(w)o(ev)o(er,)10 b(the)h(send)h(will)g(complete)g(successfully)h (only)f(if)f(a)g(matc)o(hing)g(receiv)o(e)g(is)75 2252 y(p)q(osted,)h(and)g(the)g(receiv)o(e)g(op)q(eration)g(has)f(started)g (to)g(receiv)o(e)i(the)e(message)g(sen)o(t)h(b)o(y)f(the)h(sync)o (hronous)75 2308 y(send.)29 b(\(I.e.,)19 b(the)f(receiv)o(e)h(has)f(b)q (een)h(p)q(osted,)g(and)g(the)f(incoming)h(message)f(has)g(b)q(een)i (matc)o(hed)e(to)75 2365 y(this)g(p)q(osted)g(receiv)o(e.\))28 b(Th)o(us,)18 b(the)g(completion)g(of)g(a)f(sync)o(hronous)h(send)g (not)f(only)i(indicates)g(that)75 2421 y(the)g(send)h(bu\013er)f(can)g (b)q(e)h(reused,)g(but)f(also)g(indicates)i(that)d(the)i(receiv)o(er)f (has)g(reac)o(hed)h(a)f(certain)75 2478 y(p)q(oin)o(t)e(in)h(its)f (execution,)h(namely)g(that)e(it)h(has)g(started)f(executing)i(the)f (matc)o(hing)g(receiv)o(e.)26 b(If)17 b(b)q(oth)75 2534 y(sends)f(and)h(receiv)o(es)f(are)g(blo)q(c)o(king)h(op)q(erations)f (then)h(the)f(use)g(of)f(the)h(sync)o(hronous)g(mo)q(de)g(pro)o(vides) 75 2591 y(sync)o(hronous)f(comm)o(unication)h(seman)o(tics:)k(a)14 b(comm)o(unication)i(do)q(es)f(not)g(complete)g(at)g(either)g(ends)75 2647 y(b)q(efore)d(b)q(oth)g(pro)q(cesses)g(\\attend")f(to)g(the)h (comm)o(unication;)h(the)f(completion)h(of)e(a)h(sync)o(hronous)f(send) 75 2704 y(is)16 b(a)f FF(global)i FE(ev)o(en)o(t.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 54 60 54 59 bop 75 -100 a FE(54)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FE(Tw)o(o)22 b(additional)j(send)e(functions)h(are)f(pro)o(vided)h(for)f(the)g(t)o (w)o(o)f(additional)i(comm)o(unication)75 102 y(mo)q(des.)32 b(The)19 b(comm)o(unication)h(mo)q(de)f(is)h(indicated)g(b)o(y)f(a)g (one)g(letter)h(pre\014x:)28 b Fp(R)18 b FE(for)h(ready)g(and)g Fp(S)75 158 y FE(for)c(sync)o(hronous.)166 216 y(Send)h(in)g(ready)f (mo)q(de)75 320 y Fo(MPI)p 160 320 14 2 v 16 w(RSEND)h(\(sta)o(rt,)f (count,)h(datat)o(yp)q(e,)g(dest,)g(tag,)f(comm\))117 399 y Fu(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 476 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 554 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t) 117 631 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o (teger\))117 709 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 786 y(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(\(handle\))166 912 y FE(Send)16 b(in)g(sync)o(hronous)f(mo)q(de)75 1017 y Fo(MPI)p 160 1017 V 16 w(SSEND)h(\(sta)o(rt,)e(count,)j(datat)o(yp)q(e,)f(dest,)g (tag,)f(comm\))117 1095 y Fu(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1172 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1250 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 1327 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1405 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 1482 y(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(\(handle\))166 1608 y FE(There)16 b(is)g(only)g(one)g(receiv)o(e)h(mo)q(de,)e(whic)o(h)i(can)f(matc)o(h)f (an)o(y)g(of)h(the)f(send)i(mo)q(des.)k(The)16 b(receiv)o(e)75 1664 y(op)q(eration)g(describ)q(ed)h(in)g(the)e(last)h(section)g(is)g FF(blo)q(c)o(king)p FE(:)22 b(it)16 b(returns)f(only)h(after)f(the)h (receiv)o(e)g(bu\013er)75 1721 y(con)o(tains)h(the)h(newly)g(receiv)o (ed)g(message.)26 b(It)17 b(is)h FF(async)o(hronous)p FE(:)23 b(the)17 b(completion)i(of)d(a)h(receiv)o(e)h(is)75 1777 y(a)e(lo)q(cal)i(op)q(eration,)f(and)f(a)g(receiv)o(e)i(can)f (complete)g(b)q(efore)g(the)f(matc)o(hing)h(send)g(has)f(completed)i (\(of)75 1834 y(course,)d(it)g(can)h(complete)g(only)f(after)g(the)g (matc)o(hing)g(send)h(has)f(started\).)166 1891 y(Comm)o(unication)e (imp)q(oses)g(an)f(order)h(on)f(the)h(ev)o(en)o(ts)f(o)q(ccurring)h(at) f(the)h(comm)o(unicating)g(no)q(des.)75 1948 y(It)i(is)g(alw)o(a)o(ys)e (the)i(case)g(that)e(the)i(completion)h(of)e(a)g(receiv)o(e)i(o)q (ccurs)e(after)g(the)h(start)e(of)h(the)h(matc)o(hing)75 2004 y(send.)29 b(If)19 b(the)f(sync)o(hronous)g(send)h(mo)q(de)f(is)h (used,)g(then)g(it)f(is)h(also)f(the)g(case)g(that)g(the)g(completion) 75 2061 y(of)i(a)h(send)g(o)q(ccurs)g(after)f(the)g(start)g(of)g(the)h (matc)o(hing)f(receiv)o(e.)38 b(Of)20 b(course,)i(on)f(eac)o(h)f(pro)q (cess,)i(a)75 2117 y(comm)o(unication)14 b(completes)h(after)d(it)i(is) g(started.)19 b(No)13 b(other)h(order)f(is)h(imp)q(osed)h(b)o(y)e(MPI.) g(E.g.,)g(if)h(the)75 2174 y(standard)h(send)g(mo)q(de)h(is)f(used,)h (the)f(send)h(op)q(eration)f(ma)o(y)f(complete)i(b)q(efore)g(the)f (matc)o(hing)g(receiv)o(e)75 2230 y(w)o(as)f(started.)166 2288 y(In)i(a)e(m)o(ulti-threaded)j(implemen)o(tation)f(of)f(MPI,)f(a)h (blo)q(c)o(king)h(send)g(or)e(receiv)o(e)i(op)q(eration)g(ma)o(y)75 2344 y(only)j(blo)q(c)o(k)g(the)g(executing)h(thread,)f(not)f(the)h(en) o(tire)g(pro)q(cess.)30 b(Th)o(us,)19 b(while)h(a)e(thread)h(is)g(blo)q (c)o(k)o(ed)75 2401 y(w)o(aiting)k(for)e(a)h(send)h(or)f(receiv)o(e)i (to)d(complete,)k(other)d(threads)g(ma)o(y)g(execute)h(within)h(the)e (same)75 2457 y(address)16 b(space.)22 b(In)17 b(suc)o(h)f(a)f(case)h (it)g(the)g(user)g(resp)q(onsibilit)o(y)i(not)e(to)f(mo)q(dify)i(the)f (send)g(bu\013er)g(un)o(til)75 2513 y(the)11 b(send)h(completes)g(and)g (not)e(to)h(access)g(or)g(mo)q(dify)h(the)f(receiv)o(e)i(bu\013er)e(un) o(til)h(the)f(receiv)o(e)i(completes;)75 2570 y(otherwise)i(the)h (outcome)f(of)f(the)i(computation)f(is)g(unde\014ned.)166 2704 y Fv(Discussion:)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 55 61 55 60 bop 75 -100 a Fx(4.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FE(55)166 45 y Fu(Do)13 b(w)o(e)g(w)o(an)o(t)g(to)g(sa)o(y)g(that)g (a)g(blo)q(c)o(king)g(op)q(eration)g(MA)m(Y)g(or)g(MUST)g(only)g(blo)q (c)o(k)g(the)h(executing)g(thread?)166 263 y Fv(Implemen)o(tati)o(on)c (note:)34 b Fu(A)13 b(ready)f(send)h(can)f(b)q(e)h(implemen)o(ted)d(as) i(a)g(standard)h(send;)g(in)e(suc)o(h)i(a)f(case)75 313 y(there)j(will)e(b)q(e)h(no)g(p)q(erformance)g(adv)n(an)o(tage)f(\(or)h (disadv)n(an)o(tage\))f(for)g(the)i(use)f(of)g(ready)g(send.)166 366 y(A)f(standard)h(send)h(can)e(b)q(e)h(implemen)o(ted)e(as)h(a)g (sync)o(hronous)i(send.)j(In)c(suc)o(h)g(a)f(case,)h(no)f(data)h (bu\013ering)75 416 y(is)g(needed.)166 469 y(A)g(p)q(ossible)g(comm)o (unication)c(proto)q(col)k(for)g(the)g(v)n(arious)f(comm)o(unication)e (mo)q(des)i(is)h(outlined)f(b)q(elo)o(w:)166 521 y Fn(ready)21 b(send)p Fu(:)c(The)d(message)g(is)f(sen)o(t)i(as)f(so)q(on)g(as)g(p)q (ossible.)166 574 y Fn(synchronous)19 b(send:)29 b Fu(The)20 b(sender)g(sends)h(a)e(request-to-send)j(message.)34 b(The)19 b(receiv)o(er)i(stores)g(this)75 624 y(request.)f(When)15 b(a)f(matc)o(hing)e(receiv)o(e)j(is)f(p)q(osted,)h(the)g(receiv)o(er)h (sends)f(bac)o(k)f(a)g(p)q(ermission-to-send)g(message,)75 674 y(and)g(the)g(sender)i(no)o(w)d(sends)i(the)g(message.)166 727 y Fn(standard)20 b(send:)e Fu(First)d(proto)q(col)f(ma)o(y)e(b)q(e) j(used)h(for)d(short)i(messages,)f(and)g(second)i(proto)q(col)e(for)g (long)75 777 y(messages.)166 829 y(Additional)h(con)o(trol)h(messages)h (migh)o(t)e(b)q(e)i(needed)h(for)e(\015o)o(w)g(con)o(trol)h(and)f (error)i(reco)o(v)o(ery)m(.)27 b(Of)16 b(course,)75 879 y(there)f(are)f(man)o(y)f(other)h(p)q(ossible)g(c)o(hoices.)75 1122 y Fy(4.8)59 b(Nonblo)r(cking)20 b(communication)75 1230 y FE(One)e(can)g(impro)o(v)o(e)f(p)q(erformance)g(on)h(man)o(y)e (systems)h(b)o(y)h(o)o(v)o(erlapping)f(comm)o(unication)i(and)e(com-)75 1286 y(putation.)35 b(This)21 b(is)g(esp)q(ecially)i(true)d(on)g (systems)g(where)g(comm)o(unication)h(can)g(b)q(e)g(executed)g(au-)75 1343 y(tonomously)12 b(b)o(y)h(an)g(in)o(telligen)o(t)h(comm)o (unication)g(con)o(troller.)19 b(Ligh)o(t-w)o(eigh)o(t)13 b(threads)f(are)h(one)g(mec)o(h-)75 1399 y(anism)21 b(for)g(ac)o (hieving)h(suc)o(h)g(o)o(v)o(erlap.)37 b(An)21 b(alternativ)o(e)h(mec)o (hanism)g(that)e(often)h(leads)g(to)g(b)q(etter)75 1456 y(p)q(erformance)e(is)g(to)f(use)h FF(non)o(blo)q(c)o(king)j(comm)o (unication)p FE(.)32 b(A)19 b(non)o(blo)q(c)o(king)h(send)f(call)h (initiates)75 1512 y(the)15 b(send)h(op)q(eration,)f(but)h(do)q(es)f (not)g(complete)h(it.)k(The)15 b(send)h(call)h(will)f(return)g(b)q (efore)f(the)g(message)75 1568 y(w)o(as)d(copied)h(out)f(of)g(the)g (send)h(bu\013er.)19 b(A)13 b(separate)e(call)j(is)f(needed)g(to)f (complete)h(the)g(comm)o(unication,)75 1625 y(i.e.)20 b(to)13 b(v)o(erify)i(that)e(the)h(data)g(has)g(b)q(een)h(copied)g(out) f(of)g(the)g(send)h(bu\013er.)k(With)c(suitable)g(hardw)o(are,)75 1681 y(the)k(transfer)f(of)h(data)f(out)h(of)g(the)g(sender)g(memory)g (ma)o(y)f(pro)q(ceed)i(concurren)o(tly)g(with)f(computa-)75 1738 y(tions)g(done)g(at)f(the)g(sender)h(after)f(the)h(send)g(w)o(as)f (initiated)i(and)e(b)q(efore)h(it)g(completed.)31 b(Similarly)l(,)75 1794 y(a)19 b(non)o(blo)q(c)o(king)h(receiv)o(e)g(initiates)h(the)e (receiv)o(e)h(op)q(eration,)g(but)f(do)q(es)g(not)g(complete)h(it.)32 b(The)19 b(call)75 1851 y(will)f(return)f(b)q(efore)g(a)f(message)g(is) i(stored)e(in)o(to)g(the)h(receiv)o(e)h(bu\013er.)24 b(A)17 b(separate)f(call)h(is)h(needed)g(to)75 1907 y(complete)f(the)f (receiv)o(e)i(op)q(eration)e(and)g(v)o(erify)h(that)e(the)i(data)e(has) i(b)q(een)g(receiv)o(ed)g(in)o(to)g(the)f(receiv)o(e)75 1964 y(bu\013er.)k(With)14 b(suitable)h(hardw)o(are,)f(the)g(transfer)f (of)h(data)g(in)o(to)g(the)g(receiv)o(er)h(memory)f(ma)o(y)f(pro)q (ceed)75 2020 y(concurren)o(tly)f(with)g(computations)f(done)g(after)g (the)g(receiv)o(e)h(w)o(as)f(initiated)i(and)e(b)q(efore)h(it)f (completed.)166 2080 y(Non)o(blo)q(c)o(king)16 b(sends)g(can)g(use)f (the)h(same)f(three)g(mo)q(des)h(as)f(blo)q(c)o(king)h(sends:)21 b Fp(standard)p FE(,)14 b Fp(ready)75 2136 y FE(and)21 b Fp(synchronous)p FE(.)35 b(These)21 b(carry)g(the)g(same)f(meaning.) 37 b(The)21 b(initiation)i(and)e(completion)h(of)e(a)75 2192 y(standard)e(send)i(do)f(not)f(dep)q(end)j(on)d(the)h(status)f(of) h(a)f(matc)o(hing)h(receiv)o(e.)32 b(A)19 b(ready)g(send)g(can)g(b)q(e) 75 2249 y(initiated)12 b(only)f(if)g(a)g(matc)o(hing)f(receiv)o(e)i (has)e(already)h(b)q(een)h(initiated,)g(otherwise)f(the)g(call)g(is)g (erroneous;)75 2305 y(its)18 b(completion)g(do)q(es)g(not)f(dep)q(end)i (on)f(the)f(status)g(of)g(a)g(matc)o(hing)h(receiv)o(e.)27 b(A)18 b(sync)o(hronous)f(send)75 2362 y(can)f(b)q(e)h(initiated)h(b)q (efore)f(a)f(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)i(initiated,)g (but)e(will)i(complete)f(only)g(after)75 2418 y(a)f(matc)o(hing)f (receiv)o(e)i(has)f(b)q(een)h(initiated,)g(and)f(has)g(started)f (receiving)j(the)e(message)f(generated)h(b)o(y)75 2475 y(the)f(send)h(op)q(eration.)166 2534 y(In)f(all)h(cases)e(the)h(op)q (eration)g(that)f(initiates)i(the)f(comm)o(unication)g(\(send)g(or)f (receiv)o(e\))i(is)f(a)f(lo)q(cal,)75 2591 y(non)o(blo)q(c)o(king)k (call)f(that)f(returns)h(as)f(so)q(on)g(as)g(p)q(ossible.)26 b(A)16 b(non)o(blo)q(c)o(king)i(comm)o(unication)f(call)h(ma)o(y)75 2647 y(fail)e(b)q(ecause)h(b)q(ecause)g(the)e(system)g(has)h(exhausted) g(a)o(v)m(ailable)h(resources)f(\(e.g.,)e(exceeded)j(the)f(limit)75 2704 y(on)h(n)o(um)o(b)q(er)h(of)f(p)q(ending)i(comm)o(unications)f(p)q (er)f(no)q(de\).)27 b(Go)q(o)q(d)17 b(qualit)o(y)h(implemen)o(tations)g (of)f(MPI)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 56 62 56 61 bop 75 -100 a FE(56)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(will)h(set)d (these)i(limits)g(high)g(enough)f(so)g(that)f(a)h(non)o(blo)q(c)o(king) h(comm)o(unication)g(call)g(will)h(fail)f(only)f(in)75 102 y(\\pathological")h(cases.)166 159 y(Non)o(blo)q(c)o(king)g(sends)g (can)f(b)q(e)h(matc)o(hed)f(with)h(blo)q(c)o(king)g(receiv)o(es,)g(and) f(vice-v)o(ersa.)75 287 y Fq(4.8.1)49 b(Communication)17 b(Objects)75 375 y FE(Non)o(blo)q(c)o(king)c(comm)o(unications)g(use)f (opaque)g(comm)o(unication)h(ob)s(jects)f(to)f(iden)o(tify)i(comm)o (unication)75 432 y(op)q(erations)18 b(and)h(matc)o(h)e(the)i(op)q (eration)f(that)f(initiates)j(the)e(comm)o(unication)h(with)f(the)h(op) q(eration)75 488 y(that)e(terminates)i(it.)29 b(These)18 b(are)g(system)g(ob)s(jects)g(that)f(are)h(accessed)h(via)f(a)g (handle.)30 b(An)19 b(opaque)75 545 y(comm)o(unication)f(ob)s(ject)e (iden)o(ti\014es)j(v)m(arious)f(prop)q(erties)f(of)g(a)g(comm)o (unication)h(op)q(eration,)f(suc)o(h)g(as)75 601 y(the)g(\(send)g(or)f (receiv)o(e\))h(bu\013er)g(that)f(is)h(asso)q(ciated)g(with)g(it,)g (its)g(con)o(text,)f(the)g(tag)g(and)h(destination)75 658 y(parameters)c(to)f(b)q(e)j(used)f(for)e(a)i(send,)g(or)f(the)g (tag)g(and)g(source)h(parameters)e(to)h(b)q(e)h(used)g(for)f(a)g (receiv)o(e.)75 714 y(In)j(addition,)g(this)g(ob)s(ject)f(stores)f (information)i(ab)q(out)f(the)g(status)g(of)g(the)g(p)q(ending)i(comm)o (unication)75 771 y(op)q(eration)e(that)g(is)h(p)q(erformed)f(with)h (this)f(ob)s(ject.)75 899 y Fq(4.8.2)49 b(Communication)17 b(initiation)75 987 y FE(W)l(e)h(use)g(the)g(same)g(naming)g(con)o(v)o (en)o(tions)g(as)f(for)g(blo)q(c)o(king)j(comm)o(unication:)26 b(a)17 b(pre\014x)h(of)g Fp(R)f FE(\()p Fp(S)p FE(\))g(is)75 1043 y(used)h(for)f Fp(READY)g FE(\()p Fp(SYNCHRONOUS)p FE(\))f(mo)q(de.)27 b(In)19 b(addition)f(a)g(pre\014x)g(of)f Fp(I)h FE(\(for)e Fp(IMMEDIATE)p FE(\))h(indicates)75 1100 y(that)d(the)i(call)g(is)g(non)o(blo)q(c)o(king.)166 1157 y(Initiate)g(a)f(standard)g(mo)q(de)g(non)o(blo)q(c)o(king)i(comm) o(unication.)75 1262 y Fo(MPI)p 160 1262 14 2 v 16 w(ISEND\(handle,)f (sta)o(rt,)f(count,)h(datat)o(yp)q(e,)g(dest,)h(tag,)d(comm\))117 1340 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati) o(on)d(ob)r(ject)117 1418 y(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1495 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1573 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)117 1650 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1727 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 1805 y(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(\(handle\))166 1930 y FE(Initiate)16 b(a)f(ready)g(mo)q(de)h(non)o(blo)q(c)o(king)g(comm)o(unication.)75 2035 y Fo(MPI)p 160 2035 V 16 w(IRSEND\(handle,)g(sta)o(rt,)f(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm\))117 2114 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject) 117 2191 y(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 2268 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 2346 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t) 117 2423 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in) o(teger\))117 2501 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 2578 y(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(\(handle\))166 2704 y FE(Initiate)16 b(a)f(sync)o(hronous)g(mo)q(de)h(non)o(blo)q(c)o(king)g(comm)o (unication.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 57 63 57 62 bop 75 -100 a Fx(4.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FE(57)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(ISSEND\(handle,)16 b(sta)o(rt,)f(count,)h(datat)o(yp)q(e,)h(dest,)f(tag,)f(comm\))117 126 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o (on)d(ob)r(ject)117 208 y(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 291 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 373 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h (bu\013er)g(elemen)o(t)117 455 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o(teger\))117 538 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 620 y(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(\(handle\))166 748 y FE(Initiate)16 b(a)f(non)o(blo)q(c)o(king)i(receiv)o(e.)75 855 y Fo(MPI)p 160 855 V 16 w(IRECV)e(\(handle,)h(sta)o(rt,)f(count,)h (datat)o(yp)q(e,)h(source,)e(tag,)g(comm\))117 936 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o(on)d (ob)r(ject)117 1018 y(OUT)108 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1101 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1183 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)117 1265 y(IN)155 b Fo(source)468 b Fu(rank)14 b(of)f(source)j(\(in)o (teger\))117 1348 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 1430 y(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(\(handle\))166 1558 y FE(The)22 b(call)g(allo)q(cates)g(a)f(comm)o(unication)h(ob)s(ject)f(and)h(asso)q (ciates)f(it)h(with)f(the)h(handle.)40 b(The)75 1615 y(handle)20 b(can)e(b)q(e)i(used)f(later)f(to)g(query)h(ab)q(out)f(the) h(status)f(of)g(the)g(comm)o(unication)i(or)e(w)o(ait)g(for)g(its)75 1671 y(completion.)166 1731 y(A)11 b(sender)h(should)h(not)e(up)q(date) g(an)o(y)g(part)g(of)g(a)g(send)h(bu\013er)f(after)g(a)g(non)o(blo)q(c) o(king)i(send)f(op)q(eration)75 1787 y(returns,)20 b(un)o(til)g(the)g (send)f(completes.)33 b(A)20 b(receiv)o(er)g(should)g(not)f(access)g (or)g(up)q(date)h(an)o(y)f(part)f(of)h(a)75 1844 y(receiv)o(e)d (bu\013er)f(after)g(a)g(non)o(blo)q(c)o(king)h(receiv)o(e)g(op)q (eration)g(returns,)e(un)o(til)j(the)e(receiv)o(e)h(completes.)75 1986 y Fq(4.8.3)49 b(Communication)17 b(Completion)75 2079 y FE(The)j(completion)h(of)f(a)f(send)i(op)q(eration)f(indicates)h (that)e(the)h(sender)h(is)f(no)o(w)f(free)h(to)f(up)q(date)i(the)75 2135 y(lo)q(cations)i(in)g(the)f(send)h(bu\013er,)h(or)e(an)o(y)g (other)g(lo)q(cation)h(that)e(can)i(b)q(e)g(referenced)g(b)o(y)f(the)g (send)75 2192 y(op)q(eration)d(\(the)g(send)g(op)q(eration)g(itself)h (lea)o(v)o(es)e(the)h(con)o(ten)o(t)f(of)h(the)g(send)g(bu\013er)g(unc) o(hanged\).)31 b(It)75 2248 y(do)q(es)19 b(not)f(indicate)j(that)d(the) h(message)f(has)g(b)q(een)i(receiv)o(ed;)i(rather,)c(it)h(ma)o(y)f(ha)o (v)o(e)h(b)q(een)h(bu\013ered)75 2305 y(b)o(y)f(the)g(comm)o(unication) h(subsystem.)32 b(Ho)o(w)o(ev)o(er,)19 b(if)h(a)f Fp(synchronous)f FE(mo)q(de)h(send)h(w)o(as)e(used,)j(the)75 2361 y(completion)c(of)e (the)g(send)h(op)q(eration)g(indicates)h(that)e(a)g(matc)o(hing)g (receiv)o(e)i(w)o(as)d(initiated,)j(and)f(that)75 2418 y(the)f(message)g(will)i(ev)o(en)o(tually)f(b)q(e)g(receiv)o(ed)g(b)o (y)f(this)h(matc)o(hing)f(receiv)o(e.)166 2478 y(The)20 b(completion)h(of)e(a)g(receiv)o(e)i(op)q(eration)f(indicates)h(that)e (the)h(receiv)o(e)g(bu\013er)g(con)o(tains)g(the)75 2534 y(receiv)o(ed)f(message,)f(and)g(that)f(the)h(status)f(ob)s(ject)h(is)g (set;)h(the)f(receiv)o(er)h(is)f(no)o(w)f(free)h(to)g(access)g(the)75 2591 y(receiv)o(e)c(bu\013er,)f(or)f(an)o(y)g(other)h(lo)q(cation)h (that)e(can)h(b)q(e)g(referenced)h(b)o(y)f(the)g(receiv)o(e)g(op)q (eration.)20 b(It)13 b(do)q(es)75 2647 y(not)g(indicate)i(that)d(the)i (matc)o(hing)f(send)h(op)q(eration)g(has)f(completed)h(\(but)g (indicates,)g(of)f(course,)h(that)75 2704 y(the)h(send)h(w)o(as)e (initiated\).)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 58 64 58 63 bop 75 -100 a FE(58)635 b Fx(SECTION)16 b(4.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(W)l(AIT)f(\(handle,)h(status\))117 122 y Fu(IN)155 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o (on)d(ob)r(ject)117 197 y(OUT)108 b Fo(status)476 b Fu(status)15 b(ob)r(ject)166 322 y FE(A)e(call)h(to)26 b Fo(MPI)p 446 322 V 16 w(W)l(AIT)13 b FE(returns)g(when)g(the)g(send)h(op)q (eration)f(iden)o(ti\014ed)j(b)o(y)26 b Fo(handle)14 b FE(is)g(complete.)75 378 y(If)g(the)f(comm)o(unication)i(ob)s(ject)d (asso)q(ciated)i(with)g(this)g(handle)h(w)o(as)d(created)i(b)o(y)f(a)h (non)o(blo)q(c)o(king)g(send)75 434 y(or)19 b(receiv)o(e)i(call,)h (then)e(the)g(ob)s(ject)f(is)i(deallo)q(cated)g(b)o(y)f(the)g(call)h (to)39 b Fo(MPI)p 1422 434 V 16 w(W)l(AIT)20 b FE(and)g(the)g(handle)75 491 y(b)q(ecomes)c(n)o(ull.)166 547 y(The)g(call)g(returns)f(in)32 b Fo(status)17 b FE(information)f(on)f(the)g(completed)i(op)q(eration.) j(The)c(status)f(ob)s(ject)75 604 y(for)h(a)g(receiv)o(e)i(op)q (eration)f(can)g(b)q(e)g(queried)h(using)f(the)g(functions)g(describ)q (ed)i(in)f(Section)f(4.3.1.)23 b(The)75 660 y(status)15 b(parameter)g(is)i(not)e(used)i(or)e(up)q(dated)i(for)e(a)h(send)g(op)q (eration,)g(and)g(an)g(arbitrary)g(v)m(alue)h(ma)o(y)75 717 y(b)q(e)f(passed)f(for)g(this)h(parameter)e(in)i(the)f(call.)166 849 y Fv(Discussion:)166 899 y Fu(Do)e(w)o(e)h(agree)h(on)e(the)i(last) f(sen)o(tence?)20 b(\(this)14 b(implies)e(some)h(restriction)i(on)e (the)i(implemen)o(tatio)o(n\))75 1085 y Fo(MPI)p 160 1085 V 16 w(TEST)g(\(handle,)h(\015ag,)f(status\))117 1163 y Fu(IN)155 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o (on)d(ob)r(ject)117 1237 y(OUT)124 b Fo(\015ag)502 b Fu(logical)117 1312 y(OUT)108 b Fo(status)476 b Fu(status)15 b(ob)r(ject)166 1437 y FE(A)i(call)h(to)34 b Fo(MPI)p 462 1437 V 16 w(TEST)17 b FE(returns)34 b Fo(\015ag=true)18 b FE(if)g(the)f(op)q(eration)h(iden)o(ti\014ed)h(b)o(y)34 b Fo(handle)18 b FE(is)g(com-)75 1493 y(plete.)37 b(In)21 b(suc)o(h)g(a)f(case,)i(the)e(status)g(ob)s(ject)g(is)h(set)f(to)g(con) o(tain)h(information)g(on)f(the)h(completed)75 1550 y(op)q(eration;)15 b(if)h(the)f(comm)o(unication)h(ob)s(ject)e(w)o(as)h(created)g(b)o(y)g (a)g(non)o(blo)q(c)o(king)h(send)g(or)f(receiv)o(e,)h(then)75 1606 y(it)i(is)g(deallo)q(cated)h(and)e(the)h(handle)h(b)q(ecomes)f(n)o (ull.)28 b(The)18 b(call)g(returns)35 b Fo(\015ag=false)p FE(,)18 b(otherwise.)27 b(In)75 1662 y(suc)o(h)16 b(a)e(case,)h(the)h (v)m(alue)g(of)f(the)g(status)f(ob)s(ject)h(is)h(unde\014ned.)166 1795 y Fv(Discussion:)166 1851 y Fu(W)m(e)11 b(could)h(disp)q(ose)g(of) f(the)i(\015ag)e(parameter)g(and)h(add)f(an)h(additional)e(query)i (function)f(for)h(status)g(ob)r(jects.)166 2047 y Fo(MPI)p 251 2047 V 16 w(TEST)i FE(is)h(a)f(lo)q(cal,)h(non)o(blo)q(c)o(king)g (op)q(eration.)20 b(If)14 b(rep)q(eatedly)i(called)g(for)d(an)h(op)q (eration)h(that)75 2103 y(is)h(enabled,)g(it)f(m)o(ust)g(ev)o(en)o (tually)h(succeed.)166 2160 y(The)11 b(return)g(status)f(ob)s(ject)h (for)f(a)h(receiv)o(e)h(op)q(eration)f(carries)g(information)g(that)g (can)g(b)q(e)g(accessed)75 2216 y(using)k(the)f(functions)h(describ)q (ed)h(in)g(Section)f(4.3.1.)j(The)c(status)g(argumen)o(t)f(is)i(not)f (used)g(or)g(up)q(dated)75 2273 y(for)h(a)f(send)i(op)q(eration)g(and)f (an)g(arbitrary)g(v)m(alue)h(can)f(b)q(e)h(passed)g(for)e(this)i (parameter)e(in)i(the)g(call.)166 2329 y(In)i(m)o(ulti-threaded)h(en)o (vironmen)o(t,)f(the)g(use)g(of)f(a)h(blo)q(c)o(king)h(receiv)o(e)f(op) q(eration)g(\()g Fo(MPI)p 1732 2329 V 15 w(W)l(AIT)p FE(\))75 2386 y(ma)o(y)g(allo)o(w)i(the)f(op)q(erating)g(system)g(to)g (de-sc)o(hedule)i(the)e(blo)q(c)o(k)o(ed)h(thread)f(and)h(sc)o(hedule)g (another)75 2442 y(thread)h(for)f(execution,)j(if)e(suc)o(h)g(is)h(a)o (v)m(ailable.)38 b(The)21 b(use)g(of)g(a)f(non)o(blo)q(c)o(king)j (receiv)o(e)e(op)q(eration)g(\()75 2499 y Fo(MPI)p 160 2499 V 16 w(TEST)p FE(\))15 b(allo)o(ws)h(the)g(user)g(to)f(sc)o (hedule)i(alternativ)o(e)f(activities)h(within)g(a)f(single)h(thread)e (of)g(exe-)75 2555 y(cution.)166 2612 y(Example:)75 2704 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 59 65 59 64 bop 75 -100 a Fx(4.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FE(59)75 45 y Fp(IF\(rank.EQ.0\))123 102 y(THEN)170 158 y(CALL)24 b(MPI_ISEND\(handle,)d(a\(1\),)i(10,)h(MPI_REAL,)f(1,)g (tag,)g(comm,)h(ierr\))170 214 y(****)g(do)f(some)h(computation)e(to)i (mask)f(latency)g(****)170 271 y(CALL)h(MPI_WAIT\(handle,)d(status,)i (ierr\))123 327 y(ELSE)170 384 y(CALL)h(MPI_IRECV\(handle,)d(a\(1\),)i (15,)h(MPI_REAL,)f(0,)g(tag,)g(comm,)h(ierr\))170 440 y(****)g(do)f(some)h(computation)e(to)i(mask)f(latency)g(****)170 497 y(CALL)h(MPI_WAIT\(handle,)d(status,)i(ierr\))75 553 y(END)g(IF)166 676 y FE(The)c(functions)38 b Fo(MPI)p 566 676 14 2 v 16 w(W)l(AIT)18 b FE(and)38 b Fo(MPI)p 906 676 V 16 w(TEST)19 b FE(can)g(b)q(e)g(used)g(to)f(complete)i(b)q (oth)f(sends)g(and)75 733 y(receiv)o(es;)26 b(they)c(will)i(b)q(e)e (also)g(used)h(to)f(complete)g(an)o(y)g(other)g(non)o(blo)q(c)o(king)h (comm)o(unication)g(call)75 789 y(pro)o(vided)16 b(b)o(y)f(MPI.)166 853 y(Non)o(blo)q(c)o(king)g(comm)o(unications)g(cannot)f(b)q(e)g (replaced)i(b)o(y)e(blo)q(c)o(king)h(comm)o(unications,)g(ev)o(en)f(in) 75 910 y(the)h Fp(synchronous)f FE(comm)o(unication)i(mo)q(de.)k (Consider)c(the)f(follo)o(wing)h(example:)75 1040 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1097 y(IF\(rank.EQ.0\))123 1153 y(THEN)170 1210 y(CALL)h(MPI_ISEND\(handle1,)d(a\(1\),)i(count,)g (MPI_REAL,)g(1,)h(tag1,)f(comm,)g(ierr\))170 1266 y(CALL)h (MPI_ISEND\(handle2,)d(a\(2\),)i(count,)g(MPI_REAL,)g(1,)h(tag2,)f (comm,)g(ierr\))170 1322 y(CALL)h(MPI_WAIT\(handle1,)d(status,)i (ierr\))170 1379 y(CALL)h(MPI_WAIT\(handle2,)d(status,)i(ierr\))123 1435 y(ELSE)170 1492 y(CALL)h(MPI_IRECV\(handle2,)d(a\(2\),)i(count,)g (MPI_REAL,)g(0,)h(tag2,)f(comm,)g(ierr\))170 1548 y(CALL)h (MPI_IRECV\(handle1,)d(a\(1\),)i(count,)g(MPI_REAL,)g(0,)h(tag1,)f (comm,)g(ierr\))170 1605 y(CALL)h(MPI_WAIT\(handle2,)d(status,)i (ierr\))170 1661 y(CALL)h(MPI_WAIT\(handle1,)d(status,)i(ierr\))75 1718 y(END)g(IF)166 1841 y FE(The)16 b(co)q(de)g(is)g(guaran)o(teed)f (to)g(execute)h(correctly)l(,)g(ev)o(en)g(in)g(an)g(implemen)o(tation)h (that)d(do)q(es)i(not)75 1897 y(bu\013er)g(messages)f(\(the)h(only)g (resource)g(requiremen)o(t)h(is)f(the)g(abilit)o(y)h(to)e(ha)o(v)o(e)h (t)o(w)o(o)f(p)q(ending)i(comm)o(u-)75 1954 y(nications\).)30 b(On)19 b(the)g(other)f(hand,)h(the)f(co)q(de)i(ma)o(y)d(deadlo)q(c)o (k)j(if)e(the)h(non)o(blo)q(c)o(king)h(op)q(erations)e(are)75 2010 y(replaced)e(b)o(y)g(blo)q(c)o(king)g(ones.)75 2174 y Fq(4.8.4)49 b(Multiple)17 b(Completions)75 2274 y FE(It)23 b(is)h(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)g(to)e(w)o(ait)h(for)g (the)g(completion)i(of)d(an)o(y)h(or)g(all)h(the)g(op)q(erations)f(in) 75 2331 y(a)e(set,)i(rather)e(than)g(ha)o(ving)h(to)f(w)o(ait)h(for)f (a)g(sp)q(eci\014c)i(message.)39 b(A)22 b(call)h(to)42 b Fo(MPI)p 1598 2331 V 16 w(W)l(AIT)l(ANY)22 b FE(or)75 2387 y Fo(MPI)p 160 2387 V 16 w(TEST)l(ANY)c FE(can)f(b)q(e)g(used)h (to)e(w)o(ait)g(for)h(the)g(completion)h(of)e(one)h(out)f(of)h(sev)o (eral)g(op)q(erations;)g(a)75 2443 y(call)f(to)30 b Fo(MPI)p 314 2443 V 16 w(W)l(AIT)l(ALL)15 b FE(or)30 b Fo(MPI)p 687 2443 V 16 w(TEST)l(ALL)15 b FE(can)h(b)q(e)f(used)h(to)f(w)o(ait)g (for)f(all)j(p)q(ending)g(op)q(erations)e(in)75 2500 y(a)g(list.)166 2640 y Fv(Discussion:)166 2704 y Fr(MPI)p 243 2704 13 2 v 14 w(TEST)m(ALL)i Fv(is)e(new.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 60 66 60 65 bop 75 -100 a FE(60)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(W)l(AIT)l(ANY)g(\(count,)g(a)o(rra)o(y)p 640 45 V 14 w(of)p 691 45 V 16 w(handles,)h(index,)f(status\))117 135 y Fu(IN)155 b Fo(count)482 b Fu(list)14 b(length)f(\(in)o(teger\)) 117 237 y(IN)155 b Fo(a)o(rra)o(y)p 416 237 V 15 w(of)p 468 237 V 16 w(handles)285 b Fu(arra)o(y)14 b(of)f(handles)h(to)g(comm) o(unication)c(ob)r(jects)117 338 y(OUT)108 b Fo(index)489 b Fu(index)12 b(of)f(handle)g(for)g(op)q(eration)h(that)f(completed)g (\(in)o(teger\))117 439 y(OUT)108 b Fo(status)476 b Fu(return)15 b(status)g(ob)r(ject)166 577 y FE(Blo)q(c)o(ks)j(un)o(til)g(one)g(of)f (the)g(op)q(erations)h(asso)q(ciated)g(with)f(the)h(comm)o(unication)g (handles)h(in)f(the)75 633 y(arra)o(y)13 b(has)i(completed.)21 b(Returns)15 b(the)g(index)h(of)e(that)g(handle)i(in)f(the)g(arra)o(y)l (,)f(and)g(returns)h(the)g(status)75 690 y(of)g(that)f(op)q(eration)i (in)g(the)f(ob)s(ject)g(asso)q(ciated)g(with)h(the)f(status.)166 759 y(The)i(successful)i(execution)f(of)33 b Fo(MPI)p 830 759 V 16 w(W)l(AIT)l(ANY\(count,)19 b(a)o(rra)o(y)p 1297 759 V 15 w(of)p 1349 759 V 16 w(handles,)f(index,)g(status\))h FE(has)75 815 y(the)d(same)f(e\013ect)h(as)f(the)h(successful)h (execution)f(of)31 b Fo(MPI)p 1087 815 V 16 w(W)l(AIT\(a)o(rra)o(y)p 1329 815 V 14 w(of)p 1380 815 V 16 w(handles[i],)18 b(status\))p FE(,)f(where)75 872 y Fo(i)j FE(is)g(the)h(v)m(alue)g(returned)f(b)o(y) 40 b Fo(index)p FE(.)c(In)20 b(particular,)i(the)e(asso)q(ciated)g (comm)o(unication)h(ob)s(ject)e(is)75 928 y(deallo)q(cated,)26 b(and)d(the)f(handle)i(to)e(it)h(in)47 b Fo(a)o(rra)o(y)p 969 928 V 14 w(of)p 1020 928 V 16 w(handles)25 b FE(is)e(set)f(to)h(n)o (ull.)43 b(If)23 b(more)g(then)g(one)75 985 y(op)q(eration)16 b(is)g(enabled)h(and)e(can)h(terminate,)f(one)g(is)h(arbitrarily)g(c)o (hosen.)21 b(There)16 b(is)g(no)f(requiremen)o(t)75 1041 y(that)f(the)i(c)o(hoice)g(satisfy)f(an)o(y)g(fairness)g(criterion.)166 1187 y Fv(Discussion:)166 1250 y Fu(If)e(statemen)o(t)h(of)f(fairness)i (is)f(c)o(hanged)g(in)f(Section)i(4.4)d(then)j(will)d(need)j(to)f(c)o (hange)g(here,)h(to)q(o.)75 1449 y Fo(MPI)p 160 1449 V 16 w(TEST)l(ANY\()h(count,)g(a)o(rra)o(y)p 641 1449 V 15 w(of)p 693 1449 V 16 w(handles,)g(index,)g(status\))117 1539 y Fu(IN)155 b Fo(count)482 b Fu(list)14 b(length)f(\(in)o(teger\)) 117 1641 y(IN)155 b Fo(a)o(rra)o(y)p 416 1641 V 15 w(of)p 468 1641 V 16 w(handles)285 b Fu(arra)o(y)14 b(of)f(handles)h(to)g (comm)o(unication)c(ob)r(jects)117 1742 y(OUT)108 b Fo(index)489 b Fu(index)15 b(of)e(handle)h(for)g(op)q(eration)h(that)f(completed,)g (or)g(-1)g(if)905 1798 y(none)g(completed)g(\(in)o(teger\))117 1900 y(OUT)108 b Fo(status)476 b Fu(return)15 b(status)g(ob)r(ject)166 2037 y FE(Causes)d(either)h(one)f(or)g(none)h(of)f(the)g(op)q(erations) g(asso)q(ciated)h(with)f(the)h(comm)o(unication)g(handles)75 2094 y(to)d(return.)19 b(In)12 b(the)f(former)f(case,)i(it)f(returns)g (the)g(index)i(of)d(that)h(handle)h(in)g(the)f(arra)o(y)l(,)g(and)g (returns)g(the)75 2150 y(status)g(of)h(that)f(op)q(eration)h(in)h(the)f (ob)s(ject)f(asso)q(ciated)h(with)h(the)f(status.)18 b(The)12 b(comm)o(unication)h(ob)s(ject)75 2206 y(is)18 b(deallo)q(cated)g(and)f(the)g(handle)i(to)d(it)h(in)35 b Fo(a)o(rra)o(y)p 956 2206 V 15 w(of)p 1008 2206 V 16 w(handles)19 b FE(is)e(set)g(to)f(n)o(ull.)27 b(In)18 b(the)f(latter)g(case,)g(it)75 2263 y(returns)12 b(a)g(v)m(alue)i(of)e (-1)g(in)25 b Fo(index)13 b FE(and)25 b Fo(status)14 b FE(is)f(unde\014ned.)21 b(Lik)o(e)13 b Fo(MPI)p 1322 2263 V 16 w(TEST)p FE(,)f(this)h(is)g(a)f(non)o(blo)q(c)o(king)75 2319 y(op)q(eration,)j(that)f(returns)h(immediately;)i(and)e(if)g(one)h (busy-w)o(aits)f(with)g Fo(MPI)p 1444 2319 V 16 w(TEST)l(ANY)p FE(,)h(and)f(some)75 2376 y(op)q(eration)g(on)h(the)f(list)h(is)g (enabled,)g(then)f Fo(MPI)p 909 2376 V 16 w(TEST)l(ANY)i FE(will)f(ev)o(en)o(tually)h(return)30 b Fo(\015ag)15 b(=)h(true)p FE(.)166 2521 y Fv(Discussion:)166 2584 y Fu(Could)g(ha)o(v)o(e)h(a)33 b Fr(\015ag)17 b Fu(parameter,)g(rather) h(than)f(using)f(index)h(=)g(-1)g(for)f(unsuccessful)j(return.)28 b(This)17 b(is)75 2634 y(more)c(consisten)o(t)i(and)f(more)f(elegan)o (t,)g(but)h(requires)h(an)f(additional)e(parameter.)166 2704 y(If)h(w)o(e)i(stic)o(k)f(with)f(curren)o(t)j(v)o(ersion,)d(ma)o (y)f(replace)j(-1)e(b)o(y)h(an)g(MPI)g(constan)o(t.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 61 67 61 66 bop 75 -100 a Fx(4.8.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b FE(61)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(W)l(AIT)l(ALL\()14 b(count,)i(a)o(rra)o(y)p 626 45 V 15 w(of)p 678 45 V 16 w(handles,)h(a)o(rra)o(y)p 958 45 V 14 w(of)p 1009 45 V 16 w(status\))117 122 y Fu(IN)155 b Fo(count)482 b Fu(lists)14 b(length)g(\(in)o(teger\))117 196 y(IN)155 b Fo(a)o(rra)o(y)p 416 196 V 15 w(of)p 468 196 V 16 w(handles)285 b Fu(arra)o(y)14 b(of)f(handles)h(to)g(comm)o(unication)c(ob)r(jects) 117 270 y(OUT)108 b Fo(a)o(rra)o(y)p 416 270 V 15 w(of)p 468 270 V 16 w(status)313 b Fu(arra)o(y)14 b(of)f(status)i(ob)r(jects) 166 395 y FE(Blo)q(c)o(ks)e(un)o(til)h(all)g(comm)o(unication)g(op)q (erations)f(asso)q(ciated)g(with)g(handles)h(in)g(the)f(list)h (complete,)75 451 y(and)j(return)g(the)g(status)f(of)h(all)h(these)f (op)q(erations.)26 b(Both)16 b(arra)o(ys)g(ha)o(v)o(e)h(the)g(same)f(n) o(um)o(b)q(er)i(of)e(v)m(alid)75 508 y(en)o(tries.)22 b(The)32 b Fo(i)p FE(-th)16 b(en)o(try)f(in)32 b Fo(a)o(rra)o(y)p 721 508 V 15 w(of)p 773 508 V 16 w(status)18 b FE(is)e(set)g(to)f(the)h (return)f(status)g(of)h(the)31 b Fo(i)p FE(-th)16 b(op)q(eration.)75 564 y(All)g(comm)o(unication)g(ob)s(jects)f(are)g(deallo)q(cated)h(and) g(all)g(handles)g(are)f(set)g(to)g(n)o(ull.)75 668 y Fo(MPI)p 160 668 V 16 w(TEST)l(ALL\(count,)h(a)o(rra)o(y)p 613 668 V 14 w(of)p 664 668 V 16 w(handles,)h(\015ag,)d(a)o(rra)o(y)p 1040 668 V 15 w(of)p 1092 668 V 16 w(status\))117 745 y Fu(IN)155 b Fo(count)482 b Fu(lists)14 b(length)g(\(in)o(teger\))117 819 y(IN)155 b Fo(a)o(rra)o(y)p 416 819 V 15 w(of)p 468 819 V 16 w(handles)285 b Fu(arra)o(y)14 b(of)f(handles)h(to)g(comm)o (unication)c(ob)r(jects)117 893 y(OUT)108 b Fo(\015ag)518 b Fu(\(logical\))117 967 y(OUT)108 b Fo(a)o(rra)o(y)p 416 967 V 15 w(of)p 468 967 V 16 w(status)313 b Fu(arra)o(y)14 b(of)f(status)i(ob)r(jects)166 1091 y FE(Causes)f(either)g(all)h(or)f (none)g(of)f(the)i(op)q(erations)f(asso)q(ciated)g(with)g(the)g(comm)o (unication)h(handles)75 1148 y(to)i(complete.)29 b(It)18 b(returns)36 b Fo(\015ag)18 b(=)h(true)f FE(if)h(all)g(comm)o (unications)f(asso)q(ciated)g(with)h(handles)g(in)g(the)75 1204 y(arra)o(y)10 b(ha)o(v)o(e)h(completed.)19 b(In)12 b(this)g(case,)f(eac)o(h)h(status)e(en)o(try)h(is)h(set)f(to)f(the)i (status)e(of)h(the)g(corresp)q(onding)75 1261 y(comm)o(unication.)20 b(All)15 b(comm)o(unication)g(ob)s(jects)e(are)h(deallo)q(cated,)h(and) f(all)h(handles)g(are)f(set)g(to)f(n)o(ull.)75 1317 y(Otherwise,)30 b Fo(\015ag)14 b(=)h(false)g FE(is)g(returned)g(and)g(the)g(v)m(alues)h (of)e(the)h(status)e(en)o(tries)i(are)g(unde\014ned.)21 b(This)75 1374 y(is)14 b(a)f(non)o(blo)q(c)o(king)i(op)q(eration)e (that)g(returns)g(immediately;)j(if)e(one)f(busy)h(w)o(aits)f(with)h Fo(MPI)p 1660 1374 V 15 w(TEST)l(ALL)p FE(,)75 1430 y(and)19 b(all)h(op)q(erations)f(in)g(the)g(list)h(b)q(ecome)f(enabled,)i(then)e Fo(MPI)p 1219 1430 V 16 w(TEST)l(ALL)g FE(will)h(ev)o(en)o(tually)g (return)75 1487 y Fo(\015ag)15 b(=)h(true)p FE(.)166 1543 y(Example:)75 1687 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 1744 y(IF\(rank)g(<)h(2\))123 1800 y(THEN)170 1857 y(CALL)g(MPI_ISEND\(handle,)d(a,)j(n,)f(MPI_REAL,)g(2,)h(tag,)f (comm,)g(ierr\))170 1913 y(****)h(do)f(some)h(computation)e(to)i(mask)f (latency)g(****)170 1970 y(CALL)h(MPI_WAIT\(handle,)d(status,)i(ierr\)) 123 2026 y(ELSE)95 b(!)23 b(rank.EQ.2)170 2083 y(CALL)h (MPI_IRECV\(handle_list\(0\))o(,)d(a,)j(n,)f(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 2139 y(CALL)h(MPI_IRECV\(handle_list\(1\))o(,)d(b,) j(n,)f(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))170 2195 y(****)h(do)f(some)h(computation)e(to)i(mask)f(latency)g(****)170 2252 y(CALL)h(MPI_WAITANY\(2,)e(handle_list,)g(index,)h(status,)g (ierr\))170 2308 y(IF\(index.EQ.0\))218 2365 y(THEN)266 2421 y(****)g(handle)g(message)g(from)g(process)g(0)h(****)266 2478 y(CALL)f(MPI_WAIT\(handle_list\(1\),)e(status,)i(ierr\))266 2534 y(****)g(handle)g(message)g(from)g(process)g(1)h(****)218 2591 y(ELSE)266 2647 y(****)f(handle)g(message)g(from)g(process)g(1)h (****)266 2704 y(CALL)f(MPI_WAIT\(handle_list\(0\),)e(status,)i(ierr\)) -32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 62 68 62 67 bop 75 -100 a FE(62)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)266 45 y Fp(****)23 b(handle)g(message)g(from)g(process)g(0)h(****)170 102 y(END)g(IF)75 158 y(END)f(IF)166 371 y FE(The)16 b(calls)h(in)o(tro)q (duced)g(in)g(this)g(subsection)f(can)h(b)q(e)f(used)h(to)e(w)o(ait)h (or)f(test)g(for)h(the)g(completion)75 428 y(of)d(an)h(arbitrary)f(mix) h(of)f(non)o(blo)q(c)o(king)i(comm)o(unication)g(calls,)f(including)j (a)c(mix)h(sends)g(and)g(receiv)o(es)75 484 y(and)h(of)g(an)o(y)g (additional)i(non)o(blo)q(c)o(king)f(comm)o(unication)g(calls)g(pro)o (vided)g(b)o(y)f(MPI.)166 571 y(The)40 b Fo(status)22 b FE(argumen)o(t)e(in)g Fo(MPI)p 768 571 14 2 v 16 w(W)l(AIT)l(ANY)h FE(and)f Fo(MPI)p 1181 571 V 16 w(W)l(AIT)l(ALL)g FE(and)g(the)40 b Fo(a)o(rra)o(y)p 1697 571 V 15 w(of)p 1749 571 V 16 w(status)75 627 y FE(argumen)o(t)20 b(in)h Fo(MPI)p 427 627 V 16 w(W)l(AIT)l(ALL)f FE(and)h Fo(MPI)p 828 627 V 16 w(TEST)l(ALL)f FE(are)h(not)f(used)h(or)f(up)q(dated)i(if)f(the)f (complet-)75 684 y(ing)f(comm)o(unication)h(is)g(a)e(send.)32 b(If)19 b(all)h(op)q(erations)f(in)g(the)g(list)h(pro)o(vided)g(b)o(y)f (the)g(argumen)o(t)37 b Fo(a)o(r-)75 740 y(ra)o(y)p 136 740 V 15 w(of)p 188 740 V 16 w(handles)14 b FE(are)d(send)h(op)q (erations,)h(then)f(an)f(arbitrary)g(v)m(alue)i(can)f(b)q(e)g(passed)g (to)f(the)24 b Fo(status)14 b FE(\(resp.)75 796 y Fo(a)o(rra)o(y)p 173 796 V 15 w(of)p 225 796 V 16 w(status)p FE(\))i(argumen)o(t.)166 959 y Fv(Discussion:)166 1039 y Fu(Is)e(the)h(last)e(paragraph)h (agreed?)166 1118 y(W)m(e)d(ma)o(y)e(w)o(an)o(t)i(to)g(allo)o(w)e(n)o (ull)h(p)q(oin)o(ters)i(in)f(an)g(arra)o(y)p 1007 1118 13 2 v 15 w(of)p 1056 1118 V 14 w(handles)g(This)h(will)d(allo)o(w)h (to)h(lo)q(op)f(with)h(the)h(same)75 1168 y(arra)o(y)p 172 1168 V 15 w(of)p 221 1168 V 14 w(handles,)i(un)o(til)g(all)f(comm)o (uni)o(cations)f(ha)o(v)o(e)i(completed.)k(On)d(the)f(other)h(hand,)f (this)g(will)f(encourage)75 1218 y(p)q(oten)o(tially)g(ine\016cien)o(t) h(co)q(de.)166 1304 y(I)j(added)h(the)h(requiremen)o(t)e(that)h(the)g (handed)g(to)g(a)f(comm)o(unicatio)o(n)e(that)j(completes)f(is)g(set)i (to)e(n)o(ull.)75 1361 y(This)d(prev)o(en)o(ts)h(the)g(o)q(ccurrence)h (of)d(dangling)g(references,)j(but)e(add)g(some)f(o)o(v)o(erhead.)75 1757 y Fy(4.9)59 b(Prob)r(e)20 b(and)f(Cancel)75 1917 y FE(The)27 b Fo(MPI)p 265 1917 14 2 v 15 w(PROBE)15 b FE(and)26 b Fo(MPI)p 624 1917 V 16 w(IPROBE)14 b FE(op)q(erations)f (allo)o(w)h(incoming)g(messages)f(to)f(b)q(e)i(c)o(hec)o(k)o(ed)g(for,) 75 1973 y(without)i(actually)h(receiving)h(them.)k(The)17 b(user)f(can)g(then)h(decide)g(ho)o(w)f(to)g(receiv)o(e)h(them,)f (based)g(on)75 2030 y(the)g(information)h(returned)f(b)o(y)h(the)f (prob)q(e)h(\(basically)l(,)h(the)e(information)g(returned)h(b)o(y)33 b Fo(status)p FE(\).)24 b(In)75 2086 y(particular,)16 b(the)g(user)g(ma)o(y)f(allo)q(cate)h(memory)f(for)g(the)h(receiv)o(e)h (bu\013er,)e(according)h(to)f(the)h(length)g(of)75 2143 y(the)f(prob)q(ed)h(message.)166 2229 y(The)31 b Fo(MPI)p 360 2229 V 16 w(CANCEL)16 b FE(op)q(eration)g(allo)o(ws)f(p)q(ending)j (comm)o(unications)e(to)f(b)q(e)h(cancelled.)23 b(This)16 b(is)75 2286 y(required)k(for)f(clean)o(up.)33 b(P)o(osting)19 b(a)g(send)h(or)f(a)g(receiv)o(e)h(ties)f(up)h(user)f(resources)h (\(send)f(or)g(receiv)o(e)75 2342 y(bu\013ers\),)14 b(and)i(a)f(cancel) h(ma)o(y)f(b)q(e)g(needed)i(to)d(free)i(these)f(resources)g(gracefully) l(.)166 2504 y Fv(Discussion:)166 2591 y Fu(Made)f(one)h(c)o(hange)f (from)e(appro)o(v)o(ed)i(draft)g(on)g(prob)q(e:)19 b(remo)o(v)o(ed)13 b(the)i(datat)o(yp)q(e)f(argumen)o(t)f(from)g(prob)q(e.)75 2647 y(If)18 b(generalized)h(datat)o(yp)q(e)f(are)g(adopted,)h(then)g (it)f(seems)g(more)f(natural)h(to)g(pro)o(vide)g(the)h(argumen)o(t)e (in)g(the)75 2704 y(status)e(deco)q(de)g(function.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 63 69 63 68 bop 75 -100 a Fx(4.9.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b FE(63)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(IPROBE\(source,)16 b(tag,)f(comm,)d(\015ag,)j(status\))117 125 y Fu(IN)155 b Fo(source)468 b Fu(source)15 b(rank,)f(or)27 b Fr(MPI)p 1281 125 13 2 v 15 w(ANY)p 1381 125 V 14 w(SOURCE)13 b Fu(\(in)o(teger\))117 207 y(IN)155 b Fo(tag)526 b Fu(tag)14 b(v)n(alue)f(or)28 b Fr(MPI)p 1226 207 V 14 w(ANY)p 1325 207 V 15 w(T)m(A)o(G)12 b Fu(\(in)o(teger\))117 288 y(IN)155 b Fo(comm)470 b Fu(handle)14 b(to)g(comm)o(unicator)117 369 y(OUT)108 b Fo(\015ag)518 b Fu(\(logical\))117 451 y(OUT)108 b Fo(status)476 b Fu(status)15 b(ob)r(ject)166 578 y Fo(MPI)p 251 578 14 2 v 16 w(IPROBE)j FE(returns)34 b Fo(\015ag)17 b(=)g(true)h FE(if)f(there)h(is)f(a)g(message)g(that)f (can)h(b)q(e)h(receiv)o(ed)g(and)g(that)75 634 y(matc)o(hes)e(the)g (pattern)f(sp)q(eci\014ed)j(b)o(y)e(the)g(parameters)31 b Fo(source)p FE(,)i Fo(tag)p FE(,)16 b(and)32 b Fo(comm)p FE(.)20 b(It)c(returns)32 b Fo(\015ag)75 691 y(=)19 b(false)p FE(,)g(otherwise.)31 b(If)38 b Fo(MPI)p 629 691 V 16 w(IPROBE)19 b FE(returns)38 b Fo(\015ag)18 b(=)h(true)p FE(,)h(then)f(the)g(source)g(and)g(tag)f(of)g(the)75 747 y(message)d(matc)o(hed)h(are)g(returned)g(in)g(the)g(status)f(ob)s (ject.)21 b(These)c(are)e(the)h(same)g(v)m(alues)g(that)g(w)o(ould)75 804 y(ha)o(v)o(e)f(b)q(een)h(returned)g(b)o(y)f(a)g(call)h(to)30 b Fo(MPI)p 812 804 V 16 w(RECV)16 b FE(executed)g(at)e(the)i(same)e(p)q (oin)o(t)i(in)g(the)f(program.)166 863 y(These)22 b(v)m(alues)g(can)g (b)q(e)g(deco)q(ded)h(from)e(the)g(return)h(status)e(ob)s(ject)h(using) i(the)e(status)g(query)75 920 y(functions)16 b(describ)q(ed)h(in)f (Section)g(4.3.1.)i(The)e(return)f(status)f(ob)s(ject)h(is)h (unde\014ned)h(if)31 b Fo(\015ag=false)p FE(.)166 979 y(The)16 b(call)h(to)31 b Fo(MPI)p 501 979 V 16 w(PROBE)17 b FE(do)q(es)f(not)g(carry)f(a)h(datat)o(yp)q(e)f(argumen)o(t:)20 b(the)c(message)g(t)o(yp)q(e)g(ma)o(y)75 1036 y(not)k(b)q(e)g(kno)o(wn) g(when)h(the)f(prob)q(e)g(is)h(executed.)35 b(Accordingly)l(,)23 b(the)d(prob)q(e)h(call)g(do)q(es)f(not)g(return)75 1092 y(in)g(the)f(status)f(ob)s(ject)h(the)g(length)g(of)g(the)g(prob)q(ed)h (message,)f(and)g Fo(MPI)p 1382 1092 V 16 w(GET)p 1486 1092 V 17 w(COUNT)g FE(cannot)g(b)q(e)75 1149 y(used)c(after)e(a)g (call)i(to)f Fo(MPI)p 542 1149 V 15 w(PROBE)i FE(to)d(query)h(on)g(the) g(length)g(of)g(the)g(prob)q(ed)g(message.)19 b(\(Remem)o(b)q(er)75 1205 y(that)f(the)g(length)i(of)e(a)g(message,)g(in)i(elemen)o(ts,)f (dep)q(ends)h(on)f(the)f(t)o(yp)q(es)h(of)f(these)h(elemen)o(ts;)h (some)75 1262 y(implemen)o(tations)e(ma)o(y)f(not)g(carry)f(in)j(the)e (transmitted)g(message)f(t)o(yp)q(e)i(information.\))26 b(Instead,)17 b(a)75 1318 y(di\013eren)o(t)g(status)g(deco)q(ding)h (function)g(that)e(pro)o(vides)i(the)f(datat)o(yp)q(e)g(as)f(argumen)o (t)h(can)g(b)q(e)g(used)h(in)75 1375 y(this)e(case.)75 1481 y Fo(MPI)p 160 1481 V 16 w(PROBE)p 324 1481 V 17 w(COUNT\(status,)i(datat)o(yp)q(e,)e(count\))117 1562 y Fu(IN)155 b Fo(status)476 b Fu(status)15 b(ob)r(ject)117 1643 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(message)14 b(datat)o(yp)q(e)117 1724 y(OUT)108 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(message)h(\(in)o (teger\))166 1852 y FE(Returns)k(the)g(length)h(of)e(the)h(message)g (that)f(w)o(as)g(last)h(prob)q(ed)h(with)f Fo(status)i FE(as)d(an)h(argumen)o(t.)75 1908 y(The)d(call)i(is)e(erroneous)h(if)30 b Fo(datat)o(yp)q(e)17 b FE(do)q(es)f(not)e(matc)o(h)h(the)g(t)o(yp)q (e)h(of)e(the)i(message.)166 2044 y Fv(Discussion:)166 2097 y Fu(Message)d(passing)f(in)g(MPI)g(can)h(b)q(e)g(implem)o(en)o (ted)d(without)i(app)q(ending)g(t)o(yp)q(e)h(information)c(to)j (messages.)75 2146 y(A)h(message)h(is)f(merely)f(a)h(string)h(of)f(b)o (ytes)h(and)f(the)h(in)o(terpretation)g(of)e(these)j(b)o(ytes)f(in)o (to)f(a)g(sequence)j(of)c(t)o(yp)q(ed)75 2196 y(elemen)o(ts)18 b(is)f(done)h(using)g(the)g(datat)o(yp)q(e)g(information)d(pro)o(vided) j(b)o(y)g(the)g(lo)q(cal)f(comm)o(unication)d(call.)30 b(The)75 2246 y(abilit)o(y)17 b(to)i(use)h(suc)o(h)g(an)f(implemen)o (tation)d(strategy)k(is)f(deemed)g(to)g(b)q(e)g(an)g(imp)q(ortan)o(t)f (goal.)32 b(In)19 b(suc)o(h)h(an)75 2296 y(implemen)o(tatio)o(n,)14 b(when)j(a)f(message)g(arriv)o(es,)h(it)f(is)g(not)g(kno)o(wn)g(ho)o(w) g(man)o(y)e(elemen)o(ts)j(it)f(con)o(tains,)g(or)g(ev)o(en)75 2346 y(ho)o(w)j(m)o(uc)o(h)f(storage)h(is)g(needed)i(to)e(receiv)o(e)h (that)f(message)g(\(b)q(ecause)i(of)e(p)q(ossible)g(represen)o(tation)i (con)o(v)o(er-)75 2395 y(sion)c(in)g(a)g(heterogeneous)j(en)o(vironmen) o(t\).)27 b(The)18 b(length)g(of)e(the)i(incoming)d(message)j(can)f(b)q (e)h(computed)f(b)o(y)75 2445 y Fr(MPI)p 152 2445 13 2 v 14 w(PROBE)p 304 2445 V 14 w(LEN)d Fu(from)e(the)j(b)o(yte)f (length)g(of)f(the)i(message)e(and)h(from)e(the)j(datat)o(yp)q(e)f (parameter.)166 2498 y(The)c(curren)o(t)i(solution)d(sa)o(v)o(es)i(us)f (the)h(need)g(for)f(one)g(additional)e(w)o(ord)i(p)q(er)h(message)f (that)g(w)o(ould)f(otherwise)75 2548 y(b)q(e)15 b(needed)g(to)f (transfer)g(the)h(message)f(length)f(\(in)h(elemen)o(ts\))g(with)f(the) i(message.)166 2601 y Fv(Need)g(b)q(e)h(discussed)d(b)o(y)j(MPIF)166 2654 y Fu(W)m(e)g(ma)o(y)e(w)o(an)o(t)i(a)g(call)g(to)33 b Fr(MPI)p 699 2654 V 14 w(PROBE)p 851 2654 V 13 w(COUNT)16 b Fu(to)g(b)q(e)h(signi\014can)o(t)f(not)g(just)h(after)g(prob)q(e,)g (but)g(also)75 2704 y(after)d(receiv)o(e,)h(for)e(consistency)j (reasons.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 64 70 64 69 bop 75 -100 a FE(64)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fu(It's)11 b(tempting)f(to)h(merge)22 b Fr(MPI)p 667 45 13 2 v 15 w(GET)p 763 45 V 14 w(COUNT)11 b Fu(and)22 b Fr(MPI)p 1096 45 V 14 w(PROBE)p 1248 45 V 14 w(COUNT)10 b Fu(in)o(to)h(one)g (call.)17 b(E.g.,)10 b(alw)o(a)o(ys)75 95 y(pro)o(vide)15 b(a)g(datat)o(yp)q(e)g(argumen)o(t,)f(ev)o(en)h(though)g(it)g(is)g (redundan)o(t)h(after)f(a)g(receiv)o(e.)23 b(But)15 b(then)h(w)o(e)f (imp)q(ose)f(an)75 145 y(additional)e(burden)j(on)e(regular)h(receiv)o (es.)166 195 y(W)m(e)h(ma)o(y)f(w)o(an)o(t)i(to)g(relax)g(t)o(yp)q(e)g (restrictions)h(so)f(that)32 b Fr(datat)o(yp)q(e)17 b Fu(=)32 b Fr(MPI)p 1373 195 V 15 w(BYTE)15 b Fu(could)h(b)q(e)g(used)h (as)f(an)75 245 y(argumen)o(t)e(to)30 b Fr(MPI)p 406 245 V 14 w(PROBE)p 558 245 V 14 w(COUNT)p Fu(,)13 b(ev)o(en)o(t)j(if)e (the)i(message)f(w)o(as)g(sen)o(t)h(with)e(another)i(datat)o(yp)q(e;)f (this)g(will)75 295 y(allo)o(w)d(to)i(coun)o(t)g(the)h(n)o(um)o(b)q(er) e(of)g(b)o(ytes)i(receiv)o(ed,)f(and)g(use)h(this)f(coun)o(t)g(in)f(a)h (mallo)q(c.)166 435 y FE(A)f(subsequen)o(t)i(receiv)o(e)f(executed)g (with)g(the)g(same)f(con)o(text,)g(and)g(the)h(source)f(and)h(tag)f (returned)75 491 y(b)o(y)i(the)h(call)g(to)30 b Fo(MPI)p 456 491 14 2 v 16 w(IPROBE)16 b FE(will)h(receiv)o(e)f(the)f(message)g (that)g(w)o(as)f(matc)o(hed)h(b)o(y)h(the)f(prob)q(e,)g(if)h(no)75 548 y(other)11 b(in)o(terv)o(ening)h(receiv)o(e)g(o)q(ccurred)g(after)e (the)h(prob)q(e.)19 b(If)11 b(the)g(receiving)i(pro)q(cess)e(is)h(m)o (ulti-threaded,)75 604 y(it)j(is)h(the)f(user)h(resp)q(onsibilit)o(y)h (to)e(ensure)h(that)e(the)i(last)f(condition)h(holds.)166 737 y Fv(Discussion:)166 788 y Fu(MPI)f(guaran)o(tees)g(that)g (successiv)o(e)i(messages)e(sen)o(t)h(from)d(a)h(source)i(to)f(a)f (destination)h(within)e(the)j(same)75 838 y(con)o(text)21 b(are)g(receiv)o(ed)h(in)e(the)h(order)g(they)g(are)g(sen)o(t.)39 b(Th)o(us,)22 b(MPI)e(m)o(ust)g(supp)q(ort,)i(either)g(explicitly)d(or) 75 888 y(implicitly)l(,)13 b(a)i(FIF)o(O)g(structure)j(to)d(manage)e (messages)j(b)q(et)o(w)o(een)h(eac)o(h)e(pair)g(of)g(pro)q(cesses,)j (for)d(eac)o(h)h(con)o(text.)75 937 y Fr(MPI)p 152 937 13 2 v 14 w(PROBE)e Fu(returns)j(information)c(on)i(the)h(\014rst)h (matc)o(hing)c(message)j(in)f(this)g(FIF)o(O;)h(this)f(will)f(also)h(b) q(e)h(the)75 987 y(message)10 b(receiv)o(ed)h(b)o(y)f(the)h(\014rst)g (subsequen)o(t)h(receiv)o(e)f(with)f(the)h(same)e(source,)j(tag)d(and)h (con)o(text)h(as)g(the)f(message)75 1037 y(matc)o(hed)j(b)o(y)28 b Fr(MPI)p 391 1037 V 14 w(PROBE)p Fu(.)75 1224 y Fo(MPI)p 160 1224 14 2 v 16 w(PROBE\()16 b(source,)f(tag,)g(comm,)e(datat)o(yp)q (e,)j(status\))117 1302 y Fu(IN)155 b Fo(source)468 b Fu(source)15 b(rank,)f(or)27 b Fr(MPI)p 1281 1302 13 2 v 15 w(ANY)p 1381 1302 V 14 w(SOURCE)13 b Fu(\(in)o(teger\))117 1379 y(IN)155 b Fo(tag)526 b Fu(tag)14 b(v)n(alue,)e(or)28 b Fr(MPI)p 1237 1379 V 14 w(ANY)p 1336 1379 V 15 w(T)m(A)o(G)13 b Fu(\(in)o(teger\))117 1455 y(IN)155 b Fo(comm)470 b Fu(handle)14 b(to)g(comm)o(unicator)117 1532 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(assumed)14 b(t)o(yp)q(e)g(of)f(data)h(in)f (message)h(\(status\))117 1608 y(OUT)108 b Fo(status)476 b Fu(status)15 b(ob)r(ject)166 1734 y Fo(MPI)p 251 1734 14 2 v 16 w(PROBE)h FE(b)q(eha)o(v)o(es)f(lik)o(e)31 b Fo(MPI)p 780 1734 V 16 w(IPROBE)16 b FE(except)g(that)e(it)h(is)h(a)f (blo)q(c)o(king)h(call)g(whic)o(h)g(returns)75 1790 y(only)g(after)e(a) h(matc)o(hing)g(message)g(has)g(b)q(een)h(found.)166 1923 y Fv(Discussion:)166 1974 y(CANCEL)h(has)f(b)q(een)f(c)o(hanged)f (since)h(last)g(draft)75 2161 y Fo(MPI)p 160 2161 V 16 w(CANCEL\(handle\))117 2239 y Fu(IN)155 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)166 2364 y FE(A)k(call)i(to)29 b Fo(MPI)p 454 2364 V 16 w(CANCEL)16 b FE(marks)e(for)h(cancellation)i(a)e(p)q(ending)i(non)o(blo)q(c)o (king)f(comm)o(unication)75 2421 y(op)q(eration)g(\(send)g(or)g(receiv) o(e\).)23 b(The)16 b(cancel)h(call)g(is)g(non-blo)q(c)o(king,)g(and)f (lo)q(cal.)24 b(It)16 b(returns)f(immedi-)75 2477 y(ately)l(,)g(p)q (ossibly)i(b)q(efore)e(the)h(comm)o(unication)g(is)f(actually)h (cancelled.)166 2534 y(It)c(is)g(still)h(necessary)f(to)g(complete)g(a) g(comm)o(unication)g(that)f(has)h(b)q(een)h(mark)o(ed)e(for)h (cancellation,)75 2591 y(using)h(a)e(call)j(to)23 b Fo(MPI)p 455 2591 V 16 w(W)l(AIT)12 b FE(or)23 b Fo(MPI)p 742 2591 V 16 w(TEST)12 b FE(\(or)f(an)o(y)h(of)g(the)g(deriv)o(ed)h(op)q (erations\).)18 b(If)13 b(the)f(op)q(eration)75 2647 y(has)20 b(b)q(een)h(cancelled,)i(then)d(information)g(ab)q(out)g(to)f (that)h(e\013ect)f(will)j(b)q(e)f(returned)f(in)h(the)f(status)75 2704 y(argumen)o(t)10 b(of)g(the)h(op)q(eration)g(that)f(completes)h (the)g(comm)o(unication.)19 b(If)11 b(a)g(comm)o(unication)g(is)g(mark) o(ed)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 65 71 65 70 bop 75 -100 a Fx(4.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (OBJECTS)673 b FE(65)75 45 y(for)18 b(cancellation,)j(then)e(a)37 b Fo(MPI)p 667 45 14 2 v 16 w(W)l(AIT)18 b FE(call)i(for)e(that)g(comm) o(unication)h(is)g(guaran)o(teed)f(to)g(return,)75 102 y(irresp)q(ectiv)o(e)d(of)e(the)g(status)f(of)h(other)g(pro)q(cesses;)h (similarly)h(if)27 b Fo(MPI)p 1270 102 V 16 w(TEST)13 b FE(is)h(rep)q(eatedly)g(called)h(in)f(a)75 158 y(busy)e(w)o(ait)g(lo) q(op)g(for)g(a)f(cancelled)k(comm)o(unication,)d(then)h Fo(MPI)p 1177 158 V 16 w(TEST)f FE(will)h(ev)o(en)o(tually)g(b)q(e)g (successful.)166 231 y(Either)19 b(the)f(cancellation)i(succeeds,)f(or) f(the)g(comm)o(unication)h(succeeds,)h(but)e(not)g(b)q(oth.)29 b(If)18 b(a)75 288 y(send)h(is)h(mark)o(ed)e(for)g(cancellation,)j (then)e(it)g(m)o(ust)f(b)q(e)i(the)f(case)g(that)f(either)h(the)g(send) g(completes)75 344 y(normally)l(,)f(in)h(whic)o(h)f(case)f(the)h (message)f(sen)o(t)g(w)o(as)g(receiv)o(ed)h(at)f(the)h(destination)g (pro)q(cess,)g(or)f(that)75 400 y(the)e(send)h(is)f(successfully)i (cancelled,)g(in)e(whic)o(h)h(case)f(no)g(part)f(of)h(the)g(message)f (w)o(as)g(receiv)o(ed)j(at)d(the)75 457 y(destination.)21 b(Then,)15 b(an)o(y)f(matc)o(hing)h(receiv)o(e)h(has)e(to)h(b)q(e)g (satis\014ed)h(b)o(y)e(another)h(send.)20 b(If)15 b(a)g(receiv)o(e)g (is)75 513 y(mark)o(ed)d(for)f(cancellation,)k(then)d(it)h(m)o(ust)e(b) q(e)i(the)g(case)f(that)f(either)i(the)g(receiv)o(e)g(completes)g (normally)l(,)75 570 y(or)k(that)g(the)h(receiv)o(e)h(is)f (successfully)i(cancelled,)g(in)f(whic)o(h)f(case)g(no)g(part)f(of)g (the)h(receiv)o(e)h(bu\013er)e(is)75 626 y(altered.)j(Then,)c(an)o(y)e (matc)o(hing)i(send)g(has)f(to)f(b)q(e)i(satis\014ed)g(b)o(y)f(another) g(receiv)o(e.)75 747 y Fo(MPI)p 160 747 V 16 w(TEST)p 290 747 V 16 w(CANCELLED\(status,)i(\015ag\))117 840 y Fu(IN)155 b Fo(status)476 b Fu(return)15 b(status)g(ob)r(ject)117 949 y(IN)155 b Fo(\015ag)518 b Fu(\(logical\))166 1090 y FE(Returns)28 b Fo(\015ag)13 b(=)h(true)g FE(if)g(the)g(comm)o (unication)g(asso)q(ciated)g(with)g(the)g(return)f(status)g(ob)s(ject)g (w)o(as)75 1146 y(cancelled)21 b(successfully)l(.)34 b(In)19 b(suc)o(h)g(a)g(case,)h(all)f(other)g(\014elds)h(of)38 b Fo(status)20 b FE(\(suc)o(h)f(as)38 b Fo(length)20 b FE(or)37 b Fo(tag)p FE(\))75 1203 y(are)17 b(unde\014ned.)28 b(Returns)35 b Fo(\015ag)17 b(=)h(false)p FE(,)f(otherwise.)27 b(If)17 b(a)g(receiv)o(e)h(op)q(eration)g(migh)o(t)f(b)q(e)h(cancelled) 75 1259 y(then)i(one)g(should)h(call)41 b Fo(MPI)p 613 1259 V 16 w(TEST)p 743 1259 V 16 w(CANCELLED)20 b FE(\014rst,)g(to)g(c) o(hec)o(k)g(whether)g(the)g(op)q(eration)g(w)o(as)75 1316 y(cancelled,)d(b)q(efore)f(c)o(hec)o(king)g(on)f(the)g(other)g(v)m (alues)h(of)f(the)g(return)g(status.)75 1554 y Fy(4.10)59 b(P)n(ersistent)19 b(communication)f(objects)75 1764 y Fv(Discussion:)166 1837 y(there)c(are)i(some)f(c)o(hanges)g(here)g (not)g(y)o(et)g(appro)o(v)o(ed)e(b)o(y)j(MPIF)166 1993 y FE(Often)e(a)e(comm)o(unication)i(with)g(the)f(same)g(parameter)g (list)h(is)g(rep)q(eatedly)g(executed)g(within)h(the)75 2049 y(inner)k(lo)q(op)g(of)e(a)h(parallel)i(computation.)28 b(In)18 b(suc)o(h)h(a)f(situation,)g(it)h(ma)o(y)e(b)q(e)i(p)q(ossible) g(to)f(optimize)75 2106 y(the)c(comm)o(unication)g(b)o(y)g(binding)i (the)d(list)i(of)e(comm)o(unication)i(parameters)d(to)h(the)h(comm)o (unication)75 2162 y(ob)s(ject)f(once)h(and,)g(then,)g(rep)q(eatedly)g (using)h(the)e(comm)o(unication)i(handle)g(to)d(initiate)j(and)f (complete)75 2219 y(messages.)22 b(The)16 b(comm)o(unication)h(handle)g (th)o(us)f(created)g(can)g(b)q(e)g(though)o(t)f(of)h(as)g(a)f(comm)o (unication)75 2275 y(p)q(ort)c(or)f(a)h(\\half-c)o(hannel")h(.)18 b(It)12 b(do)q(es)f(not)f(pro)o(vide)i(the)f(full)h(functionalit)o(y)h (of)d(a)h(con)o(v)o(en)o(tional)g(c)o(hannel,)75 2332 y(since)i(there)g(is)g(no)f(binding)i(of)e(the)g(send)h(p)q(ort)f(to)f (the)i(receiv)o(e)g(p)q(ort:)18 b(this)13 b(construct)f(allo)o(ws)g (reduction)75 2388 y(of)18 b(the)g(o)o(v)o(erhead)f(for)h(comm)o (unication)g(b)q(et)o(w)o(een)h(pro)q(cessor)e(and)h(comm)o(unication)h (con)o(troller,)g(but)75 2444 y(not)c(the)g(o)o(v)o(erhead)g(for)f (comm)o(unication)i(b)q(et)o(w)o(een)g(one)f(comm)o(unication)h(con)o (troller)g(and)f(another.)166 2518 y(A)j(comm)o(unication)h(ob)s(ject)f (is)g(created)h(using)f(one)h(of)e(the)i(four)f(follo)o(wing)g(calls.) 30 b(These)19 b(calls)75 2574 y(in)o(v)o(olv)o(e)d(no)f(comm)o (unication.)166 2647 y(The)e(function)g Fo(MPI)p 517 2647 V 16 w(CREA)l(TE)p 702 2647 V 17 w(SEND)g FE(creates)f(a)g(comm)o (unication)h(ob)s(ject)f(for)g(a)g(standard)g(mo)q(de)75 2704 y(send)k(op)q(eration,)f(and)g(binds)i(to)d(it)i(all)g(the)f (parameters)f(of)h(a)g(send)h(op)q(eration.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 66 72 66 71 bop 75 -100 a FE(66)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(CREA)l(TE)p 345 45 V 17 w(SEND\(handle,)h(sta)o(rt,) e(count,)h(datat)o(yp)q(e,)g(dest,)h(tag,)d(comm\))117 137 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o (on)d(ob)r(ject)117 242 y(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 348 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(sen)o(t)i(\(in)o(teger\))117 453 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)117 558 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o (teger\))117 663 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 768 y(IN)155 b Fo(comm)470 b Fu(handle)14 b(to)g(comm)o(unicator)d(ob)r(ject)166 908 y FE(The)i(call)i(allo)q(cates)f(a)f(new)h(comm)o(unication)g(ob)s (ject)e(and)i(asso)q(ciates)f(the)h(comm)o(unication)g(han-)75 964 y(dle)i(with)g(it.)166 1112 y Fv(Discussion:)166 1176 y Fu(Tw)o(o)d(c)o(hanges:)166 1241 y(1.)42 b Fr(bu\013er)p 341 1241 13 2 v 17 w(handle)16 b Fu(is)f(replaced)h(with)29 b Fr(sta)o(rt,)15 b(count,)g(datat)o(yp)q(e)p Fu(,)h(to)f(b)q(e)g(in)g (sync)g(with)g(the)g(new)h(datat)o(yp)q(e)75 1291 y(prop)q(osal.)166 1356 y(2.)g(T)m(o)q(ok)10 b(out)20 b Fr(p)q(ersistence)p Fu(.)g(Do)q(es)10 b(not)g(seem)h(v)o(ery)f(useful)h(here.)18 b(Instead,)11 b(I)f(in)o(tro)q(duce)h(in)f(the)h(last)f(section,)75 1406 y(a)k(\\univ)o(ersal")f(comm)o(unicati)o(on)e(function,)i(whic)o (h)h(w)o(ould)f(not)h(usually)f(used)i(directly)f(b)o(y)f(programmers,) f(but)75 1455 y(can)i(b)q(e)h(used)f(to)g(de\014ne)h(the)g(other)f (comm)o(unication)c(functions.)166 1610 y FE(The)16 b(function)h Fo(MPI)p 524 1610 14 2 v 16 w(CREA)l(TE)p 709 1610 V 17 w(RSEND)g FE(creates)e(a)h(comm)o(unication)h(ob)s(ject)e(for)g(a)h (ready)g(mo)q(de)75 1666 y(send)g(op)q(eration.)75 1785 y Fo(MPI)p 160 1785 V 16 w(CREA)l(TE)p 345 1785 V 17 w(RSEND\(handle,)h(sta)o(rt,)e(count,)h(datat)o(yp)q(e,)h(dest,)f(tag,) f(comm\))117 1877 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)117 1982 y(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f(send)g(bu\013er)117 2087 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 2192 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)117 2297 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o(teger\))117 2403 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 2508 y(IN)155 b Fo(comm)470 b Fu(handle)14 b(to)g(comm)o(unicator)d(ob) r(ject)166 2647 y FE(The)k(function)i Fo(MPI)p 523 2647 V 15 w(CREA)l(TE)p 707 2647 V 18 w(SSEND)f FE(creates)f(a)g(comm)o (unication)h(ob)s(ject)e(for)h(a)g(sync)o(hronous)75 2704 y(mo)q(de)g(send)h(op)q(eration.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 67 73 67 72 bop 75 -100 a Fx(4.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (OBJECTS)673 b FE(67)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(CREA)l(TE)p 345 45 V 17 w(SSEND\(handle,)17 b(sta)o(rt,)e(count,)h (datat)o(yp)q(e,)g(dest,)h(tag,)e(comm\))117 123 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject) 117 201 y(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f (send)g(bu\013er)117 278 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er) 13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 356 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)117 433 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o(teger\))117 511 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 588 y(IN)155 b Fo(comm)470 b Fu(handle)14 b(to)g(comm)o(unicator)d(ob)r (ject)166 714 y FE(The)k(function)h Fo(MPI)p 522 714 V 15 w(CREA)l(TE)p 706 714 V 18 w(RECV)g FE(creates)e(a)h(comm)o (unication)g(ob)s(ject)f(for)h(a)f(receiv)o(e)i(op)q(era-)75 771 y(tion.)75 875 y Fo(MPI)p 160 875 V 16 w(CREA)l(TE)p 345 875 V 17 w(RECV\(handle,)h(sta)o(rt,)e(count,)h(datat)o(yp)q(e,)h (source,)e(tag,)g(comm\))117 954 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati)o(on)d(ob)r(ject)117 1031 y(OUT)108 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f (receiv)o(e)h(bu\013er)117 1109 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(receiv)o(ed)j(\(in)o (teger\))117 1186 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)117 1264 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(source)j(or)d(MPI)p 1309 1264 13 2 v 16 w(ANY)p 1418 1264 V 15 w(SOUR)o(CE)h(\(in)o (teger\))117 1341 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(or)h(MPI)p 1270 1341 V 15 w(ANY)p 1378 1341 V 16 w(T)m(A)o(G)f(\(in)o(teger\))117 1419 y(IN)155 b Fo(comm)470 b Fu(handle)14 b(to)g(comm)o(unicator)d(ob)r(ject)166 1544 y FE(A)19 b(comm)o(unication)g(\(send)g(or)f(receiv)o(e\))i(that)e (uses)h(a)f(prede\014ned)j(handle)f(is)f(initiated)h(b)o(y)f(the)75 1601 y(function)d Fo(MPI)p 338 1601 14 2 v 16 w(ST)l(ART)p FE(.)75 1706 y Fo(MPI)p 160 1706 V 16 w(ST)l(ART\(handle\))117 1784 y Fu(INOUT)62 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o (unicati)o(on)d(ob)r(ject)166 1910 y FE(The)17 b(comm)o(unication)h(ob) s(ject)e(asso)q(ciated)h(with)34 b Fo(handle)18 b FE(should)g(b)q(e)f (an)g(ob)s(ject)f(that)h(w)o(as)f(cre-)75 1966 y(ated)h(b)o(y)g(one)h (of)f(the)g(previous)h(four)f(functions,)h(so)f(that)f(all)j(the)e (comm)o(unication)h(parameters)e(are)75 2023 y(already)k(de\014ned.)34 b(A)19 b(send)h(can)g(b)q(e)g(started)f(pro)o(vided)h(that)f(the)g (previous)h(send)h(using)f(the)f(same)75 2079 y(ob)s(ject)d(has)h (completed,)h(or)f(as)f(so)q(on)h(as)g(the)g(ob)s(ject)f(is)h(created,) g(if)h(it)f(has)g(not)f(y)o(et)h(b)q(een)h(used)f(in)h(a)75 2136 y(comm)o(unication.)j(In)16 b(addition,)h(if)f(the)f(comm)o (unication)h(mo)q(de)g(is)g Fp(ready)f FE(then)g(a)h(matc)o(hing)f (receiv)o(e)75 2192 y(should)k(b)q(e)f(p)q(osted.)28 b(The)18 b(send)g(bu\013er)g(should)h(not)e(b)q(e)i(up)q(dated)f(after) f(the)h(send)g(is)h(started,)e(un)o(til)75 2248 y(the)e(op)q(eration)h (completes.)166 2306 y(A)e(receiv)o(e)g(can)g(b)q(e)g(started)f(pro)o (vided)i(that)e(the)h(preceding)h(receiv)o(e)f(using)h(the)f(same)f(ob) s(ject)g(has)75 2363 y(completed,)f(or)f(as)f(so)q(on)h(as)f(the)h(ob)s (ject)f(is)i(created,)f(if)g(it)g(has)g(not)g(y)o(et)f(b)q(een)i(used)f (in)h(a)f(comm)o(unication.)75 2419 y(The)18 b(receiv)o(e)h(bu\013er)f (should)h(not)e(b)q(e)i(accessed)g(after)e(the)h(receiv)o(e)h(is)f (started,)g(un)o(til)h(the)f(op)q(eration)75 2475 y(completes.)166 2533 y(The)23 b(call)g(is)g(non)o(blo)q(c)o(king,)i(with)e(similar)g (seman)o(tics)g(as)f(the)h(non)o(blo)q(c)o(king)g(comm)o(unication)75 2590 y(op)q(erations)15 b(describ)q(ed)i(in)g(Section)f(4.8.)166 2647 y(A)10 b(comm)o(unication)h(started)e(with)i(a)f(call)h(to)e Fo(MPI)p 1016 2647 V 16 w(ST)l(ART)i FE(is)g(completed)g(b)o(y)f(a)g (call)h(to)f Fo(MPI)p 1740 2647 V 16 w(W)l(AIT)p FE(,)75 2704 y Fo(MPI)p 160 2704 V 16 w(TEST)p FE(,)i(or)f(one)h(of)g(the)g (deriv)o(ed)h(functions)f(describ)q(ed)i(in)f(Section)g(4.8.4.)k(These) 12 b(comm)o(unication)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 68 74 68 73 bop 75 -100 a FE(68)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(completion)i (functions)g(do)e(not)h(deallo)q(cate)h(the)f(comm)o(unication)g(ob)s (ject,)g(and)g(this)g(can)g(b)q(e)g(reused)75 102 y(anew)k(b)o(y)f(a)h Fo(MPI)p 394 102 14 2 v 15 w(ST)l(ART)h FE(call.)37 b(The)21 b(ob)s(ject)f(needs)i(to)e(b)q(e)h(explicitly)j(deallo)q(cated)e(b)o(y) e(a)h(call)h(to)75 158 y(the)15 b(function)g Fo(MPI)p 415 158 V 16 w(COMM)p 573 158 V 16 w(FREE)p FE(,)g(b)q(elo)o(w.)20 b(W)l(e)15 b(th)o(us)f(ha)o(v)o(e)g(t)o(w)o(o)f(t)o(yp)q(es)i(of)f (comm)o(unication)h(ob)s(jects:)75 214 y FF(p)q(ersisten)o(t)j FE(ob)s(jects,)f(whic)o(h)i(are)e(allo)q(cated)i(b)o(y)f(a)f(call)i(to) e Fo(MPI)p 1230 214 V 16 w(CREA)l(TE)p 1415 214 V 18 w(xxx)p FE(,)h(and)g(are)f(explicitly)75 271 y(deallo)q(cated)h(b)o(y)f Fo(MPI)p 465 271 V 16 w(COMM)p 623 271 V 17 w(FREE)p FE(,)g(and)g FF(ephemeral)g FE(ob)s(jects)f(that)g(p)q(ersist)i(for)e (one)h(comm)o(uni-)75 327 y(cation)e(only:)21 b(they)16 b(are)f(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)i(comm)o(unication)f (initiation)h(function,)f(and)f(are)75 384 y(freed)h(b)o(y)f(the)g (comm)o(unication)h(completion)g(call.)75 490 y Fo(MPI)p 160 490 V 16 w(COMM)p 318 490 V 16 w(FREE\(handle\))117 570 y Fu(INOUT)62 b Fo(handle)465 b Fu(handle)14 b(to)g(comm)o(unicati) o(on)d(ob)r(ject)166 697 y FE(Marks)h(the)h(comm)o(unication)g(ob)s (ject)f(for)g(deallo)q(cation.)21 b(The)13 b(ob)s(ject)f(will)j(b)q(e)e (deallo)q(cated)h(when)75 754 y(there)23 b(are)f(no)g(p)q(ending)j (comm)o(unications)e(in)o(v)o(olving)g(this)g(ob)s(ject,)h(at)e(whic)o (h)h(p)q(oin)o(t)g(the)g(handle)75 810 y(b)q(ecomes)16 b(n)o(ull.)166 869 y(The)g(call)g(is)g(non)o(blo)q(c)o(king,)g(and)g (the)f(ob)s(ject)g(ma)o(y)g(not)g(b)q(e)h(deallo)q(cated)h(when)e(the)h (call)g(returns.)75 926 y(It)f(is)g(p)q(ermissible)i(to)d(call)i Fo(MPI)p 628 926 V 16 w(COMM)p 786 926 V 16 w(FREE\(handle\))g FE(after)e(a)h(comm)o(unication)g(that)f(uses)30 b Fo(handle)75 982 y FE(has)18 b(b)q(een)h(initiated,)h(but)f(b)q(efore)f(it)h(has)f (completed.)29 b(The)19 b(ob)s(ject)e(will)j(b)q(e)f(deallo)q(cated)g (after)f(the)75 1039 y(comm)o(unication)23 b(completes.)41 b(It)22 b(is)h(erroneous)f(to)f(try)h(to)f(reuse)i(the)f(handle)h(for)f (a)f(subsequen)o(t)75 1095 y(comm)o(unication.)166 1154 y(A)15 b(correct)f(in)o(v)o(o)q(cation)i(of)e(the)h(functions)h (describ)q(ed)g(in)g(this)f(section)h(will)g(o)q(ccur)g(in)f(a)g (sequence)75 1211 y(of)g(the)g(form)613 1275 y FF(Create)i FE(\()p FF(Start)h(Complete)p FE(\))1172 1257 y Fj(\003)1209 1275 y FF(F)l(ree)e FD(;)75 1368 y FE(where)j FA(\003)f FE(indicates)i(zero)e(or)g(more)g(rep)q(etitions.)31 b(If)18 b(the)h(same)f(comm)o(unication)h(ob)s(ject)f(is)h(used)g(in)75 1425 y(sev)o(eral)12 b(concurren)o(t)g(threads,)g(it)g(is)h(the)f(user) g(resp)q(onsibilit)o(y)i(to)d(co)q(ordinate)i(calls)g(so)e(that)g(the)h (correct)75 1481 y(sequence)k(is)g(ob)q(ey)o(ed.)166 1540 y(A)11 b(send)g(op)q(eration)g(initiated)h(with)22 b Fo(MPI)p 881 1540 V 15 w(ST)l(ART)12 b FE(can)f(b)q(e)g(matc)o(hed)g (with)g(an)o(y)f(receiv)o(e)i(op)q(eration)75 1597 y(and,)g(lik)o (ewise,)h(a)e(receiv)o(e)i(op)q(eration)e(initiated)i(with)23 b Fo(MPI)p 1102 1597 V 16 w(ST)l(ART)12 b FE(can)g(receiv)o(e)g (messages)f(generated)75 1653 y(b)o(y)k(an)o(y)g(send)h(op)q(eration.) 75 1812 y Fy(4.11)59 b(Send-receive)75 1995 y Fv(Discussion:)166 2054 y(This)15 b(section)f(has)h(not)g(y)o(et)g(b)q(een)g(appro)o(v)o (ed)f(b)o(y)h(MPIF.)166 2195 y FE(The)20 b FF(send-receiv)o(e)e FE(op)q(erations)i(com)o(bines)g(in)g(one)g(call)h(the)e(sending)i(of)e (a)g(message)g(to)g(one)75 2252 y(destination)h(and)f(the)g(receiving)h (of)e(another)h(message,)g(from)f(another)g(destination,)i(p)q(ossibly) h(the)75 2308 y(same.)f(The)c FF(exc)o(hange)f FE(op)q(erations)h(are)f (the)h(same)f(as)g(send-receiv)o(e,)i(except)f(that)e(the)i(send)g (bu\013er)75 2365 y(and)i(the)g(receiv)o(e)g(bu\013er)g(are)g(iden)o (tical.)29 b(A)18 b(send-receiv)o(e)h(op)q(eration)f(is)g(v)o(ery)g (useful)h(for)e(executing)75 2421 y(a)h(shift)i(op)q(eration)f(across)f (a)g(c)o(hain)i(of)e(pro)q(cesses.)31 b(If)19 b(blo)q(c)o(king)h(sends) f(and)g(receiv)o(es)h(are)e(used)i(for)75 2478 y(suc)o(h)d(shift,)g (then)f(one)h(needs)g(to)f(order)g(correctly)h(the)f(sends)h(and)g (receiv)o(es)g(\(e.g.)23 b(ev)o(en)17 b(sends,)g(next)75 2534 y(receiv)o(es,)j(o)q(dd)f(receiv)o(es)h(\014rst,)f(next)g(sends\)) f(so)h(as)f(to)g(prev)o(en)o(t)h(cyclic)h(dep)q(endencies)i(that)c (lead)i(to)75 2591 y(deadlo)q(c)o(k.)f(When)12 b(a)e(send-receiv)o(e)j (or)d(exc)o(hange)h(op)q(eration)g(is)h(used,)f(the)g(comm)o(unication) h(subsystem)75 2647 y(tak)o(es)i(care)h(of)g(these)g(issues.)21 b(Also,)15 b(a)g(send-receiv)o(e)i(op)q(eration)e(is)h(useful)g(for)e (implemen)o(ting)k(remote)75 2704 y(pro)q(cedure)e(calls.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 69 75 69 74 bop 75 -100 a Fx(4.11.)34 b(SEND-RECEIVE)1277 b FE(69)166 45 y(A)17 b(message)f(sen)o(t)h(b)o(y)g(a)f(send-receiv)o(e)j (or)d(exc)o(hange)h(op)q(eration)g(can)g(b)q(e)h(receiv)o(ed)g(b)o(y)e (a)h(regular)75 102 y(receiv)o(e)f(op)q(eration,)f(and)h(vice)g(v)o (ersa.)75 206 y Fo(MPI)p 160 206 14 2 v 16 w(SENDRECV\(send)p 509 206 V 19 w(sta)o(rt,)26 b(send)p 736 206 V 18 w(count,)g(send)p 981 206 V 18 w(t)o(yp)q(e,)h(dest,)g(recv)p 1312 206 V 16 w(sta)o(rt,)f(recv)p 1530 206 V 16 w(count,)h(recv)p 1768 206 V 16 w(t)o(yp)q(e,)75 263 y(source,)15 b(tag,)g(comm,)e (status\))117 341 y Fu(IN)155 b Fo(send)p 404 341 V 18 w(sta)o(rt)400 b Fu(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 418 y(IN)155 b Fo(send)p 404 418 V 18 w(count)381 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 495 y(IN)155 b Fo(send)p 404 495 V 18 w(t)o(yp)q(e)406 b Fu(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f (send)i(bu\013er)117 572 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o(teger\))117 649 y(OUT)108 b Fo(recv)p 398 649 V 17 w(sta)o(rt)407 b Fu(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 726 y(IN)155 b Fo(recv)p 398 726 V 17 w(count)388 b Fu(n)o(um)o(b)q(er) 13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 803 y(IN)155 b Fo(recv)p 398 803 V 17 w(t)o(yp)q(e)413 b Fu(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(receiv)o(e)i(bu\013er)117 880 y(IN)155 b Fo(source)468 b Fu(rank)14 b(of)f(source)j(\(in)o (teger\))117 957 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 1034 y(IN)155 b Fo(comm)470 b Fu(handle)14 b(to)g(comm)o(unicator)117 1111 y(OUT)108 b Fo(status)476 b Fu(status)15 b(ob)r(ject)166 1237 y FE(Execute)e(a)f(blo)q(c)o(king)i(send)g(and)f(receiv)o(e)g(op)q (eration.)19 b(Both)13 b(send)g(and)g(receiv)o(e)h(use)f(the)f(same)h (tag)75 1293 y(v)m(alue)h(and)g(the)f(same)g(comm)o(unicator.)19 b(Ho)o(w)o(ev)o(er,)12 b(the)i(send)g(bu\013er)f(and)g(receiv)o(e)h (bu\013er)g(are)f(disjoin)o(t,)75 1350 y(and)i(ma)o(y)g(ha)o(v)o(e)g (di\013eren)o(t)g(length)h(and)f(di\013eren)o(t)h(datat)o(yp)q(es.)166 1483 y Fv(Discussion:)166 1534 y Fu(F)m(or)d(a)h(shift)f(it's)h(more)e (natural)i(to)f(ha)o(v)o(e)h(same)f(t)o(yp)q(e)h(and)g(same)e(length)i (b)q(oth)g(for)f(send)i(and)f(receiv)o(e;)g(for)75 1584 y(a)g(remote)f(pro)q(cedure)j(call,)d(the)h(additional)e(freedom)h(mak) o(es)g(sense.)75 1771 y Fo(MPI)p 160 1771 V 16 w(EX)o(CHANGE\(sta)o (rt,)j(count,)g(datat)o(yp)q(e,)h(dest,)f(source,)f(tag,)g(comm,)e (status\))117 1849 y Fu(IN)155 b Fo(sta)o(rt)501 b Fu(initial)12 b(address)j(of)f(send)g(and)g(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1926 y(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)9 b(of)g(elemen)o(ts) h(in)f(send)i(and)e(receiv)o(e)i(bu\013er)g(\(in)o(teger\))117 2003 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(and)f(receiv)o(e)h(bu\013er)117 2080 y(IN)155 b Fo(dest)511 b Fu(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2157 y(IN)155 b Fo(source)468 b Fu(rank)14 b(of)f(source)j(\(in)o(teger\))117 2234 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 2311 y(IN)155 b Fo(comm)470 b Fu(handle)14 b(to)g(comm)o(unicator)117 2388 y(OUT)108 b Fo(status)476 b Fu(status)15 b(ob)r(ject)166 2514 y FE(Execute)g(a)f(blo)q(c)o(king)i(send)f(and)g(receiv)o(e;)g (the)g(same)f(bu\013er)g(is)h(used)h(b)q(oth)e(for)g(the)h(send)g(and)g (for)75 2570 y(the)g(receiv)o(e.)166 2704 y Fv(Discussion:)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 70 76 70 75 bop 75 -100 a FE(70)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fu(I)h(omitted)e (the)j(ready)f(and)f(sync)o(hronous)i(mo)q(des)e(for)h(send/receiv)o(e) i(and)d(exc)o(hange.)28 b(Also,)17 b(I)f(omitted)75 95 y(non)o(blo)q(c)o(king)d(send/receiv)o(e)i(and)f(exc)o(hange.)19 b(Do)13 b(w)o(e)h(w)o(an)o(t)g(an)o(y)f(of)h(those?)166 234 y FE(The)20 b(seman)o(tics)h(of)f(a)g(send-receiv)o(e)i(op)q (eration)e(is)h(what)f(w)o(ould)g(obtain)h(if)g(the)f(caller)h(fork)o (ed)75 291 y(t)o(w)o(o)16 b(concurren)o(t)h(threads,)g(one)g(to)f (execute)i(the)f(send,)g(and)h(one)f(to)f(execute)i(the)f(receiv)o(e,)h (follo)o(w)o(ed)75 347 y(b)o(y)f(a)f(join)h(of)f(these)h(t)o(w)o(o)e (threads.)24 b(A)16 b(send-receiv)o(e)j(cannot)d(b)q(e)h(implemen)o (ted)i(b)o(y)d(a)g(blo)q(c)o(king)i(send)75 404 y(follo)o(w)o(ed)e(b)o (y)f(a)g(blo)q(c)o(king)i(receiv)o(e)g(or)e(a)g(blo)q(c)o(king)i (receiv)o(e,)f(follo)o(w)o(ed)g(b)o(y)f(a)g(blo)q(c)o(king)i(send.)k (Consider)75 460 y(the)15 b(follo)o(wing)h(co)q(de,)g(where)f(t)o(w)o (o)f(pro)q(cesses)i(exc)o(hange)f(messages:)75 567 y Fp(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 624 y(IF)h(\(rank.EQ.0\))123 680 y(THEN)170 737 y(CALL)g (MPI_SENDRECV\(send_buff,)c(count,)k(MPI_REAL,)e(0,)i(recv_buff,)e (count,)h(MPI_REAL,)290 793 y(1,)g(tag,)h(comm,)f(status,)g(ierr\))123 850 y(ELSE)170 906 y(CALL)h(MPI_SENDRECV\(send_buff,)c(count,)k (MPI_REAL,)e(1,)i(recv_buff,)e(count,)h(MPI_REAL,)290 962 y(0,)g(tag,)h(comm,)f(status,)g(ierr\))75 1019 y(END)g(IF)166 1126 y FE(If)12 b(the)f(send)h(receiv)o(es)g(are)g(replaced)g(either)g (b)o(y)g(blo)q(c)o(king)g(send,)h(follo)o(w)o(ed)f(b)o(y)f(blo)q(c)o (king)i(receiv)o(e,)f(or)75 1182 y(blo)q(c)o(king)g(receiv)o(e,)h (follo)o(w)o(ed)e(b)o(y)g(blo)q(c)o(king)i(send,)f(then)g(the)f(co)q (de)g(ma)o(y)g(deadlo)q(c)o(k.)19 b(On)12 b(the)f(other)g(hand,)75 1239 y(send-receiv)o(e)21 b(can)f(b)q(e)h(implemen)o(ted)g(using)g(non) o(blo)q(c)o(king)g(sends)f(and)g(receiv)o(es.)35 b(Note)19 b(that)g(some)75 1295 y(system)e(bu\013ering)i(is)f(required)h(for)e(a) h(correct)f(implemen)o(tation)i(of)35 b Fo(MPI)p 1402 1295 14 2 v 16 w(EX)o(CHANGE)p FE(.)19 b(\(Consider)75 1352 y(the)c(last)g(example,)h(with)g(send-receiv)o(e)g(replaced)h(b)o (y)e(exc)o(hange.\))75 1496 y Fy(4.12)59 b(Null)21 b(p)n(ro)r(cesses)75 1680 y Fv(Discussion:)34 b Fu(This)14 b(section)g(has)g(not)g(b)q(een)h (review)o(ed)g(b)o(y)f(MPIF.)166 1820 y FE(In)21 b(man)o(y)f (instances,)h(it)g(is)g(con)o(v)o(enien)o(t)g(to)e(sp)q(ecify)j(a)e (\\dumm)o(y")f(source)i(or)e(destination)j(for)75 1876 y(comm)o(unication.)27 b(This)18 b(simpli\014es)i(the)d(co)q(de)h(that) f(is)h(needed)g(for)f(dealing)i(with)e(b)q(oundaries,)i(e.g.,)75 1933 y(in)d(the)f(case)h(of)e(a)h(non-circular)i(shift)e(done)h(with)g (calls)g(to)e(send-receiv)o(e.)166 1989 y(The)f(sp)q(ecial)j(v)m(alue) 26 b Fr(MPI)p 608 1989 13 2 v 15 w(PROCNULL)12 b FE(can)i(b)q(e)g(used) g(instead)g(of)f(a)g(rank)g(wherev)o(er)g(a)g(source)g(or)g(a)75 2046 y(destination)i(parameter)e(is)i(required)g(in)g(a)e(call.)21 b(A)14 b(comm)o(unication)h(with)f(pro)q(cess)27 b Fr(MPI)p 1649 2046 V 15 w(PROCNULL)75 2102 y FE(has)13 b(no)h(e\013ect:)k(a)13 b(send)h(to)25 b Fr(MPI)p 633 2102 V 14 w(PROCNULL)13 b FE(succeeds)i(and)e(returns)g(as)g(so)q(on)h(as)f(p)q(ossible.)21 b(A)13 b(receiv)o(e)75 2159 y(from)26 b Fr(MPI)p 271 2159 V 14 w(PROCNULL)13 b FE(succeeds)i(and)f(returns)g(as)f(so)q(on)h (as)f(p)q(ossible)j(with)e(no)g(mo)q(di\014cations)h(to)e(the)75 2215 y(receiv)o(e)k(bu\013er.)22 b(When)16 b(a)g(receiv)o(e)h(with)32 b Fo(source)17 b FE(=)30 b Fr(MPI)p 1095 2215 V 15 w(PROCNULL)15 b FE(is)i(executed)f(then)h(the)f(status)75 2272 y(ob)s(ject)f(returns) 30 b Fo(source)16 b FE(=)29 b Fr(MPI)p 661 2272 V 14 w(PROCNULL)p FE(,)g Fo(tag)16 b FE(=)29 b Fr(MPI)p 1147 2272 V 14 w(ANY)p 1246 2272 V 15 w(T)m(A)o(G)14 b FE(and)31 b Fo(count=0)p FE(.)166 2404 y Fv(Discussion:)166 2454 y Fu(It)18 b(w)o(ould)f(b)q(e)i(nice)f(to)g(ha)o(v)o(e)36 b Fr(MPI)p 740 2454 V 14 w(PROCNULL)17 b Fu(\\equiv)n(alenced")h(to)g (-1)f(and)h(to)g(group)p 1624 2454 V 15 w(size,)h(so)f(that,)75 2504 y(when)f(a)f(shift)h(is)f(executed)i(in)e(a)h(group)f(the)h (\014rst)g(and)g(last)f(pro)q(cessors)j(automatically)13 b(comm)o(unicate)h(with)75 2554 y(the)g(n)o(ull)f(pro)q(cess.)166 2604 y(A)e(c)o(hoice)g(is)f(to)h(sa)o(y)f(that)h(a)g (source/destination)g(with)g(rank)f(-1)g(or)h(group)p 1329 2604 V 15 w(size)g(is)g(alw)o(a)o(ys)f(the)h(n)o(ull)f(pro)q (cess.)75 2654 y(W)m(e)j(then)i(lose)f(some)f(error)i(catc)o(hing)f (capabilit)o(y)m(.)166 2704 y(Another)h(c)o(hoice)f(is)g(to)f(sa)o(y)h (that)g(only)f(for)h(send-receiv)o(e)i(and)d(exc)o(hange.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 71 77 71 76 bop 75 -100 a Fx(4.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(71)166 45 y Fu(A)13 b(third)g(c)o(hoice)g(is)g(to)f(liv)o(e)h (without)f(this)h(nicet)o(y)m(.)k(It)c(means)f(that)h(one)g(still)f(ha) o(v)o(e)h(to)g(execute)h(conditional)75 95 y(co)q(de:)k(\(if)10 b(m)o(yrank=0)g(then)i(m)o(yneigh)o(b)q(or=mpi)p 850 95 13 2 v 12 w(pro)q(cn)o(ull\).)17 b(But)12 b(this)g(co)q(de)g(need)g (b)q(e)g(executed)i(only)c(when)i(the)75 145 y(comm)o(unication)e (pattern)15 b(is)f(set,)g(and)g(the)g(conditional)f(b)q(oundary)h(co)q (de)h(is)e(not)h(needed)h(in)f(the)g(inner)h(lo)q(op.)75 374 y Fy(4.13)59 b(Derived)19 b(datat)n(yp)r(es)75 553 y Fv(Discussion:)166 604 y(This)c(is)g(a)h(new)g(section,)e(and)h(the)g (material)f(has)h(not)g(y)o(et)g(b)q(een)g(discussed)f(b)o(y)h(MPIF.) 166 743 y FE(Up)10 b(to)g(no)o(w,)g(all)h(p)q(oin)o(t)g(to)f(p)q(oin)o (t)g(comm)o(unication)h(in)o(v)o(olv)o(ed)g(only)g(con)o(tiguous)f (bu\013ers)g(con)o(taining)75 800 y(a)18 b(sequence)h(of)f(elemen)o(ts) g(of)g(the)g(same)g(t)o(yp)q(e.)29 b(This)18 b(is)h(to)q(o)e (constraining)i(on)f(t)o(w)o(o)f(accoun)o(ts:)25 b(One)75 856 y(often)16 b(w)o(an)o(ts)f(to)g(pass)h(messages)f(that)h(con)o (tain)g(v)m(alues)h(with)f(di\013eren)o(t)h(datat)o(yp)q(es)e(\(e.g.,)g (an)h(in)o(teger)75 913 y(coun)o(t,)i(follo)o(w)o(ed)g(b)o(y)f(coun)o (t)h(real)g(n)o(um)o(b)q(ers\);)g(and)g(one)g(often)f(w)o(an)o(ts)f(to) h(send)i(noncon)o(tiguous)f(data)75 969 y(\(e.g.)g(a)12 b(sub-blo)q(c)o(k)i(of)e(a)g(matrix\).)18 b(One)13 b(solution)g(is)g (to)f(pro)o(vide)h(functions)g(that)e(pac)o(k)i(noncon)o(tiguous)75 1026 y(data)f(in)o(to)h(a)g(con)o(tiguous)g(bu\013er)g(at)f(the)h (sender)h(site)f(and)g(unpac)o(k)h(it)f(bac)o(k)g(at)f(the)h(receiv)o (er)h(site.)19 b(This)75 1082 y(has)e(the)h(disadv)m(an)o(tage)g(of)f (requiring)i(additional)g(memory)e(to)g(memory)g(cop)o(y)h(at)f(b)q (oth)g(sites,)i(ev)o(en)75 1139 y(when)c(the)g(comm)o(unication)h (subsystem)f(has)g(scatter-gather)e(capabilities.)23 b(Instead,)15 b(MPI)f(pro)o(vides)75 1195 y(mec)o(hanisms)j(to)f(sp)q (ecify)j(more)d(general,)h(mixed)h(and)f(noncon)o(tiguous)g(comm)o (unication)g(bu\013ers.)25 b(It)75 1251 y(is)16 b(up)g(to)f(the)h (implemen)o(tation)h(to)e(decide)i(whether)f(data)f(should)h(b)q(e)g (\014rst)g(pac)o(k)o(ed)f(in)i(a)e(con)o(tiguous)75 1308 y(bu\013er)e(b)q(efore)f(b)q(een)i(transmitted,)e(or)g(whether)h(it)g (can)g(b)q(e)g(collected)h(directly)g(from)e(where)h(it)f(resides.)166 1365 y(The)h(general)g(mec)o(hanisms)g(pro)o(vided)g(here)g(allo)o(ws)g (one)f(to)g(transfer)g(directly)l(,)i(without)f(cop)o(ying,)75 1421 y(ob)s(jects)k(of)g(v)m(arious)h(shap)q(e)h(and)f(size.)28 b(It)17 b(is)i(not)e(assumed)h(that)f(the)g(MPI)h(library)g(is)h (cognizan)o(t)e(of)75 1478 y(the)f(ob)s(jects)g(declared)h(in)g(the)f (host)g(language;)g(th)o(us,)g(if)g(one)g(w)o(an)o(ts)f(to)h(transfer)f (a)h(structure,)f(or)h(an)75 1534 y(arra)o(y)d(section,)i(it)f(will)i (b)q(e)f(necessary)f(to)g(pro)o(vide)h(in)g(MPI)f(a)g(de\014nition)i (of)e(a)g(comm)o(unication)h(bu\013er)75 1591 y(that)e(mimics)j(the)e (de\014nition)i(of)e(the)g(structure)g(or)f(arra)o(y)g(section)i(in)g (question.)20 b(These)15 b(facilities)h(can)75 1647 y(b)q(e)j(used)g(b) o(y)f(library)h(designers)h(to)d(de\014ne)j(comm)o(unication)f (functions)g(that)f(can)g(transfer)g(ob)s(jects)75 1704 y(de\014ned)g(in)f(the)f(host)g(language)g({)g(b)o(y)h(deco)q(ding)g (their)g(de\014nitions)h(as)e(a)o(v)m(ailable)i(in)f(a)f(sym)o(b)q(ol)h (table)75 1760 y(or)e(a)g(dop)q(e)g(v)o(ector.)20 b(Suc)o(h)15 b(higher-lev)o(el)j(comm)o(unication)e(functions)g(are)f(not)g(part)f (of)h(MPI.)166 1817 y(More)d(general)g(comm)o(unication)i(bu\013ers)e (are)g(sp)q(eci\014ed)i(b)o(y)f(replacing)g(the)g(basic)g(datat)o(yp)q (es)f(that)75 1874 y(ha)o(v)o(e)18 b(used)h(so)f(far)g(with)g(deriv)o (ed)i(datat)o(yp)q(es)e(that)f(are)h(constructed)h(from)e(basic)i (datat)o(yp)q(es)f(using)75 1930 y(the)13 b(constructors)e(describ)q (ed)j(in)g(this)f(section.)19 b(These)13 b(metho)q(ds)f(of)g (constructing)h(deriv)o(ed)h(datat)o(yp)q(es)75 1987 y(can)h(b)q(e)h(applied)h(recursiv)o(ely)l(.)166 2044 y(A)e FF(general)j(datat)o(yp)q(e)e FE(is)g(an)f(opaque)h(ob)s(ject)e (that)h(sp)q(eci\014es)i(t)o(w)o(o)d(things:)143 2140 y FA(\017)23 b FE(A)15 b(sequence)h(of)f(basic)h(t)o(yp)q(es)143 2237 y FA(\017)23 b FE(A)15 b(sequence)h(of)f(in)o(teger)g(\(b)o(yte\)) g(displacemen)o(ts)166 2333 y(The)22 b(displacemen)o(ts)g(are)g(not)f (required)h(to)f(b)q(e)h(p)q(ositiv)o(e,)i(distinct,)g(or)d(in)h (increasing)h(order;)75 2389 y(therefore)17 b(the)g(order)g(of)f(items) i(need)g(not)f(coincide)i(with)e(their)h(order)f(in)g(store,)g(and)g (an)g(item)h(ma)o(y)75 2446 y(app)q(ear)d(more)g(than)g(once.)166 2503 y(Let)471 2561 y FD(D)q(ataty)r(pe)e FE(=)g FA(f)p FE(\()p FD(ty)r(pe)844 2568 y Fw(0)864 2561 y FD(;)8 b(disp)969 2568 y Fw(0)987 2561 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1188 2568 y Fi(n)p Fj(\000)p Fw(1)1254 2561 y FD(;)g(disp)1359 2568 y Fi(n)p Fj(\000)p Fw(1)1426 2561 y FE(\))p FA(g)p FD(;)75 2647 y FE(b)q(e)16 b(suc)o(h)f(a)g (general)g(datat)o(yp)q(e,)f(where)h FD(ty)r(pe)851 2654 y Fi(i)881 2647 y FE(are)f(basic)i(t)o(yp)q(es,)f(and)g FD(disp)1375 2654 y Fi(i)1403 2647 y FE(are)g(displacemen)o(ts.)21 b(This)75 2704 y(general)g(datat)o(yp)q(e,)g(together)f(with)h(a)f (base)h(address)41 b Fo(sta)o(rt)p FE(,)22 b(sp)q(eci\014es)h(a)d(comm) o(unication)h(bu\013er:)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 72 78 72 77 bop 75 -100 a FE(72)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(the)22 b(comm)o(unication)g(bu\013er)f(that)g(consists)h(of)f FD(n)h FE(en)o(tries,)h(where)f(the)g FD(i)p FE(-th)f(en)o(try)g(is)h (at)f(address)75 102 y Fo(sta)o(rt)14 b FE(+)g FD(disp)308 109 y Fi(i)342 102 y FE(and)21 b(has)f(t)o(yp)q(e)h FD(ty)r(pe)714 109 y Fi(i)728 102 y FE(.)36 b(A)21 b(message)f(assem)o(bled)h(from)f (this)h(comm)o(unication)g(bu\013er)75 158 y(will)g(con)o(tain)e(a)f (sequence)j(of)d FD(n)h FE(v)m(alues,)i(where)e(v)m(alue)i FD(i)d FE(has)h(t)o(yp)q(e)g FD(ty)r(pe)1383 165 y Fi(i)1398 158 y FE(.)31 b(The)19 b(general)g(datat)o(yp)q(e)75 214 y(determines)g(the)e FF(t)o(yp)q(e)j(signature)e FE(of)f(the)h(message)f(\(i.e.)27 b(the)18 b(n)o(um)o(b)q(er)f(of)h(v)m (alues)g(carried)g(b)o(y)g(the)75 271 y(message)d(and)g(the)g(basic)h (t)o(yp)q(e)g(of)e(eac)o(h)i(v)m(alue\).)166 328 y(W)l(e)j(can)f(use)h (a)g(handle)h(to)e(a)g(general)h(datat)o(yp)q(e)f(as)h(an)f(argumen)o (t)g(in)i(a)e(send)h(or)f(receiv)o(e)i(op-)75 385 y(eration,)c(in)h (replacemen)o(t)g(of)f(a)g(basic)h(datat)o(yp)q(e)f(argumen)o(t.)23 b(The)16 b(op)q(eration)33 b Fo(MPI)p 1576 385 14 2 v 16 w(SEND\(sta)o(rt,)17 b(1,)75 441 y(datat)o(yp)q(e,...\))35 b FE(will)22 b(use)e(the)g(send)h(bu\013er)f(de\014ned)i(b)o(y)e(the)g (base)g(address)40 b Fo(sta)o(rt)21 b FE(and)f(the)g(general)75 498 y(datat)o(yp)q(e)c(asso)q(ciated)g(with)33 b Fo(datat)o(yp)q(e)p FE(;)17 b(it)g(will)h(generate)e(a)f(message)h(with)h(the)f(t)o(yp)q(e) g(signature)g(de-)75 554 y(termined)g(b)o(y)g(the)31 b Fo(datat)o(yp)q(e)17 b FE(argumen)o(t.)40 b Fo(MPI)p 932 554 V 16 w(RECV\(sta)o(rt,)16 b(1,)f(datat)o(yp)q(e,...\))21 b FE(will)c(use)f(the)f(receiv)o(e)75 611 y(bu\013er)e(de\014ned)h(b)o (y)f(the)g(base)g(address)26 b Fo(sta)o(rt)13 b FE(and)g(the)g(general) g(datat)o(yp)q(e)g(asso)q(ciated)g(with)26 b Fo(datat)o(yp)q(e)p FE(.)166 668 y(General)16 b(datat)o(yp)q(es)e(can)i(b)q(e)g(used)f(in)h (all)h(send)e(and)h(receiv)o(e)g(op)q(erations.)k(W)l(e)c(address)f (later)g(in)75 724 y(Section)h(4.13.3)d(the)j(case)f(where)g(the)h (second)f(argumen)o(t)30 b Fo(count)16 b FE(has)f(v)m(alue)i FD(>)c FE(1.)166 782 y(The)22 b(prede\014ned)h(basic)f(datat)o(yp)q(es) f(presen)o(ted)g(in)i(Section)f(4.2.1)e(are)h(particular)h(cases)f(of)g (a)75 838 y(general)c(datat)o(yp)q(e.)22 b(Th)o(us,)33 b Fo(MPI)p 675 838 V 16 w(INT)16 b FE(is)h(a)f(prede\014ned)i(handle)f (to)f(the)g(datat)o(yp)q(e)g FA(f)p FE(\()p Fp(int)o FD(;)8 b FE(0\))p FA(g)p FE(,)15 b(with)75 895 y(one)g(en)o(try)g(of)g (t)o(yp)q(e)g Fp(int)g FE(and)g(displacemen)o(t)i(zero.)j(And)c (similarly)h(for)d(all)i(other)f(basic)h(datat)o(yp)q(es.)166 952 y(The)f FF(exten)o(t)f FE(of)g(a)g(datat)o(yp)q(e)g(is)h(de\014ned) g(to)f(b)q(e)h(the)g(span)f(from)g(the)g(\014rst)g(b)o(yte)h(to)e(the)i (last)f(b)o(yte)75 1008 y(o)q(ccupied)j(b)o(y)e(en)o(tries)h(in)g(this) g(datat)o(yp)q(e,)e(rounded)i(up)f(to)g(satisfy)g(alignmen)o(t)h (requiremen)o(ts.)k(I.e.,)15 b(if)471 1113 y FD(D)q(ataty)r(pe)e FE(=)g FA(f)p FE(\()p FD(ty)r(pe)844 1120 y Fw(0)864 1113 y FD(;)8 b(disp)969 1120 y Fw(0)987 1113 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1188 1120 y Fi(n)p Fj(\000)p Fw(1)1254 1113 y FD(;)g(disp)1359 1120 y Fi(n)p Fj(\000)p Fw(1)1426 1113 y FE(\))p FA(g)p FD(;)75 1218 y(disp)159 1225 y Fi(r)190 1218 y FE(=)13 b(min)314 1225 y Fi(j)340 1218 y FD(disp)424 1225 y Fi(j)457 1218 y FE(and)i FD(disp)629 1225 y Fi(s)659 1218 y FE(=)e(max)792 1225 y Fi(j)818 1218 y FD(disp)902 1225 y Fi(j)935 1218 y FE(then)471 1335 y FD(extent)p FE(\()p FD(D)q(ataty)r(pe)p FE(\))h(=)f FD(disp)967 1342 y Fi(s)995 1335 y FE(+)d FD(siz)r(eof)5 b FE(\()p FD(ty)r(pe)1272 1342 y Fi(s)1291 1335 y FE(\))10 b FA(\000)g FD(disp)1448 1342 y Fi(r)1467 1335 y FE(;)302 b(\(4)p FD(:)p FE(1\))75 1421 y(if)18 b(furthermore,)e FD(ty)r(pe)469 1428 y Fi(i)501 1421 y FE(requires)h(alignmen)o(t)h(to)e (a)h(b)o(yte)g(address)g(that)f(is)i(is)g(a)e(m)o(ultiple)j(of)e FD(k)1743 1428 y Fi(i)1757 1421 y FE(,)g(then)75 1478 y FD(extent)p FE(\()p FD(D)q(ataty)r(pe)p FE(\))g(is)e(rounded)h(up)g (to)e(the)i(next)f(m)o(ultiple)i(of)e(max)1280 1485 y Fi(i)1302 1478 y FD(k)1326 1485 y Fi(i)1340 1478 y FE(.)166 1535 y(Example:)20 b(Assume)c(that)652 1640 y FD(T)6 b(y)r(pe)p FE(1)13 b(=)g FA(f)p FE(\()p Fp(double)n FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FA(g)75 1745 y FE(\(a)17 b Fp(double)f FE(at)h(displacemen)o(t)i(zero,)f(follo) o(w)o(ed)f(b)o(y)h(a)f Fp(char)g FE(at)g(displacemen)o(t)i(eigh)o(t\).) 26 b(Assume,)18 b(fur-)75 1801 y(thermore,)f(that)f(doubles)i(ha)o(v)o (e)f(to)f(b)q(e)i(strictly)f(aligned)h(at)f(addresses)g(that)f(are)h(m) o(ultiple)i(of)d(eigh)o(t.)75 1858 y(Then,)g(the)h(exten)o(t)e(of)h (this)h(datat)o(yp)q(e)e(is)i(16)e(\(9)h(rounded)h(to)e(the)h(next)h(m) o(ultiple)h(of)d(8\).)22 b(A)16 b(datat)o(yp)q(e)75 1914 y(that)d(consists)g(of)g(a)g(c)o(haracter)g(immediately)i(follo)o(w)o (ed)e(b)o(y)h(a)f(double)h(will)h(also)f(ha)o(v)o(e)f(an)g(exten)o(t)g (of)g(16.)75 2041 y Fq(4.13.1)49 b(Datat)o(yp)q(e)17 b(constructo)o(rs)75 2129 y FE(The)d(simplest)h(datat)o(yp)q(e)f (constructor)f(is)h Fo(MPI)p 901 2129 V 16 w(TYPE)p 1034 2129 V 17 w(CONTIGUOUS)i FE(whic)o(h)f(allo)o(ws)f(replication)h(of)75 2185 y(a)g(datat)o(yp)q(e)g(in)o(to)g(con)o(tiguous)g(lo)q(cations.)75 2290 y Fo(MPI)p 160 2290 V 16 w(TYPE)p 293 2290 V 17 w(CONTIGUOUS\(count,)i(oldt)o(yp)q(e,)f(newt)o(yp)q(e\))117 2368 y Fu(IN)155 b Fo(count)482 b Fu(replication)14 b(coun)o(t)g(\(in)o (teger\))117 2445 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Fu(handle)14 b(to)g(input)f(datat)o(yp)q(e)i(ob)r(ject)117 2522 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Fu(handle)14 b(to)g(output)g(datat)o(yp)q(e)g (ob)r(ject)166 2647 y Fo(newt)o(yp)q(e)k FE(is)e(the)g(datat)o(yp)q(e)f (obtained)i(b)o(y)f(concatenating)32 b Fo(count)17 b FE(copies)g(of)31 b Fo(oldt)o(yp)q(e)p FE(.)23 b(P)o(adding)75 2704 y(ma)o(y)14 b(b)q(e)i(added)g(to)f(satisfy)g(alignmen)o(t)h (requiremen)o(ts)f(of)g(the)g(underlying)i(arc)o(hitecture.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 73 79 73 78 bop 75 -100 a Fx(4.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(73)166 45 y(Example:)20 b(let)31 b Fo(oldt)o(yp)q(e)16 b FE(b)q(e)g(a)f(handle)i(to)d(the)h(datat)o(yp)q(e)738 144 y FA(f)p FE(\()p Fp(double)o FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FA(g)p FD(;)75 243 y FE(with)16 b(exten)o(t)e(16,)h(and)g(let)h Fo(count)e FE(=)f(3.)19 b(The)d(resulting)g(datat)o(yp)q(e)f(returned)g(b)o(y)30 b Fo(newt)o(yp)q(e)17 b FE(is)257 342 y FA(f)p FE(\()p Fp(double)o FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FD(;)g FE(\()p Fp(double)l FD(;)g FE(16\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(24\))p FD(;)g FE(\()p Fp(double)l FD(;)g FE(32\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(40\))p FA(g)p FE(;)75 442 y(i.e.,)15 b(alternating)g Fp(double)g FE(and)g Fp(char)g FE(elemen)o(ts,)g(with)h(displacemen)o (ts)g(0)p FD(;)8 b FE(8)p FD(;)g FE(16)p FD(;)g FE(24)o FD(;)f FE(32)p FD(;)g FE(40.)166 498 y(In)16 b(general,)f(assume)g (that)30 b Fo(oldt)o(yp)q(e)16 b FE(is)g(a)f(handle)h(to)f(the)g(datat) o(yp)q(e)595 597 y FA(f)p FE(\()p FD(ty)r(pe)720 604 y Fw(0)740 597 y FD(;)8 b(disp)845 604 y Fw(0)863 597 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1064 604 y Fi(n)p Fj(\000)p Fw(1)1130 597 y FD(;)g(disp)1235 604 y Fi(n)p Fj(\000)p Fw(1)1302 597 y FE(\))p FA(g)p FD(;)75 696 y FE(with)13 b(exten)o(t)g FD(extent)p FE(.)20 b(Then)26 b Fo(newt)o(yp)q(e)14 b FE(is)g(a)e(handle)i(to)e(the)h(datat)o(yp)q(e) f(with)h Fp(count)5 b FA(\001)g Fp(n)12 b FE(en)o(tries)i(de\014ned)75 753 y(b)o(y:)75 861 y FA(f)p FE(\()p FD(ty)r(pe)200 868 y Fw(0)219 861 y FD(;)8 b(disp)324 868 y Fw(0)343 861 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)544 868 y Fi(n)p Fj(\000)p Fw(1)609 861 y FD(;)g(disp)714 868 y Fi(n)p Fj(\000)p Fw(1)782 861 y FE(\))p FD(;)g FE(\()p FD(ty)r(pe)923 868 y Fw(0)941 861 y FD(;)g(disp)1046 868 y Fw(0)1065 861 y FE(+)p FD(extent)p FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1428 868 y Fi(n)p Fj(\000)p Fw(1)1495 861 y FD(;)g(disp)1600 868 y Fi(n)p Fj(\000)p Fw(1)1667 861 y FE(+)p FD(extent)p FE(\))p FD(;)g(:::;)157 960 y FE(\()p FD(ty)r(pe)259 967 y Fw(0)278 960 y FD(;)g(disp)383 967 y Fw(0)412 960 y FE(+)i FD(extent)i FA(\001)e FE(\()p Fp(count)f FA(\000)h FE(1\)\))p FD(;)e(:::;)g FE(\()o FD(ty)r(pe)1052 967 y Fi(n)p Fj(\000)p Fw(1)1118 960 y FD(;)g(disp)1223 967 y Fi(n)p Fj(\000)p Fw(1)1301 960 y FE(+)i FD(extent)i FA(\001)d FE(\()p Fp(count)g FA(\000)i FE(1\)\))p FA(g)p FD(:)166 1042 y Fo(MPI)p 251 1042 14 2 v 16 w(TYPE)p 384 1042 V 17 w(VECTOR)i FE(is)f(a)g(more)f(general)h (constructor)g(that)f(allo)o(ws)h(replication)h(of)f(a)f(datat)o(yp)q (e)75 1098 y(in)o(to)18 b(lo)q(cations)i(that)d(consist)i(of)f(equally) i(spaced)f(con)o(tiguous)f(blo)q(c)o(ks)h(of)f(equal)h(size)h({)e(blo)q (c)o(k)h(sizes)75 1155 y(and)c(blo)q(c)o(k)h(displacemen)o(ts)h(are)e (all)h(m)o(ultiples)h(of)e(the)g(old)h(t)o(yp)q(e)f(exten)o(t.)75 1258 y Fo(MPI)p 160 1258 V 16 w(TYPE)p 293 1258 V 17 w(VECTOR\()h(count,)g(oldt)o(yp)q(e,)g(stride,)f(blo)q(cklen,)h(newt)o (yp)q(e\))117 1336 y Fu(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 1410 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Fu(handle)14 b(to)g(input)f(datat)o(yp)q(e)i (ob)r(ject)117 1485 y(IN)155 b Fo(stride)484 b Fu(n)o(um)o(b)q(er)13 b(of)f(elemen)o(ts)h(b)q(et)o(w)o(een)h(start)g(of)f(eac)o(h)g(blo)q(c) o(k)g(\(in)o(te-)905 1541 y(ger\))117 1616 y(IN)155 b Fo(blo)q(cklen)433 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (eac)o(h)g(blo)q(c)o(k)g(\(p)q(ositiv)o(e)g(in)o(teger\))117 1691 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Fu(handle)14 b(to)g(output)g(datat)o(yp)q(e)g(ob)r(ject)166 1815 y FE(Example:)20 b(Assume,)15 b(again,)g(that)30 b Fo(oldt)o(yp)q(e)16 b FE(p)q(oin)o(ts)g(to)e(the)i(t)o(yp)q(e)738 1914 y FA(f)p FE(\()p Fp(double)o FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FA(g)p FD(;)75 2013 y FE(with)19 b(exten)o(t)f(16.)30 b(A)18 b(call)i(to)e Fo(MPI)p 696 2013 V 16 w(TYPE)p 829 2013 V 16 w(VECTOR\()i(2,)f(oldt)o(yp)q(e,)h(4,) f(3,)f(newt)o(yp)q(e\))i FE(will)h(create)d(the)75 2070 y(datat)o(yp)q(e)269 2169 y FA(f)p FE(\()p Fp(double)n FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FD(;)g FE(\()p Fp(double)m FD(;)g FE(16\))p FD(;)g FE(\()p Fp(char)l FD(;)g FE(24\))p FD(;)g FE(\()p Fp(double)m FD(;)g FE(32\))p FD(;)g FE(\()p Fp(char)l FD(;)g FE(40\))p FD(;)228 2268 y FE(\()p Fp(double)o FD(;)g FE(64\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(72\))p FD(;)g FE(\()p Fp(dou)o(ble)m FD(;)g FE(80\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(88\))p FD(;)g FE(\()p Fp(doub)o(le)m FD(;)g FE(96\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(104\))p FA(g)i FE(:)75 2350 y(t)o(w)o(o)k(blo)q(c)o(ks)i(with)f(three)h(copies) g(eac)o(h)f(of)g(the)g(old)h(t)o(yp)q(e,)f(starting)f(4*16)g(apart.)166 2406 y(A)h(call)h(to)f Fo(MPI)p 439 2406 V 16 w(TYPE)p 572 2406 V 17 w(VECTOR\(oldt)o(yp)q(e,)h(3,)f(-2,)g(1,)f(newt)o(yp)q (e\))j FE(will)g(create)e(the)g(datat)o(yp)q(e)186 2505 y FA(f)p FE(\()p Fp(double)o FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FD(;)g FE(\()p Fp(double)m FD(;)g FA(\000)p FE(32\))p FD(;)g FE(\()p Fp(char)m FD(;)g FA(\000)p FE(24\))p FD(;)g FE(\()p Fp(double)l FD(;)g FA(\000)p FE(64\))p FD(;)g FE(\()p Fp(char)m FD(;)g FA(\000)p FE(56\))p FA(g)p FD(:)166 2604 y FE(In)16 b(general,)f(assume)g(that)30 b Fo(oldt)o(yp)q(e)16 b FE(is)g(a)f(handle)h(to)f(the)g(datat)o(yp)q(e) 595 2704 y FA(f)p FE(\()p FD(ty)r(pe)720 2711 y Fw(0)740 2704 y FD(;)8 b(disp)845 2711 y Fw(0)863 2704 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1064 2711 y Fi(n)p Fj(\000)p Fw(1)1130 2704 y FD(;)g(disp)1235 2711 y Fi(n)p Fj(\000)p Fw(1)1302 2704 y FE(\))p FA(g)p FD(;)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 74 80 74 79 bop 75 -100 a FE(74)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(with)d(exten)o(t) g FD(extent)p FE(.)21 b(The)13 b(newly)h(created)f(datat)o(yp)q(e)f(is) h(a)g(sequence)h(of)f(length)h Fp(count)5 b FA(\001)h Fp(blocklen)s FA(\001)g FD(n)75 102 y FE(with)16 b(en)o(tries:)606 164 y FA(f)p FE(\()p FD(ty)r(pe)731 171 y Fw(0)751 164 y FD(;)8 b(disp)856 171 y Fw(0)874 164 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1075 171 y Fi(n)p Fj(\000)p Fw(1)1141 164 y FD(;)g(disp)1246 171 y Fi(n)p Fj(\000)p Fw(1)1313 164 y FE(\))p FD(;)405 259 y FE(\()p FD(ty)r(pe)507 266 y Fw(0)527 259 y FD(;)g(disp)632 266 y Fw(0)660 259 y FE(+)j FD(extent)p FE(\))p FD(;)d(:::;)g FE(\()p FD(ty)r(pe)1034 266 y Fi(n)p Fj(\000)p Fw(1)1101 259 y FD(;)g(disp)1206 266 y Fi(n)p Fj(\000)p Fw(1)1283 259 y FE(+)i FD(extent)p FE(\))p FD(;)e(:::;)96 355 y FE(\()p FD(ty)r(pe)198 362 y Fw(0)218 355 y FD(;)g(disp)323 362 y Fw(0)352 355 y FE(+)i(\()p Fp(blocklen)f FA(\000)h FE(1\))g FA(\001)g FD(extent)p FE(\))p FD(;)e(:::;)g FE(\()p FD(ty)r(pe)1063 362 y Fi(n)p Fj(\000)p Fw(1)1130 355 y FD(;)g(disp)1235 362 y Fi(n)p Fj(\000)p Fw(1)1312 355 y FE(+)i(\()p Fp(blocklen)f FA(\000)i FE(1\))e FA(\001)h FD(extent)p FE(\))p FD(;)229 450 y FE(\()p FD(ty)r(pe)331 457 y Fw(0)351 450 y FD(;)e(disp)456 457 y Fw(0)484 450 y FE(+)j Fp(stride)e FA(\001)h FD(extent)p FE(\))p FD(;)e(:::;)g FE(\()p FD(ty)r(pe)1034 457 y Fi(n)p Fj(\000)p Fw(1)1101 450 y FD(;)g(disp)1206 457 y Fi(n)p Fj(\000)p Fw(1)1283 450 y FE(+)i Fp(stride)g FA(\001)f FD(extent)p FE(\))p FD(;)f(:::;)448 546 y FE(\()p FD(ty)r(pe)550 553 y Fw(0)570 546 y FD(;)g(disp)675 553 y Fw(0)703 546 y FE(+)j(\()p Fp(stride)e FE(+)h Fp(blocklen)f FA(\000)i FE(1\))e FA(\001)h FD(extent)p FE(\))p FD(;)e(:::;)393 641 y FE(\()p FD(ty)r(pe)495 648 y Fi(n)p Fj(\000)p Fw(1)564 641 y FD(;)g(disp)669 648 y Fi(n)p Fj(\000)p Fw(1)746 641 y FE(+)i(\()p Fp(stride)f FE(+)i Fp(blocklen)e FA(\000)h FE(1\))g FA(\001)g FD(extent)p FE(\))p FD(;)e(::::;)495 737 y FE(\()p FD(ty)r(pe)597 744 y Fw(0)617 737 y FD(;)g(disp)722 744 y Fw(0)751 737 y FE(+)i Fp(stride)f FA(\001)h FE(\()p Fp(count)f FA(\000)i FE(1\))e FA(\001)h FD(extent)p FE(\))p FD(;)e(:::;)446 833 y FE(\()p FD(ty)r(pe)548 840 y Fi(n)p Fj(\000)p Fw(1)617 833 y FD(;)g(disp)722 840 y Fi(n)p Fj(\000)p Fw(1)799 833 y FE(+)j Fp(stride)e FA(\001)h FE(\()p Fp(count)f FA(\000)h FE(1\))g FA(\001)g FD(extent)p FE(\))p FD(;)e(:::;)315 928 y FE(\()p FD(ty)r(pe)417 935 y Fw(0)437 928 y FD(;)g(disp)542 935 y Fw(0)571 928 y FE(+)i(\()p Fp(stride)f FA(\001)h FE(\()p Fp(count)f FA(\000)h FE(1\))g(+)g Fp(blocklen)f FA(\000)i FE(1\))e FA(\001)h FD(extent)p FE(\))p FD(;)e(:::;)290 1024 y FE(\()p FD(ty)r(pe)392 1031 y Fi(n)p Fj(\000)p Fw(1)461 1024 y FD(;)g(disp)566 1031 y Fi(n)p Fj(\000)p Fw(1)643 1024 y FE(+)j(\()p Fp(stride)e FA(\001)h FE(\()p Fp(count)f FA(\000)h FE(1\))g(+)g Fp(blocklen)f FA(\000)i FE(1\))e FA(\001)h FD(extent)p FE(\))p FA(g)166 1191 y Fv(Discussion:)166 1243 y Fu(I)k(c)o(hanged)28 b Fr(count)15 b Fu(from)d(b)q(eing)i(the)g (n)o(um)o(b)q(er)f(of)h(en)o(tries)h(\(or)e(replicates\),)i(as)f(in)f (the)i(old)e(draft,)g(to)h(b)q(eing)75 1293 y(the)i(n)o(um)o(b)q(er)e (of)h(blo)q(c)o(ks.)22 b(The)16 b(reason)g(is)f(that)h(in)e(the)i(next) g(calls)f(I)g(allo)o(w)f(blo)q(c)o(k)h(lengths)g(to)g(v)n(ary;)g(it)g (is)h(easy)75 1343 y(to)f(sa)o(y)f(what)h(it)f(means)g(to)g(ha)o(v)o(e) h Fm(n)f Fu(blo)q(c)o(ks)h(of)f(sizes)i Fm(b)950 1349 y Fh(0)969 1343 y Fm(;)7 b(:::;)g(b)1061 1349 y Fg(n)o Ff(\000)p Fh(1)1123 1343 y Fu(;)14 b(it)h(is)f(harder)i(to)e(explain)g (what)h(it)f(means)g(to)75 1393 y(ha)o(v)o(e)g Fm(n)f Fu(elemen)o(ts)h(\014lling)f(up)g(successiv)o(e)k(p)q(ositions)c(in)h (blo)q(c)o(ks)f(of)h(sizes)h Fm(b)1256 1399 y Fh(0)1274 1393 y Fm(;)7 b(b)1311 1399 y Fh(1)1329 1393 y Fm(;)g(::::)p Fu(.)166 1444 y(W)m(e)19 b(lo)q(ose)h(the)h(abilit)o(y)d(to)h(use)i Fr(MPI)p 776 1444 13 2 v 14 w(TYPE)p 898 1444 V 15 w(VECTOR)d Fu(to)i(create)h(a)e(partially)g(\014lled)g(last)h(blo)q(c)o(k)f(\(the) 75 1494 y(more)11 b(general)i Fr(MPI)p 396 1494 V 14 w(TYPE)p 518 1494 V 14 w(INDEXED)f Fu(function)f(is)i(needed\);)g(w)o (e)g(gain)e(consistency)j(with)e(the)h(next)f(calls,)g(and)75 1544 y(con)o(v)o(enience)j(in)f(these)h(calls.)166 1596 y(Opinions?)166 1737 y FE(A)h(call)i(to)e Fo(MPI)p 443 1737 14 2 v 16 w(TYPE)p 576 1737 V 16 w(CONTIGUOUS\(oldt)o(yp)q(e,)j (count,)f(newt)o(yp)q(e\))g FE(is)f(equiv)m(alen)o(t)h(to)e(a)g(call)i (to)75 1793 y Fo(MPI)p 160 1793 V 16 w(TYPE)p 293 1793 V 17 w(VECTOR\(oldt)o(yp)q(e,)e(count,)h(1,)d(1,)h(newt)o(yp)q(e\))p FE(.)166 1851 y(The)k(function)g Fo(MPI)p 529 1851 V 16 w(TYPE)p 662 1851 V 17 w(HVECTOR)h FE(is)g(iden)o(tical)g(to)e Fo(MPI)p 1287 1851 V 16 w(TYPE)p 1420 1851 V 17 w(VECTOR)p FE(,)h(except)h(that)75 1908 y Fo(stride)12 b FE(is)f(giv)o(en)h(in)f (b)o(ytes,)g(rather)g(then)g(in)h(elemen)o(ts.)19 b(The)11 b(use)g(for)f(b)q(oth)h(t)o(yp)q(es)g(of)g(v)o(ector)f(constructors)75 1964 y(is)16 b(illustrated)g(in)g(Section)g(4.13.4.)i(\()p Fp(H)d FE(is)h(used)g(for)e(\\heterogeneous"\).)75 2070 y Fo(MPI)p 160 2070 V 16 w(TYPE)p 293 2070 V 17 w(HVECTOR\()i(count,)g (oldt)o(yp)q(e,)g(stride,)g(blo)q(cklen,)g(newt)o(yp)q(e\))117 2149 y Fu(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 2228 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Fu(handle)14 b(to)g(input)f(datat)o(yp)q(e)i (ob)r(ject)117 2307 y(IN)155 b Fo(stride)484 b Fu(n)o(um)o(b)q(er)13 b(of)g(b)o(ytes)h(b)q(et)o(w)o(een)h(start)f(of)f(eac)o(h)h(blo)q(c)o (k)f(\(in)o(teger\))117 2385 y(IN)155 b Fo(blo)q(cklen)433 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o (k)g(\(p)q(ositiv)o(e)g(in)o(teger\))117 2464 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Fu(handle)14 b(to)g(output)g(datat)o(yp)q(e)g (ob)r(ject)166 2591 y Fo(MPI)p 251 2591 V 16 w(TYPE)p 384 2591 V 17 w(INDEXED)24 b FE(is)h(a)f(more)g(general)h(function)g (that)f(allo)o(ws)g(replication)i(of)e(an)g(old)75 2647 y(datat)o(yp)q(e)12 b(in)o(to)h(a)f(sequence)i(of)f(con)o(tiguous)g (blo)q(c)o(ks,)g(where)g(blo)q(c)o(k)h(lengths)f(and)g(blo)q(c)o(k)h (displacemen)o(ts)75 2704 y(can)h(all)h(di\013er)g({)f(but)g(are)g(all) h(m)o(ultiples)h(of)e(the)g(old)h(t)o(yp)q(e)f(exten)o(t.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 75 81 75 80 bop 75 -100 a Fx(4.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(75)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(INDEXED\()14 b(count,)j(oldt)o(yp)q(e,)e(a)o(rra)o(y)p 932 45 V 15 w(of)p 984 45 V 16 w(indices,)h(a)o(rra)o(y)p 1249 45 V 15 w(of)p 1301 45 V 16 w(blo)q(cklen,)g(newt)o(yp)q(e\))117 124 y Fu(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 202 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Fu(handle)14 b(to)g(input)f(datat)o(yp)q(e)i (ob)r(ject)117 281 y(IN)155 b Fo(a)o(rra)o(y)p 416 281 V 15 w(of)p 468 281 V 16 w(indices)299 b Fu(displacemen)o(t)17 b(for)g(eac)o(h)h(blo)q(c)o(k,)f(in)g(m)o(ultiples)f(of)34 b Fr(oldt)o(yp)q(e)905 337 y Fu(exten)o(t)15 b(\(arra)o(y)f(of)f(in)o (teger\))117 416 y(IN)155 b Fo(a)o(rra)o(y)p 416 416 V 15 w(of)p 468 416 V 16 w(blo)q(cklen)270 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(p)q(er)i(blo)q(c)o(k)f(\(arra)o(y)f(of)h(in)o (teger\))117 494 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Fu(handle)14 b(to)g(output)g(datat)o(yp)q(e)g(ob)r(ject)166 620 y FE(Example:)20 b(let)31 b Fo(oldt)o(yp)q(e)16 b FE(b)q(e)g(a)f(handle)i (to)d(the)h(t)o(yp)q(e)738 727 y FA(f)p FE(\()p Fp(double)o FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FA(g)p FD(;)75 834 y FE(with)17 b(exten)o(t)g(16.)25 b(Let)17 b Fp(I)24 b(=)f(\(64,)h(0\))16 b FE(and)i(let)f Fp(B)24 b(=)f(\(3,)h(1\))p FE(.)h(A)17 b(call)h(to)e Fo(MPI)p 1480 834 V 16 w(TYPE)p 1613 834 V 17 w(INDEXED\(2,)75 890 y(oldt)o(yp)q(e,)g(I,)e(B,)h(newt)o(yp)q(e\))i FE(returns)e(in)h Fp(newtype)e FE(a)h(handle)i(to)d(the)i(t)o(yp)q(e)241 997 y FA(f)p FE(\()p Fp(double)n FD(;)8 b FE(64\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(72\))p FD(;)g FE(\()p Fp(double)l FD(;)g FE(80\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(88\))p FD(;)g FE(\()p Fp(double)l FD(;)g FE(96\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(104\))743 1109 y(\()p Fp(double)o FD(;)g FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FA(g)j FE(:)75 1199 y(three)k(copies)h(of)e(the)i(old)f(t)o(yp)q(e)g (starting)f(at)h(displacemen)o(t)h(64,)e(and)h(one)h(cop)o(y)e (starting)h(at)f(displace-)75 1255 y(men)o(t)h(0.)166 1313 y(In)h(general,)f(assume)g(that)30 b Fo(oldt)o(yp)q(e)16 b FE(is)g(a)f(handle)h(to)f(the)g(datat)o(yp)q(e)595 1420 y FA(f)p FE(\()p FD(ty)r(pe)720 1427 y Fw(0)740 1420 y FD(;)8 b(disp)845 1427 y Fw(0)863 1420 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1064 1427 y Fi(n)p Fj(\000)p Fw(1)1130 1420 y FD(;)g(disp)1235 1427 y Fi(n)p Fj(\000)p Fw(1)1302 1420 y FE(\))p FA(g)p FD(;)75 1527 y FE(with)14 b(exten)o(t)g Ft(extent)p FE(.)19 b(Let)29 b Fo(B)14 b FE(b)q(e)h(the)28 b Fo(a)o(rra)o(y)p 855 1527 V 15 w(of)p 907 1527 V 16 w(blo)q(cklen)15 b FE(parameter)f(and)28 b Fo(I)14 b FE(b)q(e)h(the)28 b Fo(a)o(rra)o(y)p 1684 1527 V 15 w(of)p 1736 1527 V 15 w(indices)75 1584 y FE(parameter.)18 b(The)13 b(newly)h(created)e(t)o(yp)q(e)h(returned)g(b)o(y)25 b Fo(newt)o(yp)q(e)15 b FE(is)e(the)g(sequence)h(of)e FD(n)5 b FA(\001)1612 1551 y Fe(P)1656 1564 y Fd(count)n Fj(\000)p Fw(1)1656 1596 y Fi(i)p Fw(=0)1809 1584 y Fo(B)p FE([)p Fo(i)p FE(])75 1640 y(en)o(tries)300 1701 y FA(f)p FE(\()p FD(ty)r(pe)425 1708 y Fw(0)445 1701 y FD(;)j(disp)550 1708 y Fw(0)578 1701 y FE(+)i Fo(I)p FE([)p Fo(0)p FE(])f FA(\001)h FD(extent)p FE(\))p FD(;)e(:::;)g FE(\()p FD(ty)r(pe)1045 1708 y Fi(n)p Fj(\000)p Fw(1)1112 1701 y FD(;)g(disp)1217 1708 y Fi(n)p Fj(\000)p Fw(1)1294 1701 y FE(+)j Fo(I)p FE([)p Fo(0)p FE(])d FA(\001)i FD(extent)p FE(\))p FD(;)e(:::;)75 1795 y FE(\()p FD(ty)r(pe)177 1802 y Fw(0)197 1795 y FD(;)g(disp)302 1802 y Fw(0)329 1795 y FE(+)h(\()p Fo(I)p FE([)p Fo(0)p FE(])e(+)i Fo(B)p FE([)p Fo(0)p FE(])f FA(\000)h FE(1\))f FA(\001)h FD(extent)p FE(\))p FD(;)f(:::;)g FE(\()p FD(ty)r(pe)1034 1802 y Fi(n)p Fj(\000)p Fw(1)1101 1795 y FD(;)g(disp)1206 1802 y Fi(n)p Fj(\000)p Fw(1)1282 1795 y FE(+)h(\()p Fo(I)p FE([)p Fo(0)p FE(])e(+)i Fo(B)p FE([)p Fo(0)p FE(])f FA(\000)h FE(1\))f FA(\001)g FD(extent)p FE(\))p FD(;)g(:::;)150 1889 y FE(\()p FD(ty)r(pe)252 1896 y Fw(0)271 1889 y FD(;)g(disp)376 1896 y Fw(0)405 1889 y FE(+)i Fo(I)p FE([)p Fo(count)h FA(\000)f Fo(1)p FE(])g FA(\001)g FD(extent)p FE(\))p FD(;)e(:::;)g FE(\()p FD(ty)r(pe)1034 1896 y Fi(n)p Fj(\000)p Fw(1)1101 1889 y FD(;)g(disp)1206 1896 y Fi(n)p Fj(\000)p Fw(1)1283 1889 y FE(+)i Fo(I)p FE([)p Fo(count)h FA(\000)f Fo(1)p FE(])g FA(\001)g FD(extent)p FE(\))p FD(;)e(:::;)384 1983 y FE(\()p FD(ty)r(pe)486 1990 y Fw(0)506 1983 y FD(;)g(disp)611 1990 y Fw(0)639 1983 y FE(+)j(\()p Fo(I)p FE([)p Fo(count)f FA(\000)g Fo(1)p FE(])g(+)g Fo(B)p FE([)p Fo(count)i FA(\000)e Fo(1)p FE(])g FA(\000)g FE(1\))g FA(\001)f FD(extent)p FE(\))p FD(;)f(:::;)353 2077 y FE(\()p FD(ty)r(pe)455 2084 y Fi(n)p Fj(\000)p Fw(1)523 2077 y FD(;)g(disp)628 2084 y Fi(n)p Fj(\000)p Fw(1)706 2077 y FE(+)i(\()p Fo(I)p FE([)p Fo(count)h FA(\000)f Fo(1)p FE(])f(+)i Fo(B)p FE([)p Fo(count)g FA(\000)g Fo(1)p FE(])e FA(\000)i FE(1\))e FA(\001)h FD(extent)p FE(\))p FA(g)p FD(:)166 2168 y FE(A)k(call)g(to)f Fo(MPI)p 434 2168 V 16 w(TYPE)p 567 2168 V 17 w(VECTOR\(count,)j(oldt)o(yp)q(e,) f(stride,)f(blo)q(cklen,)h(newt)o(yp)q(e\))g FE(is)f(equiv)m(alen)o(t)i (to)75 2225 y(a)f(call)h(to)f Fo(MPI)p 337 2225 V 16 w(TYPE)p 470 2225 V 16 w(INDEX\(count,)h(oldt)o(yp)q(e,)g(I,)f(B,)g (newt)o(yp)q(e\))h FE(where)626 2332 y Fo(I)p FE([)p Fo(j)p FE(])c(=)h FD(j)f FA(\001)e Fo(stride)16 b FD(;)8 b(j)14 b FE(=)f(0)p FD(;)8 b(:::;)g Fp(count)f FA(\000)j FE(1)p FD(;)75 2439 y FE(and)619 2500 y Fo(B)p FE([)p Fo(j)p FE(])i(=)h Fo(blo)q(cklen)k FD(;)8 b(j)14 b FE(=)f(0)p FD(;)8 b(:::;)g Fp(cou)o(nt)f FA(\000)k FE(1)p FD(:)166 2591 y FE(The)k(function)h Fo(MPI)p 522 2591 V 16 w(TYPE)p 655 2591 V 17 w(HINDEXED)f FE(is)h(iden)o(tical)g(to)f Fo(MPI)p 1280 2591 V 16 w(TYPE)p 1413 2591 V 16 w(INDEXED)p FE(,)g(except)h(that)75 2647 y(blo)q(c)o(k)j(displacemen)o(ts)i(in)38 b Fo(a)o(rra)o(y)p 665 2647 V 14 w(of)p 716 2647 V 16 w(indices)20 b FE(are)f(sp)q(eci\014ed)h(in)g(b)o(ytes,)f(rather)f (than)g(in)i(m)o(ultiples)g(of)75 2704 y(the)15 b(old)h(t)o(yp)q(e)f (exten)o(t.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 76 82 76 81 bop 75 -100 a FE(76)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(HINDEXED\()f(count,)h(oldt)o (yp)q(e,)g(a)o(rra)o(y)p 965 45 V 14 w(of)p 1016 45 V 16 w(indices,)h(a)o(rra)o(y)p 1282 45 V 14 w(of)p 1333 45 V 16 w(blo)q(cklen,)f(newt)o(yp)q(e\))117 123 y Fu(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o (teger\))117 198 y(IN)155 b Fo(oldt)o(yp)q(e)450 b Fu(handle)14 b(to)g(input)f(datat)o(yp)q(e)i(ob)r(ject)117 274 y(IN)155 b Fo(a)o(rra)o(y)p 416 274 V 15 w(of)p 468 274 V 16 w(indices)299 b Fu(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h(blo)q(c)o(k)g (\(arra)o(y)g(of)f(in)o(teger\))117 350 y(IN)155 b Fo(a)o(rra)o(y)p 416 350 V 15 w(of)p 468 350 V 16 w(blo)q(cklen)270 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o(k)g (\(arra)o(y)g(of)f(in)o(teger\))117 425 y(OUT)108 b Fo(newt)o(yp)q(e) 433 b Fu(handle)14 b(to)g(output)g(datat)o(yp)q(e)g(ob)r(ject)166 550 y Fp(MPI)p 241 550 15 2 v 17 w(TYPE)p 354 550 V 16 w(STRUCT)j FE(is)h(the)g(most)e(general)i(constructor:)24 b(It)17 b(further)h(generalizes)h(the)e(previous)75 607 y(one)e(in)h(that)f(it)g(allo)o(ws)h(eac)o(h)f(blo)q(c)o(k)h(to)e (consist)i(of)f(replications)i(of)d(a)h(di\013eren)o(t)h(datat)o(yp)q (e.)75 711 y Fo(MPI)p 160 711 14 2 v 16 w(TYPE)p 293 711 V 17 w(STRUCT\(count,)g(a)o(rra)o(y)p 733 711 V 15 w(of)p 785 711 V 16 w(t)o(yp)q(es,)g(a)o(rra)o(y)p 1022 711 V 14 w(of)p 1073 711 V 16 w(indices,)h(a)o(rra)o(y)p 1339 711 V 14 w(of)p 1390 711 V 16 w(blo)q(cklen,)f(newt)o(yp)q(e\))117 788 y Fu(IN)155 b Fo(count)482 b Fu(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))117 864 y(IN)155 b Fo(a)o(rra)o(y)p 416 864 V 15 w(of)p 468 864 V 16 w(t)o(yp)q(es)327 b Fu(t)o(yp)q(e)15 b(of)f(elemen)o(ts)g(in)g(eac)o(h)h(blo)q(c)o(k)f (\(arra)o(y)g(of)g(datat)o(yp)q(e)g(ob-)905 920 y(jects\))117 996 y(IN)155 b Fo(a)o(rra)o(y)p 416 996 V 15 w(of)p 468 996 V 16 w(indices)299 b Fu(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o (h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1072 y(IN)155 b Fo(a)o(rra)o(y)p 416 1072 V 15 w(of)p 468 1072 V 16 w(blo)q(cklen)270 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in) o(teger\))117 1147 y(OUT)108 b Fo(newt)o(yp)q(e)433 b Fu(handle)14 b(to)g(output)g(datat)o(yp)q(e)g(ob)r(ject)166 1272 y FE(Example:)20 b(Let)31 b Fo(t)o(yp)q(e1)16 b FE(b)q(e)g(a)f(handle)h(to)f(the)g(t)o(yp)q(e)738 1375 y FA(f)p FE(\()p Fp(double)o FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(8\))p FA(g)p FD(;)75 1478 y FE(with)24 b(exten)o(t)g(16.)45 b(Let)24 b Fp(I)g(=)g(\(0,)f(16,)g (26\))p FE(,)j Fp(B)d(=)h(\(2,)g(1,)f(3\))h FE(and)g Fp(T)f(=)h(\(MPI)p 1551 1478 15 2 v 17 w(FLOAT,)f(type1,)75 1535 y(MPI)p 150 1535 V 17 w(BYTE\))p FE(.)18 b(Then)i(a)f(call)h(to)f Fo(MPI)p 712 1535 14 2 v 16 w(TYPE)p 845 1535 V 16 w(STRUC\(3,)h(T,)f (I,)g(B,)g(newt)o(yp)q(e\))i FE(returns)e(in)39 b Fo(newt)o(yp)q(e)21 b FE(a)75 1591 y(handle)16 b(to)f(the)g(datat)o(yp)q(e)166 1694 y FA(f)p FE(\()p Fp(float)o FD(;)8 b FE(0\))p FD(;)g FE(\()p Fp(float)m FD(;)g FE(4\))p FD(;)g FE(\()p Fp(double)m FD(;)g FE(16\))p FD(;)g FE(\()p Fp(cha)o(r)m FD(;)g FE(24\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(26\))p FD(;)g FE(\()p Fp(char)m FD(;)g FE(27\))p FD(;)g FE(\()p Fp(cha)o(r)m FD(;)g FE(28\))p FA(g)j FE(:)75 1797 y(t)o(w)o(o)19 b(copies)i(of)40 b Fo(MPI)p 467 1797 V 16 w(FLO)o(A)l(T)20 b FE(starting)g(at)g(0,)g (follo)o(w)o(ed)h(b)o(y)f(one)h(cop)o(y)f(of)40 b Fo(t)o(yp)q(e1)21 b FE(starting)e(at)h(16,)75 1853 y(follo)o(w)o(ed)d(b)o(y)f(three)h (copies)g(of)33 b Fo(MPI)p 724 1853 V 16 w(CHAR)p FE(,)17 b(starting)e(at)h(26.)23 b(\(W)l(e)17 b(assume)f(that)g(a)33 b Fo(\015oat)16 b FE(o)q(ccupies)75 1910 y(four)f(b)o(ytes.\))166 1967 y(In)h(general,)f(let)31 b Fo(T)15 b FE(b)q(e)h(the)30 b Fo(a)o(rra)o(y)p 772 1967 V 15 w(of)p 824 1967 V 16 w(t)o(yp)q(es)16 b FE(parameter,)e(where)31 b Fo(T[i])15 b FE(is)h(a)f(handle)h(to)547 2070 y FD(ty)r(pe)631 2077 y Fi(i)659 2070 y FE(=)d FA(f)p FE(\()p FD(ty)r(pe)832 2051 y Fi(i)832 2081 y Fw(0)851 2070 y FD(;)8 b(disp)956 2051 y Fi(i)956 2081 y Fw(0)974 2070 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1175 2051 y Fi(i)1175 2081 y(n)1196 2086 y Fc(i)1209 2070 y FD(;)g(disp)1314 2051 y Fi(i)1314 2081 y(n)1335 2086 y Fc(i)1349 2070 y FE(\))p FA(g)p FD(;)75 2173 y FE(with)13 b(exten)o(t)f FD(extent)439 2180 y Fi(i)455 2173 y FE(.)19 b(Let)25 b Fo(B)13 b FE(b)q(e)h(the)25 b Fo(a)o(rra)o(y)p 867 2173 V 15 w(of)p 919 2173 V 16 w(blo)q(cklen)14 b FE(parameter)d(and)26 b Fo(I)12 b FE(b)q(e)i(the)25 b Fo(a)o(rra)o(y)p 1684 2173 V 15 w(of)p 1736 2173 V 15 w(indices)75 2229 y FE(parameter.)18 b(Then)12 b(the)f(newly)i(created)e(datat)o(yp)q(e)g(is)h(the)g(sequence)g(of)f (length)1452 2197 y Fe(P)1495 2210 y Fd(count)n Fj(\000)p Fw(1)1495 2242 y Fi(i)p Fw(=0)1648 2229 y Fo(B)p FE([)p Fo(i)p FE(])r FA(\001)s FD(extent)1860 2236 y Fi(i)75 2286 y FE(with)16 b(en)o(tries)489 2343 y FA(f)p FE(\()p FD(ty)r(pe)614 2324 y Fw(0)614 2354 y(0)634 2343 y FD(;)8 b(disp)739 2324 y Fw(0)739 2354 y(0)767 2343 y FE(+)j Fo(I)p FE([)p Fo(0)p FE(])n(\))p FD(;)d(:::;)g FE(\()p FD(ty)r(pe)1074 2324 y Fw(0)1074 2354 y Fi(n)1095 2359 y FC(0)1112 2343 y FD(;)g(disp)1217 2324 y Fw(0)1217 2354 y Fi(n)1238 2359 y FC(0)1266 2343 y FE(+)j Fo(I)p FE([)p Fo(0)p FE(])o(\))p FD(;)d(:::;)75 2428 y FE(\()p FD(ty)r(pe)177 2409 y Fw(0)177 2439 y(0)197 2428 y FD(;)g(disp)302 2409 y Fw(0)302 2439 y(0)330 2428 y FE(+)i Fo(I)p FE([)p Fo(0)p FE(])e(+)i(\()p Fo(B)p FE([)p Fo(0)p FE(])f FA(\000)h FE(1\))f FA(\001)g FD(extent)840 2435 y Fw(0)861 2428 y FE(\))p FD(;)f(:::;)g FE(\()p FD(ty)r(pe)1062 2409 y Fw(0)1062 2439 y Fi(n)1083 2444 y FC(0)1100 2428 y FD(;)g(disp)1205 2409 y Fw(0)1205 2439 y Fi(n)1226 2444 y FC(0)1254 2428 y FE(+)i Fo(I)p FE([)p Fo(0)p FE(])f(+)h(\()p Fo(B)p FE([)p Fo(0)p FE(])f FA(\000)h Fo(1)p FE(\))f FA(\001)g FD(extent)1765 2435 y Fw(0)1787 2428 y FE(\))p FD(;)f(:::;)91 2513 y FE(\()p FD(ty)r(pe)193 2494 y Fd(count)n Fj(\000)p Fw(1)193 2526 y(0)338 2513 y FD(;)g(disp)443 2494 y Fd(count)m Fj(\000)p Fw(1)443 2526 y(0)597 2513 y FE(+)j Fo(I)p FE([)p Fo(count)f FA(\000)h Fo(1)p FE(])o(\))p FD(;)d(:::;)g FE(\()p FD(ty)r(pe)1066 2495 y Fd(co)o(unt)l Fj(\000)p Fw(1)1066 2525 y Fi(n)1087 2534 y Fd(cou)o(nt)m Fb(\000)p FC(1)1226 2513 y FD(;)g(disp)1331 2495 y Fd(count)m Fj(\000)p Fw(1)1331 2525 y Fi(n)1352 2534 y Fd(count)m Fb(\000)p FC(1)1503 2513 y FE(+)i Fo(I)p FE([)p Fo(count)h FA(\000)f Fo(1)p FE(]\))p FD(;)e(:::;)186 2611 y FE(\()p FD(ty)r(pe)288 2592 y Fd(count)n Fj(\000)p Fw(1)288 2623 y(0)433 2611 y FD(;)g(disp)538 2592 y Fd(count)m Fj(\000)p Fw(1)538 2623 y(0)692 2611 y FE(+)j Fo(I)p FE([)p Fo(count)f FA(\000)h Fo(1)p FE(])e(+)i(\()p Fo(B)p FE([)p Fo(count)g FA(\000)f Fo(1)p FE(])f FA(\000)i FE(1\))e FA(\001)h FD(extent)1529 2618 y Fd(count)p Fj(\000)p Fw(1)1676 2611 y FE(\))p FD(;)e(:::;)192 2696 y FE(\()p FD(ty)r(pe)294 2677 y Fd(count)n Fj(\000)p Fw(1)294 2707 y Fi(n)315 2717 y Fd(count)o Fb(\000)p FC(1)457 2696 y FD(;)g(disp)562 2677 y Fd(count)m Fj(\000)p Fw(1)562 2707 y Fi(n)583 2717 y Fd(count)m Fb(\000)p FC(1)734 2696 y FE(+)i Fo(I)p FE([)p Fo(count)h FA(\000)f Fo(1)p FE(])g(+)g(\()p Fo(B)p FE([)p Fo(count)h FA(\000)g Fo(1)p FE(])e FA(\000)i Fo(1)p FE(\))e FA(\001)h FD(extent)1571 2703 y Fd(count)o Fj(\000)p Fw(1)1717 2696 y FE(\))p FA(g)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 77 83 77 82 bop 75 -100 a Fx(4.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(77)166 45 y(A)17 b(call)g(to)f Fo(MPI)p 443 45 14 2 v 16 w(TYPE)p 576 45 V 17 w(HINDEXED\()g(count,)i(oldt)o(yp)q (e,)g(I,)e(B,)g(newt)o(yp)q(e\))i FE(is)f(equiv)m(alen)o(t)i(to)d(a)g (call)75 102 y(to)f Fo(MPI)p 216 102 V 15 w(TYPE)p 348 102 V 17 w(STRUC\()h(count,)g(T,)f(I,)f(B,)h(newt)o(yp)q(e\))p FE(,)h(where)g(eac)o(h)f(en)o(try)g(of)g Fo(T)g FE(is)g(equal)h(to)30 b Fo(oldt)o(yp)q(e)p FE(.)166 237 y Fv(Discussion:)166 290 y Fu(I)14 b(am)e(op)q(en)i(to)g(suggestions)g(for)g(b)q(etter)h (names)f(to)f(these)j(functions.)75 512 y Fq(4.13.2)49 b(Additional)19 b(functions)75 604 y FE(The)i(displacemen)o(ts)h(in)g (a)f(general)g(datat)o(yp)q(e)f(are)h(relativ)o(e)g(to)g(some)f(start)g (address.)37 b FF(Absolute)75 660 y(addresses)13 b FE(can)h(b)q(e)h (substituted)g(for)e(these)i(displacemen)o(ts:)21 b(w)o(e)13 b(treat)h(them)g(as)g(displacemen)o(ts)h(rel-)75 717 y(ativ)o(e)f(to)e(\\address)i(zero",)f(the)h(start)e(of)h(the)h (address)f(space.)20 b(This)14 b(initial)i(address)e(zero)f(is)h (indicated)75 773 y(b)o(y)k(the)g(constan)o(t)33 b Fr(MPI)p 501 773 13 2 v 14 w(BOTTOM)p FE(.)17 b(Th)o(us,)h(a)g(datat)o(yp)q(e)f (can)h(sp)q(ecify)h(the)f(absolute)h(address)e(of)h(the)75 830 y(en)o(tries)e(in)g(the)f(comm)o(unication)h(bu\013er,)f(in)h(whic) o(h)g(case)f(the)31 b Fo(sta)o(rt)16 b FE(parameter)e(is)i(passed)f (the)h(v)m(alue)75 886 y Fr(MPI)p 152 886 V 14 w(BOTTOM)p FE(.)166 946 y(The)10 b(address)g(of)g(a)g(lo)q(cation)h(in)g(memory)f (can)g(b)q(e)h(found)f(b)o(y)g(in)o(v)o(oking)h(the)f(function)h Fo(MPI)p 1701 946 14 2 v 16 w(ADDRESS)p FE(.)75 1053 y Fo(MPI)p 160 1053 V 16 w(ADDRESS\(lo)q(cation,)16 b(address\))117 1133 y Fu(IN)155 b Fo(lo)q(cation)437 b Fu(lo)q(cation)13 b(in)h(caller)f(memory)f(\(c)o(hoice\))117 1214 y(OUT)108 b Fo(address)449 b Fu(address)15 b(of)f(lo)q(cation)f(\(in)o(teger\)) 166 1342 y FE(Returns)j(the)f(\(b)o(yte\))f(address)i(of)29 b Fo(lo)q(cation)p FE(.)166 1401 y(Another)11 b(useful)g(auxiliary)h (function)g(is)f Fo(MPI)p 950 1401 V 16 w(EXTENT)p FE(,)f(that)g (returns)h(the)g(exten)o(t)f(of)g(a)g(datat)o(yp)q(e)75 1458 y({)15 b(where)g(exten)o(t)g(is)h(as)f(de\014ned)h(in)g(Eq.)f (4.1.)75 1565 y Fo(MPI)p 160 1565 V 16 w(TYPE)p 293 1565 V 17 w(EXTENT\(datat)o(yp)q(e,)h(extent\))117 1645 y Fu(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(handle)14 b(to)g(datat)o(yp)q(e) g(ob)r(ject)117 1726 y(OUT)108 b Fo(extent)471 b Fu(datat)o(yp)q(e)14 b(exten)o(t)h(\(in)o(teger\))166 1854 y FE(As)i(de\014ned)i(no)o(w,)e (the)h(exten)o(t)f(of)g(a)g(datat)o(yp)q(e)g(go)q(es)g(from)g(its)h (\014rst)f(o)q(ccupied)i(b)o(yte)f(to)e(its)i(last)75 1910 y(o)q(ccupied)23 b(b)o(yte.)37 b(It)21 b(is)g(often)g(con)o(v)o (enien)o(t)g(to)g(de\014ne)h(a)e(datat)o(yp)q(e)h(that)f(has)h(a)g (\\hole")g(at)f(its)h(b)q(e-)75 1967 y(ginning)e(and/or)e(its)h(end;)h (or)f(a)f(datat)o(yp)q(e)g(with)h(en)o(tries)g(that)f(extend)i(b)q(ey)o (ond)f(the)g(datat)o(yp)q(e)f(\\ex-)75 2023 y(ten)o(t".)22 b(This)17 b(allo)o(ws)g(to)f(use)g(the)h(functions)33 b Fo(MPI)p 969 2023 V 16 w(TYPE)p 1102 2023 V 17 w(CONTIGUOUS)p FE(,)h Fo(MPI)p 1536 2023 V 15 w(TYPE)p 1668 2023 V 17 w(VECTOR)p FE(,)75 2079 y(or)24 b Fo(MPI)p 225 2079 V 16 w(TYPE)p 358 2079 V 16 w(INDEX)13 b FE(to)e(replicate)j(a)d(more)h (general)h(pattern;)f(examples)h(of)f(suc)o(h)g(a)g(usage)g(are)g(pro-) 75 2136 y(vided)g(in)g(Section)g(4.13.4.)k(T)l(o)11 b(ac)o(hiev)o(e)h (this,)f(w)o(e)g(add)g(t)o(w)o(o)f(additional)i(\\pseudo-datat)o(yp)q (es")22 b Fo(MPI)p 1795 2136 V 15 w(LB)p FE(,)75 2192 y(and)j Fo(MPI)p 258 2192 V 16 w(UB)13 b FE(that)e(can)i(b)q(e)g(used,) g(resp)q(ectiv)o(ely)l(,)i(to)d(mark)f(the)i(lo)o(w)o(er)f(b)q(ound)h (or)f(the)h(upp)q(er)g(b)q(ound)g(of)75 2249 y(a)i(datat)o(yp)q(e.)k (these)c(pseudo-datat)o(yp)q(es)g(o)q(ccup)o(y)h(no)f(space)g(\()p FD(extent)p FE(\()p Fo(MP)p 1369 2249 V 17 w(LB)p FE(\))e(=)g FD(extent)p FE(\()p Fo(MP)p 1734 2249 V 17 w(UB)p FE(\))g(=)75 2305 y(0\).)30 b(They)19 b(do)f(not)g(a\013ect)g(the)h(signature)g(of)f (a)h(message)f(created)g(with)h(this)h(datat)o(yp)q(e.)29 b(Ho)o(w)o(ev)o(er,)75 2362 y(they)16 b(do)g(a\013ect)f(the)h (de\014nition)i(of)d(the)h(exten)o(t)g(of)f(a)h(datat)o(yp)q(e)f(and,)h (therefore,)f(a\013ect)g(the)h(outcome)75 2418 y(of)f(a)g(replication)i (of)d(this)i(datat)o(yp)q(e)f(b)o(y)g(a)g(datat)o(yp)q(e)f (constructor.)166 2478 y(Example:)30 b(Let)21 b Fp(I)j(=)f(\(-3,)h(0,)f (6\))p FE(;)f Fp(T)i(=)g(\(MPI)p 1033 2478 15 2 v 16 w(LB,)g(MPI)p 1217 2478 V 17 w(INT,)f(MPI)p 1425 2478 V 17 w(UB\))p FE(,)c(and)h Fp(B)k(=)g(\(1,)f(1,)75 2534 y(1\))p FE(.)c(Then)c(a)f(call)h(to)e Fo(MPI)p 531 2534 14 2 v 16 w(TYPE)p 664 2534 V 17 w(STRUC\(3,)h(T,)g(I,)f(B,)h(t)o(yp)q (e1\))g FE(creates)g(a)g(new)g(datat)o(yp)q(e)g(that)f(has)h(an)75 2591 y(exten)o(t)h(of)g(9)g(\(from)f(-3)i(to)e(6\),)h(and)g(con)o (tains)h(an)f(in)o(teger)h(at)e(displacemen)o(t)j(0.)j(This)c(is)g(the) f(datat)o(yp)q(e)75 2647 y(de\014ned)f(b)o(y)e(the)g(sequence)i FA(f)p Fp(\(lb,)23 b(-3\),)g(\(int,)g(0\),)h(\(ub,)f(6\))p FA(g)g FE(.)c(If)13 b(this)g(t)o(yp)q(e)f(is)h(replicated)h(t)o(wice)75 2704 y(b)o(y)h(a)g(call)h(to)f Fo(MPI)p 400 2704 V 15 w(TYPE)p 532 2704 V 17 w(CONTIGUOUS\(2,)h(t)o(yp)q(e1,)g(t)o(yp)q(e2\)) f FE(then)h(the)f(newly)h(created)f(t)o(yp)q(e)g(can)g(b)q(e)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 78 84 78 83 bop 75 -100 a FE(78)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(describ)q(ed)d(b) o(y)d(the)h(sequence)h FA(f)p Fp(\(lb,)23 b(-3\),)g(\(int,)g(0\),)h (\(int,9\),)e(\(ub,)i(15\))p FA(g)f FE(.)18 b(\(En)o(tries)11 b(of)f(t)o(yp)q(e)75 102 y Fp(lb)15 b FE(or)g Fp(ub)f FE(can)i(b)q(e)g(deleted)g(if)g(they)f(are)g(not)g(at)f(the)i(endp)q (oin)o(ts)g(of)f(the)g(datat)o(yp)q(e.\))166 158 y(In)h(general,)f(if) 471 253 y FD(D)q(ataty)r(pe)e FE(=)g FA(f)p FE(\()p FD(ty)r(pe)844 260 y Fw(0)864 253 y FD(;)8 b(disp)969 260 y Fw(0)987 253 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1188 260 y Fi(n)p Fj(\000)p Fw(1)1254 253 y FD(;)g(disp)1359 260 y Fi(n)p Fj(\000)p Fw(1)1426 253 y FE(\))p FA(g)p FD(;)75 348 y FE(then)16 b(the)f FF(lo)o(w)o(er)h(b)q(ound)g FE(of)f FD(D)q(ataty)r(pe)h FE(is)f(de\014ned)i(to)e(b)q(e)131 474 y FD(l)q(b)p FE(\()p FD(D)q(ataty)r(pe)p FE(\))d(=)449 402 y Fe(\()503 445 y FE(min)579 452 y Fi(j)604 445 y FD(disp)688 452 y Fi(j)1062 445 y FE(if)j(no)g(en)o(try)g(has)g(basic)h (t)o(yp)q(e)f FD(ty)r(pe)1666 452 y Fi(j)1698 445 y FE(=)e Fp(lb)503 502 y FE(min)q FA(f)p FD(disp)686 509 y Fi(j)731 502 y FE(:)27 b FD(ty)r(pe)855 509 y Fi(j)886 502 y FE(=)13 b Fp(lb)p FA(g)41 b FE(otherwise)166 599 y(Similarly)l(,)17 b(the)e FF(upp)q(er)j(b)q(ound)e FE(of)e FD(D)q(ataty)r(pe)i FE(is)g(de\014ned)h(to)d(b)q(e)108 724 y FD(ub)p FE(\()p FD(D)q(ataty)r(pe)p FE(\))e(=)437 653 y Fe(\()492 696 y FE(max)576 703 y Fi(j)602 696 y FD(disp)686 703 y Fi(j)714 696 y FE(+)e FD(siz)r(eof)5 b FE(\()p FD(ty)r(pe)991 703 y Fi(j)1010 696 y FE(\))56 b(if)16 b(no)f(en)o(try)g(has)g(basic)h (t)o(yp)q(e)f FD(ty)r(pe)1689 703 y Fi(j)1720 696 y FE(=)e Fp(ub)492 753 y FE(max)o FA(f)p FD(disp)683 760 y Fi(j)728 753 y FE(:)28 b FD(ty)r(pe)853 760 y Fi(j)884 753 y FE(=)13 b Fp(ub)p FA(g)66 b FE(otherwise)166 850 y(Then)472 906 y FD(extent)p FE(\()p FD(D)q(ataty)r(pe)p FE(\))14 b(=)f FD(ub)p FE(\()p FD(D)q(ataty)r(pe)p FE(\))d FA(\000)g FD(l)q(b)p FE(\()p FD(D)q(ataty)r(pe)p FE(\);)166 986 y(F)l(urthermore,)18 b(if)h FD(ty)r(pe)575 993 y Fi(i)608 986 y FE(requires)h(alignmen)o(t)f(to)f(an)g(address)h(that)f(is)h(a)f (m)o(ultiple)j(of)d FD(k)1741 993 y Fi(i)1755 986 y FE(,)h(then)75 1043 y FD(extent)p FE(\()p FD(D)q(ataty)r(pe)p FE(\))14 b(is)g(further)f(rounded)g(up)h(to)e(the)h(nearest)g(m)o(ultiple)i(of)d (max)1466 1050 y Fi(i)1488 1043 y FD(k)1512 1050 y Fi(i)1525 1043 y FE(.)19 b(The)14 b(formal)e(de\014-)75 1099 y(nitions)j(giv)o (en)g(for)e(the)h(v)m(arious)h(datat)o(yp)q(e)e(constructors)h(apply)g (no)o(w,)g(with)g(the)g(amended)h(de\014nition)75 1156 y(of)g FF(exten)o(t)p FE(.)166 1212 y(A)j(datat)o(yp)q(e)f(ob)s(ject)g (has)h(to)f(b)q(e)i FF(committed)f FE(b)q(efore)g(it)g(can)g(b)q(e)h (used)f(in)h(a)f(comm)o(unication.)75 1269 y(A)d(committed)h(datat)o (yp)q(e)e(ob)s(ject)h(cannot)g(b)q(e)h(used)g(to)e(build)k(new)d(deriv) o(ed)h(datat)o(yp)q(es.)k(The)15 b(system)75 1325 y(ma)o(y)e(use)g(a)g (di\013eren)o(t)h(in)o(ternal)g(represen)o(tation)f(for)g(committed)g (datat)o(yp)q(e)g(ob)s(ject)g(so)f(as)h(to)g(facilitate)75 1381 y(comm)o(unication,)h(e.g.)k(c)o(hange)c(from)e(a)g(compacted)i (represen)o(tation)f(to)f(a)h(\015at)f(represen)o(tation)h(of)g(the)75 1438 y(datat)o(yp)q(e,)h(and)i(select)g(the)f(most)f(con)o(v)o(enien)o (t)i(transfer)e(mec)o(hanism.)75 1542 y Fo(MPI)p 160 1542 14 2 v 16 w(TYPE)p 293 1542 V 17 w(COMMIT\(int)o(yp)q(e,)h(outt)o (yp)q(e\))117 1619 y Fu(IN)155 b Fo(int)o(yp)q(e)473 b Fu(handle)14 b(to)g(datat)o(yp)q(e)g(ob)r(ject)h(to)e(b)q(e)i (committed)117 1693 y(OUT)108 b Fo(outt)o(yp)q(e)445 b Fu(handle)14 b(to)g(committed)d(datat)o(yp)q(e)j(ob)r(ject)166 1817 y FE(There)i(is)g(no)f(need)h(to)f(commit)h(basic)g(datat)o(yp)q (es;)f(they)g(are)g(\\pre-committed".)21 b(They)16 b(are)f(the)75 1874 y(exception)h(to)e(the)h(rule)g(that)g(committed)f(datat)o(yp)q (es)h(cannot)f(b)q(e)i(used)f(to)f(form)g(deriv)o(ed)i(datat)o(yp)q (es.)166 2006 y Fv(Discussion:)166 2056 y Fu(1.)i(Do)13 b(w)o(e)h(really)f(need)i(a)f(commit)d(here?)166 2106 y(2.)18 b(Do)13 b(w)o(e)h(w)o(an)o(t)g(one)g(INOUT)m(argumen)o(t,)e(or) i(t)o(w)o(o)f(distinct)h(argumen)o(ts?)166 2156 y(3.)19 b(Are)c(committed)d(and)i(uncommitted)e(ob)r(jects)k(of)e(the)h(same)e (t)o(yp)q(e?)20 b(\(The)15 b(answ)o(er)g(m)o(ust)e(b)q(e)i(y)o(es,)g (if)e(it)75 2205 y(is)h(an)f(INOUT)m(argumen)o(t.\))166 2255 y(4.)k(If)c(w)o(e)h(ha)o(v)o(e)f(only)f(one)h(INOUT)m(argumen)o (t,)f(do)q(es)i(a)f(commit)d(c)o(hange)k(the)f(datat)o(yp)q(e)h(ob)r (ject,)f(or)g(do)q(es)h(it)75 2305 y(create)i(a)f(new)g(ob)r(ject)h (and)f(return)h(a)f(p)q(oin)o(ter)g(to)g(it)f(in)29 b Fr(datat)o(yp)q(e)p Fu(?)23 b(I.e.,)14 b(if)g(w)o(e)h(ha)o(v)o(e)g (another)g(p)q(oin)o(ter)h(to)e(the)75 2355 y(same)f(uncommitted)f(ob)r (ject,)i(can)g(w)o(e)g(con)o(tin)o(ue)g(to)g(use)h(it?)166 2405 y(W)m(e)g(prohibit)h(the)g(use)h(of)f(committed)d(datat)o(yp)q(e)j (in)g(the)h(construction)g(of)e(new)h(datat)o(yp)q(es)h(in)e(order)i (to)75 2455 y(allo)o(w)10 b(the)j(commit)c(function)j(to)f(c)o(hange)i (a)e(datat)o(yp)q(e)h(represen)o(tation)i(in)o(to)d(one)h(suitable)g (for)g(comm)o(unicatio)o(n,)75 2504 y(and)17 b(p)q(erhaps)i(unsuitable) e(for)g(recursiv)o(e)i(datat)o(yp)q(e)e(construction.)30 b(This)17 b(restriction)h(could)g(b)q(e)f(relaxed)h(b)o(y)75 2554 y(assuming)13 b(that)i(the)g(\\uncommitted")d(datat)o(yp)q(e)j (represen)o(tation)h(is)f(k)o(ept)g(alongside)f(the)h(\\committed")d (one.)75 2604 y(With)k(suc)o(h)h(an)f(implem)o(en)o(tation)30 b Fr(MPI)p 733 2604 13 2 v 14 w(TYPE)p 855 2604 V 14 w(COMMIT)16 b Fu(could)g(ha)o(v)o(e)g(only)f(one)i(argumen)o(t,)e(and)h (only)g(one)75 2654 y(t)o(yp)q(e)f(of)e(datat)o(yp)q(e)i(ob)r(ject)g (is)f(needed.)20 b(Suc)o(h)15 b(represen)o(tation)g(could)f(also)g (simplify)d(the)k(implemen)o(tatio)o(n)d(of)h(a)75 2704 y(datat)o(yp)q(e)h(\\\015atten")g(function)g(\(e.g.,)e(for)i (debugging\).)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 79 85 79 84 bop 75 -100 a Fx(4.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(79)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(FREE\(datat)o(yp)q(e\))117 124 y Fu(INOUT)62 b Fo(datat)o(yp)q(e)424 b Fu(handle)14 b(to)g(datat)o(yp)q(e)g(ob)r (ject)166 251 y FE(Marks)h(the)g(datat)o(yp)q(e)g(ob)s(ject)g(asso)q (ciated)h(with)32 b Fo(datat)o(yp)q(e)17 b FE(for)e(deallo)q(cation.)22 b(The)16 b(ob)s(ject)f(will)75 307 y(b)q(e)h(deallo)q(cated)h(after)d (an)o(y)i(p)q(ending)h(comm)o(unication)f(that)e(uses)i(this)g(ob)s (ject)f(completes,)g(at)g(whic)o(h)75 364 y(p)q(oin)o(t)31 b Fo(datat)o(yp)q(e)17 b FE(b)q(ecomes)e(n)o(ull.)166 498 y Fv(Discussion:)166 550 y Fu(If)f(w)o(e)h(ha)o(v)o(e)f(di\013eren) o(t)i(datat)o(yp)q(es)f(for)f(committed)e(and)i(uncommitted,)e(then)j (w)o(e)g(ha)o(v)o(e)f(2)h(free)g(functions)75 600 y(\(or)f(nev)o(er)h (free)f(uncommitted)e(datat)o(yp)q(e\))75 816 y Fq(4.13.3)49 b(Use)16 b(of)h(general)f(datat)o(yp)q(es)g(in)h(communication)75 906 y FE(Handles)g(to)f(deriv)o(ed)h(datat)o(yp)q(es)f(can)h(b)q(e)f (passed)h(to)f(a)g(comm)o(unication)h(call)g(wherev)o(er)f(a)g(datat)o (yp)q(e)75 963 y(argumen)o(t)22 b(is)h(required.)43 b(A)23 b(call)h(of)e(the)h(form)f Fo(MPI)p 1050 963 V 15 w(SEND\(sta)o(rt,)j (count,)g(datat)o(yp)q(e,)h(...\))p FE(,)d(where)75 1019 y Fp(count)12 b FE(=)h Fp(1)f FE(uses)h(the)g(send)g(bu\013er)g (de\014ned)h(b)o(y)25 b Fo(sta)o(rt)13 b FE(and)26 b Fo(datat)o(yp)q(e)p FE(;)15 b(i.e.)k(if)26 b Fo(datat)o(yp)q(e)14 b FE(is)f(a)g(handle)h(to)75 1076 y(the)k(datat)o(yp)q(e)g(de\014ned)i (b)o(y)e(the)g(sequence)h FA(f)p FE(\()p FD(ty)r(pe)973 1083 y Fw(0)993 1076 y FD(;)8 b(disp)1098 1083 y Fw(0)1116 1076 y FE(\))p FD(;)g(:::;)g FE(\()p FD(ty)r(pe)1317 1083 y Fi(n)p Fj(\000)p Fw(1)1383 1076 y FD(;)g(disp)1488 1083 y Fi(n)p Fj(\000)p Fw(1)1555 1076 y FE(\))p FA(g)p FE(,)18 b(\(with)g(empt)o(y)75 1132 y(en)o(tries)j(deleted\))h(then)g (the)f(send)g(bu\013er)g(consists)g(of)g(the)g(en)o(tries)g(of)g(t)o (yp)q(es)g FD(ty)r(pe)1579 1139 y Fi(i)1593 1132 y FE(,)h(at)f (addresses)75 1188 y Fp(start)10 b FE(+)i FD(disp)336 1195 y Fi(i)349 1188 y FE(,)17 b FD(i)d FE(=)h(0)p FD(;)8 b(:::;)g(n)h FA(\000)i FE(1.)24 b(The)17 b(message)f(generated)g (consists)h(of)f(a)g(sequence)i(of)e FD(n)h FE(en)o(tries,)75 1245 y(of)g(t)o(yp)q(es)h FD(ty)r(pe)336 1252 y Fw(0)356 1245 y FD(;)8 b(:::;)g(ty)r(pe)521 1252 y Fi(n)p Fj(\000)p Fw(1)587 1245 y FE(.)28 b(The)18 b(receiv)o(e)h(bu\013er)e(for)g(a)h (call)h Fo(MPI)p 1292 1245 V 16 w(RECV\()f(sta)o(rt,)g(count,)h(datat)o (yp)q(e)75 1301 y(,)h(...\))p FE(,)e(is)i(similarly)h(de\014ned,)g(for) e Fp(count)k(=1)p FE(.)32 b(The)19 b(same)g(applies)i(to)e(all)h(other) f(send)h(and)f(receiv)o(e)75 1358 y(op)q(erations)c(that)g(ha)o(v)o(e)g (a)30 b Fo(datat)o(yp)q(e)16 b FE(parameter.)166 1416 y(A)d(datat)o(yp)q(e)f(used)i(in)f(a)g(receiv)o(e)h(op)q(eration)f(ma)o (y)f(not)g(ha)o(v)o(e)h(o)o(v)o(erlapping)g(en)o(tries;)h(otherwise)f (the)75 1473 y(outcome)i(of)g(the)g(receiv)o(e)h(op)q(eration)f(is)h (unde\014ned.)166 1531 y(If)g(a)g(send)h(op)q(eration)f(using)h(a)f (general)g(datat)o(yp)q(e)32 b Fo(t)o(yp)q(e1)16 b FE(is)h(matc)o(hed)f (b)o(y)g(a)g(receiv)o(e)h(op)q(eration)75 1588 y(using)e(a)f(general)h (datat)o(yp)q(e)28 b Fo(t)o(yp)q(e2)16 b FE(then)e(the)h(sequence)g(of) f(\(nonempt)o(y\))g(basic)h(t)o(yp)q(e)g(en)o(tries)g(de\014ned)75 1644 y(b)o(y)f(eac)o(h)g(datat)o(yp)q(e)f(should)i(b)q(e)g(iden)o (tical.)21 b(F)l(urthermore,)13 b(eac)o(h)h(en)o(try)g(in)h(the)f(send) g(bu\013er)g(should)h(b)q(e)75 1701 y(of)i(t)o(yp)q(e)g(that)g(matc)o (hes)g(the)h(t)o(yp)q(e)f(of)g(the)g(corresp)q(onding)i(en)o(try)e(in) 35 b Fo(t)o(yp)q(e1)19 b FE(and)e(eac)o(h)h(en)o(try)f(in)h(the)75 1757 y(receiv)o(e)f(bu\013er)f(should)i(of)d(a)h(t)o(yp)q(e)h(that)e (matc)o(hes)h(the)g(t)o(yp)q(e)h(of)f(the)g(corresp)q(onding)h(en)o (try)f(in)33 b Fo(t)o(yp)q(e2)75 1814 y FE(\(with)13 b(t)o(yp)q(e)f(matc)o(hing)h(de\014ned)h(as)e(in)h(Section)g(4.5\).)18 b(Th)o(us,)13 b(t)o(yp)q(e)f(matc)o(hing)h(is)g(de\014ned)h(b)o(y)e (the)h(\\\015at")75 1870 y(structure)i(of)g(a)g(datat)o(yp)q(e,)f(and)h (do)q(es)h(not)f(dep)q(end)h(on)g(its)f(de\014ning)i(expression.)166 1929 y(Example:)75 2033 y Fp(...)75 2089 y(MPI_TYPE_CONTIGUOUS\()k (MPI_REAL,)i(2,)g(type2\))75 2146 y(MPI_TYPE_CONTIGUOUS\()e(MPI_REAL,)i (4,)g(type4\))75 2202 y(MPI_TYPE_CONTIGUOUS\()e(type2,)i(2,)h(type22\)) 166 2304 y FE(A)15 b(message)g(sen)o(t)g(with)g(datat)o(yp)q(e)30 b Fo(t)o(yp)q(e4)16 b FE(can)f(b)q(e)h(receiv)o(ed)h(with)e(datat)o(yp) q(e)30 b Fo(t)o(yp)q(e22)p FE(.)166 2363 y(T)o(yp)q(e)22 b(matc)o(hing)g(requiremen)o(ts)h(imply)h(that)d(\(nonempt)o(y\))g(en)o (tries)i(of)f(di\013eren)o(t)g(t)o(yp)q(es)g(in)h(a)75 2419 y(datat)o(yp)q(e)13 b(cannot)g(o)o(v)o(erlap,)g(unless)i(they)e (coincide)i(and)f(ha)o(v)o(e)f(the)g(same)g(basic)h(t)o(yp)q(e,)g(or)f (one)g(of)g(them)75 2476 y(ha)o(v)o(e)i(basic)h(t)o(yp)q(e)30 b Fo(b)o(yte)p FE(.)166 2534 y(The)12 b(length)g(of)f(a)h(message)f (receiv)o(ed)i(\(i.e.,)f(the)f(v)m(alue)i(returned)f(when)g(the)g (receiv)o(e)h(status)e(ob)s(ject)75 2591 y(is)k(deco)q(ded)h(b)o(y)f (the)f(function)i Fo(MPI)p 698 2591 V 16 w(GET)p 802 2591 V 16 w(LEN)p FE(\))f(is)g(equal)g(to)f(the)h(n)o(um)o(b)q(er)g(of) f(basic)h(en)o(tries)g(receiv)o(ed.)75 2647 y(Consider)h(again)f(the)g (datat)o(yp)q(es)g(de\014ned)i(in)f(the)f(last)g(example.)21 b(If)16 b(a)f(message)f(is)i(sen)o(t)f(using)h(either)75 2704 y(datat)o(yp)q(e)38 b Fo(t)o(yp)q(e22)19 b FE(or)38 b Fo(t)o(yp)q(e4)20 b FE(and)f(is)h(receiv)o(ed)g(using)g(either)g(of)f (these)g(t)o(w)o(o)f(datat)o(yp)q(es,)h(then)h(the)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 80 86 80 85 bop 75 -100 a FE(80)635 b Fx(SECTION)16 b(4.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FE(length)h(of)f (the)h(message)f(receiv)o(ed)i(will)g(b)q(e)f(four.)24 b(In)17 b(the)g(general)g(case)f(the)h(basic)g(en)o(tries)g(need)g(not) 75 102 y(b)q(e)f(all)g(of)f(the)g(same)g(t)o(yp)q(e.)166 236 y Fv(Discussion:)166 287 y Fu(An)f(alternativ)o(e)f(de\014nition)h (that)f(\014nds)i(fa)o(v)o(or)d(with)i(man)o(y)e(mem)o(b)q(ers)g(of)h (the)i(sub)q(committee)d(is)i(that)f(the)75 337 y(length)h(of)g(the)h (message)f(receiv)o(ed)h(is)f(measured)h(in)e(terms)i(of)e(subunits)i (of)f(the)h(receiving)f(datat)o(yp)q(e.)20 b(Th)o(us,)14 b(if)75 387 y(four)h(reals)g(are)h(receiv)o(ed)g(using)30 b Fr(t)o(yp)q(e4)p Fu(,)16 b(the)g(length)f(of)f(the)i(receiv)o(ed)g (message)f(is)g(four;)g(if)f(they)i(are)g(receiv)o(ed)75 437 y(using)29 b Fr(t)o(yp)q(e22)p Fu(,)14 b(the)h(length)f(of)g(the)h (receiv)o(ed)h(message)e(is)g(t)o(w)o(o.)20 b(With)13 b(suc)o(h)i(a)g(de\014nition,)e(it)h(is)h(required)g(that)75 486 y(the)f(arriving)f(message)h(\014lls)f(an)h(in)o(teger)g(n)o(um)o (b)q(er)f(of)h(subunits.)166 538 y(The)g(rationale)e(is)h(that)g(the)h (these)h(are)e(the)h(units)g(that)f(are)g(meaningful)e(to)i(the)h (programmer.)i(Consider)75 588 y(the)e(case)h(of)e(an)g(arra)o(y)g(of)g (structures)j(that)e(is)f(passed)i(in)e(a)g(message)h(\(example)e(4.1)g (b)q(elo)o(w\).)18 b(Then)c(one)g(migh)o(t)75 638 y(w)o(an)o(t)f(to)g (enforce)i(the)f(restriction)g(that)f(only)g(complete)g(structures)i (are)f(passed;)g(and)g(is)f(natural)g(to)g(return)h(in)75 687 y(the)h(receiv)o(e)g(status)g(the)f(n)o(um)o(b)q(er)f(of)h (structures)i(receiv)o(ed,)f(rather)g(than)f(this)g(n)o(um)o(b)q(er)f (m)o(ultiplied)f(b)o(y)h(14,)g(the)75 737 y(n)o(um)o(b)q(er)g(of)g(en)o (tries)i(p)q(er)g(structure.)166 789 y(Consider,)c(on)g(the)g(other)g (hand,)g(the)g(case)h(of)e(a)g(sequence)j(of)d(reals)h(that)g(is)f (receiv)o(ed)i(in)e(a)h(3D)f(arra)o(y)g(section)75 839 y(\(de\014ned)16 b(as)e(in)h(the)g(\014rst)g(example)e(b)q(elo)o(w\).) 20 b(W)m(e)15 b(ma)o(y)d(w)o(an)o(t)j(to)f(receiv)o(e)i(an)e(arbitrary) h(n)o(um)o(b)q(er)f(of)g(reals,)g(and)75 888 y(the)g(1D)g(and)g(2D)f (subsections)i(ma)o(y)d(not)i(carry)g(an)o(y)g(signi\014cance)g(for)g (the)g(purp)q(ose)h(of)f(this)f(comm)o(unication.)166 940 y(Opinions?)166 1081 y FE(A)18 b(call)h(of)f(the)g(form)f Fo(MPI)p 635 1081 14 2 v 16 w(SEND\(sta)o(rt,)i(count,)h(datat)o(yp)q (e)f(,)g(...\))p FE(,)e(where)h Fp(count)f FD(>)h Fp(1)p FE(,)g(is)h(in)o(ter-)75 1137 y(preted)c(as)g(if)g(the)g(call)h(w)o(as) e(passed)h(a)f(new)h(datat)o(yp)q(e)g(whic)o(h)g(is)h(the)f (concatenation)g(of)29 b Fo(count)16 b FE(copies)75 1194 y(of)35 b Fo(datat)o(yp)q(e)p FE(,)19 b(suc)o(h)f(as)g(w)o(ould)g(b)q (e)g(pro)q(duced)h(b)o(y)e(a)h(call)g(to)f Fo(MPI)p 1240 1194 V 16 w(CONTIGUOUS\(count,)k(datat)o(yp)q(e,)75 1250 y(newt)o(yp)q(e\))p FE(,)16 b(and)e(a)29 b Fo(count)15 b FE(argumen)o(t)f(of)g(one.)20 b(The)14 b(same)g(holds)i(true)e(for)g (the)g(other)g(comm)o(unication)75 1307 y(functions)i(that)e(ha)o(v)o (e)h(a)g(datat)o(yp)q(e)g(argumen)o(t.)75 1438 y Fq(4.13.4)49 b(Examples)75 1527 y FE(The)15 b(follo)o(wing)h(examples)g(illustrate)h (the)e(use)g(of)g(deriv)o(ed)h(datat)o(yp)q(es.)166 1585 y(First)f(example:)20 b(extract)15 b(a)g(section)h(of)e(a)h(3D)g (matrix.)218 1687 y Fp(REAL)23 b(a\(100,100,100\),)f(e\(9,9,9\))218 1744 y(INTEGER)h(oneslice,)g(twoslice,)f(threeslice,)h(finaltype,)f (sizeofreal)75 1857 y(C)143 b(extract)23 b(the)g(section)g(a\(1:17:2,)g (3:11,)g(2:10\))75 1913 y(C)143 b(and)23 b(store)h(it)f(in)h (e\(*,*,*\).)218 2026 y(CALL)f(MPI_TYPE_EXTENT\()f(MPI_REAL,)h (sizeofreal,)f(ierr\))75 2139 y(C)119 b(create)23 b(datatype)g(for)g(a) h(1D)g(section)218 2195 y(CALL)f(MPI_TYPE_VECTOR\()f(9,)i(MPI_REAL,)46 b(2,)24 b(1,)f(oneslice,)g(ierr\))75 2308 y(C)119 b(create)23 b(datatype)g(for)g(a)h(2D)g(section)218 2365 y(CALL)f (MPI_TYPE_HVECTOR\()f(9,)i(oneslice,)46 b(100*sizeofreal,)22 b(1,)i(twoslice,)e(ierr\))75 2478 y(C)119 b(create)23 b(datatype)g(for)g(the)h(entire)f(section)218 2534 y(CALL)g (MPI_TYPE_HVECTOR\()f(9,)i(twoslice,)46 b(100*100*sizeofreal,)21 b(1,)505 2591 y(threeslice,)h(ierr\))218 2704 y(CALL)h (MPI_TYPE_COMMIT\()f(threeslice,)h(finaltype,)f(ierr\))1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 81 87 81 86 bop 75 -100 a Fx(4.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FE(81)218 45 y Fp(CALL)23 b(MPI_SENDRECV\(a\(1,3,2\),)e(1,)j (finaltype,)e(MPI_ME,)h(e,)h(9*9*9,)505 102 y(MPI_REAL,)e(MPI_ME,)h(0,) h(MPI_ALL,)e(status,)h(ierr\))166 196 y FE(Second)16 b(example:)21 b(Cop)o(y)14 b(the)i(\(strictly\))f(lo)o(w)o(er)f (triangular)i(part)e(of)h(a)g(matrix.)218 291 y Fp(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 348 y(INTEGER)47 b(index\(100\),)22 b(blocklen\(100\),)g(ltype,)h(finaltype)75 461 y(C)119 b(copy)23 b(lower)h(triangular)e(part)h(of)h(array)f(a)75 517 y(C)119 b(onto)23 b(lower)h(triangular)e(part)h(of)h(array)f(b)75 630 y(C)119 b(compute)23 b(start)g(and)h(size)f(of)h(each)f(column)218 687 y(DO)h(i=1,)f(100)266 743 y(index\(i\))g(=)g(100*\(i-1\))g(+)h(i) 266 800 y(block\(i\))f(=)g(100-i)218 856 y(END)h(DO)75 969 y(C)119 b(create)23 b(datatype)g(for)g(lower)h(triangular)e(part) 218 1025 y(CALL)h(MPI_TYPE_INDEX\()f(MPI_REAL,)h(100,)g(index,)g (block,)g(ltype,)g(ierr\))218 1138 y(CALL)g(MPI_TYPE_COMMIT\(ltype,)e (finaltype,)i(ierr\))218 1195 y(CALL)g(MPI_SENDRECV\()g(a,)g(1,)h (finaltype,)e(MPI_ME,)h(b,)h(1,)457 1251 y(finaltype,)e(MPI_ME,)h(0,)h (MPI_ALL,)f(status,)f(ierr\))166 1346 y FE(Third)16 b(example:)21 b(T)l(ransp)q(ose)15 b(a)g(matrix)218 1441 y Fp(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 1498 y(INTEGER)g(row,)g(xpose,)g (finaltype,)g(sizeofreal)75 1610 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 1780 y(CALL)g(MPI_TYPE_EXTENT\()f (MPI_REAL,)h(sizeofreal,)f(ierr\))75 1893 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 1949 y(CALL)f(MPI_TYPE_VECTOR\()f (100,)h(MPI_REAL,)g(100,)g(1,)h(row,)f(ierr\))75 2062 y(C)119 b(create)23 b(datatype)g(for)g(matrix)g(in)h(row-major)f(order) 218 2119 y(CALL)g(MPI_TYPE_HVECTOR\()f(100,)h(row,)h(sizeofreal,)e(1,)i (xpose,)f(ierr\))218 2231 y(CALL)g(MPI_TYPE_COMMIT\()f(xpose,)h (finaltype,)g(ierr\))75 2344 y(C)119 b(send)23 b(matrix)h(in)f (row-major)g(order)g(and)g(receive)g(in)h(column)f(major)g(order)218 2401 y(CALL)g(MPI_SENDRECV\()g(a,)g(1,)h(finaltype,)e(MPI_ME,)h(b,)h (100*100,)457 2457 y(MPI_REAL,)e(MPI_ME,)h(0,)h(MPI_ALL,)f(status,)g (ierr\))166 2552 y FE(Another)15 b(approac)o(h)g(to)f(the)i(transp)q (ose)f(problem:)218 2647 y Fp(REAL)23 b(a\(100,100\),)g(b\(100,100\)) 218 2704 y(INTEGER)47 b(index\(2\),)23 b(blen\(2\),)f(type\(2\),)h (row,)g(row1,)h(finaltype,)e(sizeofreal)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 82 88 82 87 bop 75 -100 a FE(82)635 b Fx(SECTION)16 b(4.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 158 y Fp(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 327 y(CALL)g (MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 440 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 497 y(CALL)f(MPI_TYPE_VECTOR\()f(MPI_REAL,)h(100,)g(100,)g(1,)h(row,)f (ierr\))75 610 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row,)f (with)h(the)f(extent)g(of)h(one)f(real)g(number)218 666 y(index\(1\))g(=)h(0)218 723 y(index\(2\))f(=)h(sizeofreal)218 779 y(type\(1\))47 b(=)24 b(row)218 835 y(type\(2\))47 b(=)24 b(MPI_UB)218 892 y(blen\(1\))47 b(=)24 b(1)218 948 y(blen\(2\))47 b(=)24 b(1)218 1005 y(CALL)f(MPI_TYPE_STRUC\()f(2,)i (type,)f(index,)g(blen,)g(row1\))218 1118 y(CALL)g(MPI_TYPE_COMMIT\()f (row1,)h(finaltype,)g(ierr\))75 1231 y(C)119 b(send)23 b(100)h(rows)f(and)h(receive)f(in)g(column)g(major)g(order)218 1287 y(CALL)g(MPI_SENDRECV\()g(a,)g(100,)g(finaltype,)g(MPI_ME,)g(b,)g (100*100,)457 1344 y(MPI_REAL,)f(MPI_ME,)h(0,)h(MPI_ALL,)f(status,)g (ierr\))166 1431 y FE(F)l(ourth)15 b(example:)20 b(manipulate)d(an)e (arra)o(y)f(of)h(structures.)75 1518 y Fp(struct)23 b(Partstruct)147 1574 y({)147 1631 y(int)g(index;)95 b(/*)23 b(particle)g(type)g(*/)147 1687 y(double)g(d[6])47 b(/*)23 b(particle)g(coordinates)g(*/)147 1744 y(char)g(b[7])95 b(/*)23 b(some)h(additional)e(information)h(*/) 147 1800 y(};)75 1913 y(struct)g(Partstruct)g(particle[1000];)337 2026 y(/*)h(build)f(datatype)g(describing)g(first)g(array)g(entry)g(*/) 75 2139 y(MPI_datatype)f(Particletype;)75 2195 y(MPI_datatype)g (stype[3])h(=)h({MPI_int,)e(MPI_double,)h(MPI_char};)75 2252 y(int)238 b(sblock[3])23 b(=)h({1,)f(6,)h(7};)75 2308 y(int)238 b(sindex[3];)75 2421 y(MPI_address\()22 b(\(void)h(*\)particle,)g(sindex\);)75 2478 y(MPI_address\()f(\(void)h (*\)particle[0].d,)f(sindex+1\);)75 2534 y(MPI_address\()g(\(void)h (*\)particle[0].b,)f(sindex+2\);)75 2591 y(MPI_type_struct\()g(3,)h (stype,)g(sindex,)g(sblock,)g(Particletype\);)75 2647 y(/*)h(Particletype)e(describes)g(first)i(array)f(entry)g(--)h(using)f (absolute)147 2704 y(addresses)f(*/)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 83 89 83 88 bop 75 -100 a Fx(4.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FE(83)266 158 y Fp(/*)23 b(4.1:)h(send)f(the)h(entire)f(array)g (*/)75 271 y(MPI_datatype)f(Type1;)75 327 y(MPI_commit\()g (Particletype,)g(Type1\);)75 384 y(MPI_send\()h(MPI_bottom,)f(1000,)h (Type1,)g(dest,)g(tag,)h(comm\);)266 553 y(/*)f(4.2:)h(send)f(the)h (entries)e(with)i(index)f(value)g(zero,)314 610 y(preceded)f(by)i(the)f (number)g(of)h(such)f(entries)g(*/)75 723 y(MPI_datatype)f(Zparticles;) 70 b(/*)24 b(datatype)f(describing)f(all)i(particles)934 779 y(with)f(index)h(zero)f(\(needs)g(to)h(be)f(recomputed)934 835 y(if)h(indices)f(change\))f(*/)75 892 y(MPI_datatype)g(Ztype,)h (Type2;)75 1005 y(int)g(zindex[1000],)g(zblock[1000],)f(i,)h(j,)h(k;)75 1061 y(int)f(zzblock[2])g(=)h({1,1};)75 1118 y(int)f(zztype[2],)g (zzindex[2];)75 1287 y(/*)h(compute)e(indices)h(of)h(particles)f(with)g (index)g(zero)g(*/)75 1344 y(j)h(=)f(0;)75 1400 y(for\(i=0;)g(i)g(<)h (1000;)f(i++\))123 1456 y(if)g(\(particle[i].index==0\))e(then)194 1513 y({)194 1569 y(zindex[j])i(=)h(i;)194 1626 y(zblock[i])f(=)h(1;) 194 1682 y(j++;)194 1739 y(})75 1852 y(/*)g(create)f(datatype)f(for)i (particles)e(with)i(index)f(zero)g(*/)75 1908 y(MPI_type_index\()f(j,)h (Particletype,)46 b(zindex,)23 b(zblock,)g(Zparticles\);)75 2021 y(/*)h(prepend)e(particle)h(count)g(*/)75 2077 y (MPI_address\(\(void)f(*\)&j,)h(zzindex\);)75 2134 y(zzindex[1])f(=)i (MPI_bottom;)75 2190 y(zztype[0])f(=)g(MPI_int;)75 2247 y(zztype[1])g(=)g(Zparticles;)75 2303 y(MPI_type_struc\(2,)f(zztype,)g (zzindex,)h(zzblock,)g(Ztype\);)75 2416 y(MPI_type_commit\()f(Ztype,)h (Type2\);)75 2473 y(MPI_send\()g(MPI_bottom,)f(1,)i(Type2,)f(dest,)g (tag,)g(comm\);)266 2642 y(/*)g(A)h(possibly)f(more)g(efficient)g(way)g (of)h(defining)f(Zparticles)f(*/)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 84 90 84 89 bop 75 -100 a FE(84)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(/*)24 b(consecutive)e(particles)h(with)g(index)g(zero)g(are)h(handled)f(as)g (one)h(block)f(*/)75 102 y(j=0;)75 158 y(for)g(\(i=0;)h(i)f(<)h(1000;)f (i++\))123 214 y(if)g(\(particle[i].index==0\))e(then)170 271 y({)170 327 y(for)j(\(k=i+1;)f(\(k)g(<)h (1000\)&&\(particle[k].index)d(==)i(0\))h(;)f(k++\);)170 384 y(zindex[j])g(=)h(i;)170 440 y(zblock[j])f(=)h(k-i;)170 497 y(j++;)170 553 y(i)g(=)g(k;)170 610 y(})75 666 y(MPI_type_index\()e (j,)h(Particletype,)f(zindex,)h(zblock,)g(Zparticles\);)266 835 y(/*)g(4.3:)h(send)f(the)h(first)f(two)g(coordinates)g(of)g(all)h (entries)f(*/)75 948 y(MPI_datatype)f(Allpairs;)142 b(/*)24 b(datatype)f(for)g(all)g(pairs)h(of)f(coordinates)g(*/)75 1005 y(MPI_datatype)f(Type3;)75 1118 y(int)238 b(sizeofentry;)75 1231 y(MPI_extent\()22 b(Particletype,)g(sizeofentry\);)194 1344 y(/*)i(sizeofentry)e(can)i(also)f(be)h(computed)e(by)i (subtracting)e(the)i(address)266 1400 y(of)f(particle[0])g(from)g(the)h (address)f(of)g(particle[1])f(*/)75 1513 y(MPI_type_hvector\()g(1000,)h (MPI_real,)f(sizeofentry,)h(2,)g(Allpairs\);)75 1569 y(MPI_type_commit\()f(Allpairs,)g(Type3\);)75 1626 y(MPI_send\()h (particle.d,)f(1,)i(Type3,)f(dest,)g(tag,)g(comm\);)218 1739 y(/*)h(an)f(alternative)g(solution)f(to)i(4.3)g(*/)75 1852 y(MPI_datatype)e(Onepair;)71 b(/*)23 b(datatype)g(for)h(one)f (pair)g(of)h(coordinates,)e(with)695 1908 y(the)i(extent)f(of)h(one)f (particle)g(entry)g(*/)75 1965 y(MPI_datatype)f(Type4;)75 2077 y(int)h(indexp[3];)75 2134 y(int)g(typep[3])g(=)h({MPI_lb,)f (MPI_double,)f(MPI_ub};)75 2190 y(int)h(blockp[3])g(=)h({1,)f(2,)h(1};) 75 2303 y(MPI_address\()e(\(void)h(*\)particle,)g(indexp\);)75 2360 y(MPI_address\()f(\(void)h(*\)particle[0].d,)f(indexp+1\);)75 2416 y(MPI_address\()g(\(void)h(*\)\(particle+1\),)f(indexp+2\);)75 2529 y(MPI_type_struc\()g(3,)h(typep,)g(indexp,)g(blockp,)g(Onepair\);) 75 2586 y(MPI_commit\()f(Onepair,)h(Type4\);)75 2642 y(MPI_send\()g(MPI_bottom,)f(1000,)h(Type4,)g(dest,)g(tag,)h(comm\);) 1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 85 91 85 90 bop 75 -100 a Fx(4.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FE(85)166 45 y Fv(Discussion:)166 95 y Fu(The)14 b(examples)e(carry)i(t)o(w)o(o)f(implicit)e(assumptions)h(on)h(the)h(w) o(a)o(y)f(padding)f(is)h(done)h(in)f(an)g(arra)o(y)g(of)g(struc-)75 145 y(tures.)166 195 y(First,)19 b(that)g(padding)e(is)i(done)f(in)g (the)h(same)f(w)o(a)o(y)g(in)g(the)h(successiv)o(e)h(arra)o(y)e(en)o (tries,)j(so)d(that)g(all)g(en-)75 244 y(tries)d(ha)o(v)o(e)g(an)f (iden)o(tical)g(memory)f(la)o(y)o(out.)19 b(E.g.,)13 b(w)o(e)i(assume)f(that)h(displacemen)o(t)f(from)f Fn(particle[i].b)f Fu(to)75 294 y Fn(particle[j].b)f Fu(is)j(the)g(same)f(as)h(the)h (displacemen)o(t)e(from)f Fn(particle[i].d)f Fu(to)j Fn(particle[j].d)p Fu(.)166 344 y(Second,)e(that)f(the)h(amoun)o(t)d (of)i(padding)f(added)i(is,)f(in)g(some)f(sense,)j(minim)o(al)o(.)i (I.e.,)10 b(if)h(the)g(most)f(stringen)o(t)75 394 y(alignmen)o(t)k (requiremen)o(t)j(for)g(a)f(basic)h(comp)q(onen)o(t)f(of)g(a)h (structure)h(is)f(alignmen)o(t)e(to)h(m)o(ultiples)f(of)h(2)1745 379 y Fg(k)1765 394 y Fu(,)h(then)75 444 y(no)h(con)o(tiguous)g (padding)f(space)i(has)f(size)h(greater)g(or)f(equal)g(to)g(2)1177 429 y Fg(k)1197 444 y Fu(.)31 b(This)18 b(assumption)e(is)i(implicit)e (in)h(the)75 493 y(de\014nition)c(of)h Fv(exten)o(t)p Fu(.)166 543 y(I)e(don't)g(think)h(either)g(assumptions)e(are)i (mandated)f(b)o(y)g(C)g(of)g(F)m(ortran)h(standards,)g(although)e(I)i (w)o(ould)e(b)q(e)75 593 y(v)o(ery)k(surprised)i(to)d(see)j(an)d (implemen)o(tation)e(that)j(do)q(es)g(not)g(ful\014ll)e(these)k (conditions.)k(If)14 b(I)h(am)e(wrong)i(\(e.g.,)75 643 y(if)g(some)h(compilers)f(alw)o(a)o(ys)g(align)g(structures)j(to)e (double)g(w)o(ord)g(b)q(oundaries,)h(ev)o(en)g(if)e(they)i(ha)o(v)o(e)f (no)g(double)75 693 y(w)o(ord)e(comp)q(onen)o(ts\),)f(then)i(the)f (de\014nition)f(of)28 b Fr(extent)15 b Fu(should)f(b)q(e)g(revisited.) 75 897 y Fq(4.13.5)49 b(Co)o(rrect)15 b(use)h(of)g(addresses)75 983 y FE(Successiv)o(ely)23 b(declared)f(v)m(ariables)h(in)e(C)g(or)g (F)l(ortran)e(are)i(not)g(necessarily)h(stored)f(at)f(con)o(tiguous)75 1039 y(lo)q(cations.)i(Th)o(us,)16 b(care)g(m)o(ust)f(b)q(e)h (exercised)i(that)d(displacemen)o(ts)i(do)f(not)f(cross)h(from)f(one)h (v)m(ariable)75 1095 y(to)e(another.)20 b(Also,)15 b(in)g(mac)o(hines)h (with)f(segmen)o(ted)g(address)g(space,)g(addresses)g(are)g(not)f (unique)j(and)75 1152 y(address)11 b(arithmetic)h(has)g(some)f(p)q (eculiar)i(prop)q(erties.)19 b(Th)o(us,)12 b(use)g(of)f FF(addresses)p FE(,)f(i.e.)20 b(displacemen)o(ts)75 1208 y(relativ)o(e)c(to)e(the)i(start)e(address)29 b Fr(MPI)p 733 1208 13 2 v 14 w(BOTTOM)p FE(,)14 b(has)h(to)g(b)q(e)h(restricted.) 166 1265 y(V)l(ariables)k(b)q(elong)f(to)f(the)h(same)f FF(sequen)o(tial)k(storage)d FE(if)g(they)g(b)q(elong)g(to)f(the)h (same)f(arra)o(y)l(,)75 1321 y(to)g(the)g(same)g Fp(COMMON)g FE(blo)q(c)o(k)h(in)g(F)l(ortran,)f(or)g(to)f(the)i(same)f(structure)g (in)h(C.)f(V)l(alid)i(addresses)e(are)75 1378 y(de\014ned)f(recursiv)o (ely)f(as)f(follo)o(ws:)131 1482 y(1.)22 b(The)13 b(function)h Fo(MPI)p 541 1482 14 2 v 16 w(ADDRESS)g FE(returns)f(a)g(v)m(alid)h (address,)g(when)f(passed)h(as)e(argumen)o(t)h(a)f(host)189 1538 y(program)i(v)m(ariable.)131 1632 y(2.)22 b(The)30 b Fo(sta)o(rt)15 b FE(parameter)f(of)h(a)f(comm)o(unication)i(function) g(ev)m(aluates)f(to)g(a)f(v)m(alid)j(address,)d(when)189 1688 y(passed)h(as)g(argumen)o(t)f(a)h(host)g(program)f(v)m(ariable.) 131 1781 y(3.)22 b(If)33 b Fo(v)16 b FE(is)h(a)f(v)m(alid)i(address,)e (and)33 b Fo(i)17 b FE(is)g(an)f(in)o(teger,)g(then)33 b Fo(v+i)17 b FE(is)g(a)f(v)m(alid)i(address,)e(pro)o(vided)34 b Fo(v)189 1838 y FE(and)c Fo(v+i)15 b FE(are)g(in)h(the)g(same)f (sequen)o(tial)h(storage.)131 1931 y(4.)22 b(If)30 b Fo(v)15 b FE(is)h(a)f(v)m(alid)i(address)e(then)29 b Fr(MPI)p 835 1931 13 2 v 15 w(BOTTOM)13 b(+)h(v)i FE(is)f(a)g(v)m(alid) i(address.)166 2035 y(A)22 b(correct)g(program)g(uses)g(only)h(v)m (alid)i(addresses)d(to)g(iden)o(tify)i(the)e(lo)q(cations)h(of)f(en)o (tries)h(in)75 2091 y(comm)o(unication)14 b(bu\013ers.)19 b(F)l(urthermore,)12 b(if)27 b Fo(u)13 b FE(and)26 b Fo(v)13 b FE(are)g(t)o(w)o(o)e(v)m(alid)k(addresses,)e(then)g(the)g (\(in)o(teger\))75 2148 y(di\013erence)32 b Fo(u)16 b(-)f(v)h FE(can)f(b)q(e)h(computed)g(only)g(if)g(b)q(oth)31 b Fo(u)15 b FE(and)h Fp(v)f FE(are)g(in)h(the)g(same)f(sequen)o(tial)i (storage;)75 2204 y(no)e(other)g(arithmetic)h(op)q(erations)f(can)g(b)q (e)h(meaningfully)i(executed)e(on)f(addresses.)166 2261 y(W)l(e)f(sa)o(y)g(that)g(a)g(datat)o(yp)q(e)g(is)h FF(absolute)g FE(if)g(all)h(displacemen)o(ts)g(within)f(this)g(datat)o(yp)q(e)f(are)g (v)m(alid)75 2317 y(\(absolute\))h(addresses;)g(it)g(is)h FF(relativ)o(e)g FE(otherwise.)166 2374 y(A)f(correct)g(program)f(ob)q (eys)i(the)f(follo)o(wing)h(constrain)o(ts)e(in)i(the)g(use)f(of)g (datat)o(yp)q(es:)143 2478 y FA(\017)23 b FE(If)d(the)41 b Fo(oldt)o(yp)q(e)21 b FE(argumen)o(t)e(used)i(in)41 b Fo(MPI)p 981 2478 14 2 v 16 w(TYPE)p 1114 2478 V 17 w(CONTIGUOUS,)22 b(MPI)p 1536 2478 V 15 w(TYPE)p 1668 2478 V 17 w(VECTOR,)189 2534 y(MPI)p 274 2534 V 15 w(TYPE)p 406 2534 V 17 w(HVECTOR,)d(MPI)p 748 2534 V 16 w(TYPE)p 881 2534 V 17 w(INDEX)f FE(or)36 b Fo(MPI)p 1210 2534 V 16 w(TYPE)p 1343 2534 V 17 w(HINDEX)18 b FE(is)h(absolute,)g(then)189 2591 y(all)c(addresses)g(within)g(it)g(m)o(ust)f(b)q(e)h(to)f(v)m (ariables)i(con)o(tained)f(within)g(the)g(same)f(arra)o(y)f(or)h (struc-)189 2647 y(ture;)j(the)g(result)h(datat)o(yp)q(e)e(is)i(also)f (absolute,)h(and)f(all)h(computed)f(addresses)h(m)o(ust)e(also)h(fall) 189 2704 y(within)f(the)f(same)g(arra)o(y)f(or)h(structure.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 86 92 86 91 bop 75 -100 a FE(86)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)143 45 y FA(\017)23 b FE(If)c(all)h(en)o(tries)f(of)f(the)38 b Fo(a)o(rra)o(y)p 708 45 14 2 v 15 w(of)p 760 45 V 16 w(t)o(yp)q(es)20 b FE(argumen)o(t)e(of)h Fo(MPI)p 1238 45 V 16 w(TYPE)p 1371 45 V 16 w(STRUC)h FE(are)f(absolute)g(ad-)189 102 y(dresses)f(\(computed)h(b)o(y)g Fo(MPI)p 727 102 V 15 w(ADDR)p FE(\),)g(then)f(the)h(result)g(datat)o(yp)q(e)f(is)h(also)g (absolute.)30 b(Eac)o(h)189 158 y(new)14 b(address)h(computed)g(from)f (an)g(old)h(address)g(m)o(ust)f(fall)h(within)h(the)f(same)f(arra)o(y)f (or)h(record)189 214 y(as)h(the)i(old)f(address.)23 b(\(Addresses)16 b(in)h(the)f(old)h(t)o(yp)q(e)f(ma)o(y)f(not)h(b)q(e)h(all)g(within)g (the)f(same)g(arra)o(y)189 271 y(or)e(record.\))143 374 y FA(\017)23 b FE(If)17 b(a)f(comm)o(unication)i(call)g(is)f(in)o(v)o (ok)o(ed)g(with)g(parameters)33 b Fo(sta)o(rt)17 b FE(and)34 b Fo(datat)o(yp)q(e)p FE(,)18 b(then)f(either)189 430 y Fo(sta)o(rt)f FE(=)31 b Fr(MPI)p 434 430 13 2 v 15 w(BOTTOM)15 b FE(and)33 b Fo(datat)o(yp)q(e)18 b FE(is)e(a)g(handle)i (to)d(an)h(absolute)h(datat)o(yp)q(e,)e(or)32 b Fo(sta)o(rt)16 b FE(is)189 487 y(set)h(to)f(the)h(address)h(of)f(a)f(program)g(v)m (ariable,)j(and)f(all)g(displacemen)o(ts)g(in)36 b Fo(datat)o(yp)q(e)18 b FE(relativ)o(e)189 543 y(to)c(this)h(base)g(yield)i(addresses)e(that) f(are)h(within)h(the)f(same)g(arra)o(y)f(or)g(record)h(as)g(this)g(v)m (ariable.)166 661 y(In)j(summary)l(,)g(the)f(t)o(yp)q(e)h(constructors) 34 b Fo(MPI)p 984 661 14 2 v 16 w(TYPE)p 1117 661 V 17 w(CONTIGUOUS,)19 b(MPI)p 1536 661 V 15 w(TYPE)p 1668 661 V 17 w(VECTOR,)75 718 y(MPI)p 160 718 V 16 w(TYPE)p 293 718 V 17 w(HVECTOR,)c(MPI)p 631 718 V 16 w(TYPE)p 764 718 V 17 w(INDEX)f FE(and)29 b Fo(MPI)p 1114 718 V 16 w(TYPE)p 1247 718 V 17 w(HINDEX)14 b FE(can)h(b)q(e)g(recursiv)o (ely)h(ap-)75 774 y(plied)22 b(to)e(build)j(datat)o(yp)q(es)d(that)f (will)k(com)o(bine)e(v)m(ariables)h(that)e(b)q(elong)h(to)f(the)h(same) f(sequen)o(tial)75 831 y(storage;)d(v)m(ariables)i(that)e(do)g(not)g(b) q(elong)i(to)d(the)i(same)f(sequen)o(tial)i(storage)d(can)i(b)q(e)g (com)o(bined)g(to-)75 887 y(gether)d(using)h(one)f(application)i(of)e Fo(MPI)p 788 887 V 16 w(TYPE)p 921 887 V 16 w(STRUC)p FE(.)166 1022 y Fv(Discussion:)166 1074 y Fu(It)f(is)g(not)g(exp)q (ected)j(that)d(MPI)g(implemen)o(tations)d(will)i(b)q(e)i(able)f(to)g (detect)i(erroneous,)f(\\out)f(of)f(b)q(ound")75 1124 y(displacemen)o(ts)h({)f(unless)i(those)g(o)o(v)o(er\015o)o(w)f(the)g (user)h(address)h(space)f({)f(since)g(the)h(MPI)f(call)g(ma)o(y)e(not)i (no)o(w)f(the)75 1174 y(exten)o(t)i(of)e(the)i(arra)o(ys)f(and)f (records)j(in)d(the)i(host)f(program.)75 1413 y Fy(4.14)59 b(Universal)20 b(communication)d(functions)75 1519 y FE(Although)g(the)f(n)o(um)o(b)q(er)g(of)g(comm)o(unication)g (functions)h(de\014ned)h(in)e(this)h(c)o(hapter)f(is)g(fairly)h(large,) f(it)75 1575 y(is)e(p)q(ossible)g(to)f(construct)g(all)h(of)e(them)h (from)g(a)f(m)o(uc)o(h)h(smaller)h(n)o(um)o(b)q(er)g(of)e(primitiv)o(e) j(functions.)20 b(The)75 1632 y(construction)d(is)h(outlined)g(in)g (this)f(section.)26 b(The)17 b(construction)g(can)g(b)q(e)h(used)g(to)e (build)j(a)d(p)q(ortable)75 1688 y(implemen)o(tation)22 b(of)e(MPI,)h(although)g(it)g(is)g(exp)q(ected)h(that)e(more)h (e\016cien)o(t)g(implemen)o(tation)h(will)75 1744 y(implemen)o(t)h (directly)f(most)f(or)g(all)h(of)f(the)h(functions)g(in)g(this)g (section.)39 b(Also,)23 b(this)f(construction)75 1801 y(can)15 b(b)q(e)g(used)g(in)g(order)g(to)e(deriv)o(e)j(the)e(b)q(eha)o (vior)h(of)f(MPI)h(p)q(oin)o(t)g(to)f(p)q(oin)o(t)h(comm)o(unication)g (from)f(the)75 1857 y(b)q(eha)o(vior)e(of)f(a)g(small)h(subset)g(of)f (functions.)19 b(W)l(e)12 b(assume)f(the)g(a)o(v)m(ailabilit)o(y)j(of)d (the)g(primitiv)o(e)i(functions)75 1914 y(listed)j(at)f(the)g(end)h(of) f(the)g(section,)h(and)f(the)g(abilit)o(y)i(to)d(allo)q(cate)i(memory)f (for)f(new)i(v)m(ariables.)166 1973 y(W)l(e)g(assume)g(that)f(the)i (function)g Fo(MPI)p 846 1973 V 15 w(TYPE)p 978 1973 V 17 w(COMMIT)f FE(is)g(not)g(c)o(hanging)h(the)f(represen)o(tation)75 2029 y(of)f(a)g(datat)o(yp)q(e,)f(but)h(merely)h(creates)f(a)g(new)g (cop)o(y)g(of)g(it.)166 2088 y(Comm)o(unication)g(with)g(a)30 b Fo(count)16 b FE(argumen)o(t)e(that)g(is)i FD(>)d FE(1)h(can)h(b)q(e) h(reduced)g(to)e(comm)o(unication)75 2144 y(with)33 b Fo(count)17 b(=1)p FE(,)f(b)o(y)g(replicating)i(the)32 b Fo(datat)o(yp)q(e)18 b FE(argumen)o(t)32 b Fo(count)17 b FE(times.)23 b(Comm)o(unication)17 b(with)75 2201 y(a)29 b Fo(a)14 b(sta)o(rt)h FE(argumen)o(t)e(that)h(is)h(not)28 b Fr(MPI)p 780 2201 13 2 v 14 w(BOTTOM)14 b FE(can)h(b)q(e)g(reduced)g (to)f(comm)o(unication)h(with)30 b Fo(sta)o(rt)75 2257 y FE(=)f Fr(MPI)p 216 2257 V 15 w(BOTTOM)14 b FE(b)o(y)i(creating)f(a)g (suitable)h(displaced)h(new)f(datat)o(yp)q(e.)j(Th)o(us:)166 2316 y Fo(MPI)p 251 2316 14 2 v 16 w(SEND\()c(sta)o(rt,)g(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm\))e FE(is)75 2421 y Fp(Type[1])23 b(=)h(datatype)75 2478 y(MPI_ADDRESS\()e(start,)h (Index[1]\);)75 2534 y(Block[1])g(=)g(count;)75 2591 y(MPI_TYPE_STRUCT\()f(1,)h(Type,)g(Index,)g(Block,)h(Newtype\))75 2647 y(MPI_TYPE_COMMIT\()e(Newtype,)g(datatype1\))75 2704 y(MPI_SEND\()h(MPI_BOTTOM,)f(1,)i(datatype1,)e(dest,)h(tag,)h (comm\))1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 87 93 87 92 bop 75 -100 a Fx(4.14.)34 b(UNIVERSAL)17 b(COMMUNICA)l(TION)f (FUNCTIONS)633 b FE(87)166 45 y(The)18 b(same)g(construction)h(applies) g(to)f(all)h(other)f(comm)o(unication)g(functions)h(with)g(argumen)o (ts)75 102 y Fo(count)d FE(and)29 b Fo(datat)o(yp)q(e)p FE(.)21 b(W)l(e)15 b(shall)h(henceforth)f(restrict)f(ourselv)o(es)h(to) f(comm)o(unication)i(that)d(in)o(v)o(olv)o(es)75 158 y(only)j(one)f(elemen)o(t)h(with)g(an)f(absolute)g(address)g(\()g Fo(count)i(=1)p FE(,)e(and)30 b Fo(sta)o(rt)16 b FE(=)29 b Fr(MPI)p 1521 158 13 2 v 14 w(BOTTOM)p FE(\).)166 216 y(\(If)11 b(the)g(op)q(eration)g Fo(MPI)p 583 216 14 2 v 16 w(TYPE)p 716 216 V 17 w(COMMIT\))f FE(is)h(not)g(assumed)g(to)g (b)q(e)g(trivial,)i(then)e(the)g(ab)q(o)o(v)o(e)g(con-)75 273 y(struction)f(cannot)g(b)q(e)h(used;)h(this)f(adds)f(one)g(more)g (primitiv)o(e)h(function,)h(namely)f Fo(MPI)p 1562 273 V 16 w(TYPE)p 1695 273 V 16 w(COMMIT)p FE(,)75 329 y(to)k(our)f(list,)i (and)g(in)o(tro)q(duces)g(small)g(c)o(hanges)f(in)h(the)f (construction.\))166 387 y(It)10 b(is)h(con)o(v)o(enien)o(t)g(to)e(to)h (mak)o(e)f(explicit)k(the)d FF(p)q(ersistence)g FE(attribute)g(of)g (comm)o(unication)h(ob)s(jects.)75 444 y(A)30 b Fo(p)q(ersistent)18 b FE(comm)o(unication)d(ob)s(ject)g(need)h(b)q(e)f(explicitly)j(deallo) q(cated)f(b)o(y)e(a)29 b Fo(MPI)p 1594 444 V 16 w(COMM)p 1752 444 V 16 w(FREE)75 500 y FE(op)q(eration.)j(On)20 b(the)f(other)g(hand,)h(an)38 b Fo(ephemeral)19 b FE(comm)o(unication)h (ob)s(ject)f(is)g(go)q(o)q(d)g(for)g(a)g(single)75 557 y(comm)o(unication.)h(It)14 b(is)g(deallo)q(cated)i(b)o(y)e(the)g (system)f(when)h(the)g(\014rst)g(comm)o(unication)h(it)f(is)g(used)h (for)75 613 y(completes.)166 672 y(The)c(function)g Fo(MPI)p 513 672 V 16 w(COMM)p 671 672 V 16 w(INIT)f FE(is)h(a)f(univ)o(ersal)i (function)f(for)f(the)h(creation)f(of)g(comm)o(unication)75 728 y(ob)s(jects:)75 834 y Fo(MPI)p 160 834 V 16 w(COMM)p 318 834 V 16 w(INIT\(handle,)15 b(datat)o(yp)q(e,)i(source-dest,)g (tag,)e(comm,)e(op-mo)q(de,)h(p)q(ersistence\))117 913 y Fu(OUT)108 b Fo(handle)465 b Fu(handle)14 b(to)g(newly)f(created)j (comm)o(unicatio)o(n)11 b(ob)r(ject)117 991 y(IN)155 b Fo(datat)o(yp)q(e)424 b Fu(datat)o(yp)q(e)14 b(of)g(elemen)o(t)f(sen) o(t)i(or)f(receiv)o(ed)117 1070 y(IN)155 b Fo(source-dest)377 b Fu(rank)14 b(of)f(destination)h(or)g(source)h(\(in)o(teger\))117 1149 y(IN)155 b Fo(tag)526 b Fu(message)14 b(tag)f(\(in)o(teger\))117 1228 y(IN)155 b Fo(comm)470 b Fu(comm)o(unication)11 b(ob)r(ject)117 1306 y(IN)155 b Fo(op-mo)q(de)422 b Fu(one)10 b(of)18 b Fr(MPI)p 1106 1306 13 2 v 14 w(ST)m(AND)o(ARD)p Fu(,)f Fr(MPI)p 1443 1306 V 14 w(READ)o(Y)p Fu(,)g Fr(MPI)p 1698 1306 V 15 w(SYNCHRONOUS)905 1363 y Fu(or)28 b Fr(MPI)p 1047 1363 V 14 w(RECV)117 1442 y Fu(IN)155 b Fo(p)q(ersistence)383 b Fu(one)14 b(of)27 b Fr(MPI)p 1119 1442 V 15 w(PERSISTENT)13 b Fu(or)28 b Fr(MPI)p 1534 1442 V 14 w(EPHEMERAL)166 1568 y FE(The)15 b(comm)o(unication)h(in)o(v)o(olv)o(es)g(only)g(one)f (elemen)o(t,)h(with)f(an)g(absolute)h(address.)75 1700 y Fq(4.14.1)49 b(P)o(ersistent)14 b(communication)j(objects)75 1789 y Fo(MPI)p 160 1789 14 2 v 16 w(CREA)l(TE)p 345 1789 V 17 w(SEND\(handle,)g(MPI)p 731 1789 V 15 w(BOTTOM,)f(1,)e(datat) o(yp)q(e,)j(dest,)f(tag,)f(comm\))e FE(is)75 1907 y Fp (MPI_COMM_INIT\(handle,)21 b(datatype,)i(dest,)g(tag,)g(comm,)g (MPI_STANDARD,)f(MPI_PERSISTENT\))166 2022 y FE(The)f(functions)g Fo(MPI)p 551 2022 V 16 w(CREA)l(TE)p 736 2022 V 17 w(RSEND)p FE(,)g Fo(MPI)p 1015 2022 V 16 w(CREA)l(TE)p 1200 2022 V 17 w(RSEND)g FE(and)g Fo(MPI)p 1560 2022 V 16 w(CREA)l(TE)p 1745 2022 V 17 w(RECV)75 2078 y FE(are)15 b(dealt)h(in)g(a)e(similar)j (manner.)166 2137 y(The)e(functions)h Fo(MPI)p 540 2137 V 16 w(ST)l(ART)g FE(and)g Fo(MPI)p 885 2137 V 15 w(COMM)p 1042 2137 V 17 w(FREE)g FE(are)e(primitiv)o(e.)75 2269 y Fq(4.14.2)49 b(Nonblo)q(cking)19 b(communication)d(initiation)75 2358 y Fo(MPI)p 160 2358 V 16 w(ISEND\(handle,)g(MPI)p 557 2358 V 15 w(BOTTOM,)g(1,)f(datat)o(yp)q(e,)h(dest,)g(tag,)f(comm\)) e FE(is)75 2475 y Fp(MPI_COMM_INIT\(handle,)21 b(datatype,)i(dest,)g (tag,)g(comm,)g(MPI_STANDARD,)f(MPI_EPHEMERAL\))75 2532 y(MPI_START\(handle\))166 2647 y FE(The)g(functions)g Fo(MPI)p 553 2647 V 16 w(IRSEND)p FE(,)f Fo(MPI)p 844 2647 V 16 w(ISSEND)h FE(and)g Fo(MPI)p 1214 2647 V 16 w(IRECV)g FE(are)f(handled)i(in)g(a)e(similar)75 2704 y(manner.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 88 94 88 93 bop 75 -100 a FE(88)630 b Fx(SECTION)15 b(4.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fq(4.14.3)49 b(Communication)17 b(completion)75 131 y FE(The)e(t)o(w)o(o)f(primitiv) o(e)j(completion)f(op)q(erations)g(are)e Fo(MPI)p 1063 131 14 2 v 16 w(W)l(AIT)l(ANY)i FE(and)g Fo(MPI)p 1467 131 V 16 w(TEST)l(ALL)p FE(.)166 187 y(The)c(function)h Fo(MPI)p 516 187 V 15 w(W)l(AIT)f FE(can)g(b)q(e)h(implemen)o(ted)g(b)o (y)f(a)f(call)i(to)e Fo(MPI)p 1368 187 V 16 w(W)l(AIT)l(ANY)i FE(with)f(an)24 b Fo(a)o(rra)o(y-)75 244 y(of-handles)17 b FE(argumen)o(t)d(of)h(length)h(one.)166 300 y(The)h(function)h Fo(MPI)p 526 300 V 16 w(W)l(AIT)l(ALL)f FE(can)g(b)q(e)h(implemen)o (ted)h(as)d(a)h(lo)q(op)h(where)f(an)34 b Fo(MPI)p 1664 300 V 16 w(W)l(AIT)17 b FE(call)75 357 y(is)f(executed)g(for)e(eac)o(h) i(successiv)o(e)g(handle)h(in)f(the)30 b Fo(a)o(rra)o(y-of-handles)p FE(.)166 413 y(The)13 b(function)g Fo(MPI)p 517 413 V 16 w(TEST)g FE(can)f(b)q(e)i(implemen)o(ted)g(b)o(y)e(a)h(call)g(to)f Fo(MPI)p 1377 413 V 16 w(TEST)l(ALL)h FE(with)g(an)25 b Fo(a)o(rra)o(y-)75 470 y(ofhandles)17 b FE(of)d(length)i(one.)166 526 y(The)h(function)h Fo(MPI)p 526 526 V 16 w(TEST)l(ALL)e FE(can)h(b)q(e)h(implemen)o(ted)h(as)d(a)h(lo)q(op)g(where)g(an)34 b Fo(MPI)p 1663 526 V 16 w(TEST)17 b FE(call)75 583 y(is)f(executed)g (for)e(eac)o(h)i(successiv)o(e)g(handle)h(in)f(the)30 b Fo(a)o(rra)o(y-of-handle)p FE(.)75 704 y Fq(4.14.4)49 b(Blo)q(cking)18 b(communication)75 789 y Fo(MPI)p 160 789 V 16 w(SEND\(MPI)p 393 789 V 16 w(BOTTOM,)d(1,)g(datat)o(yp)q(e,)h (dest,)h(tag,)d(comm\))f FE(is)75 890 y Fp(MPI_COMM_INIT\(handle,)21 b(datatype,)i(dest,)g(tag,)g(comm,)g(MPI_STANDARD,)f(MPI_EPHEMERAL\))75 946 y(MPI_START\(handle\))75 1003 y(MPI_WAIT\(handle,)g (dontcarestatus\))166 1104 y FE(The)15 b(functions)h Fo(MPI)p 540 1104 V 16 w(RSEND)g FE(and)f Fo(MPI)p 888 1104 V 16 w(SSEND)h FE(are)f(handled)i(in)f(a)f(similar)h(manner.)166 1160 y Fo(MPI)p 251 1160 V 16 w(RECV\(MPI)p 482 1160 V 16 w(BOTTOM,)g(1,)e(datat)o(yp)q(e,)j(source,)e(tag,)g(comm\))e FE(is)75 1261 y Fp(MPI_COMM_INIT\(handle,)21 b(datatype,)i(dest,)g (tag,)g(comm,)g(MPI_RECV,)g(MPI_EPHEMERAL\))75 1317 y (MPI_START\(handle\))75 1374 y(MPI_WAIT\(handle,)f(status\))75 1495 y Fq(4.14.5)49 b(Prob)q(e)17 b(and)f(cancel)75 1581 y FE(The)f(functions)h Fo(MPI)p 449 1581 V 16 w(PROBE)h FE(and)e Fo(MPI)p 800 1581 V 16 w(CANCEL)g FE(are)g(primitiv)o(e.)75 1701 y Fq(4.14.6)49 b(Return)16 b(status)75 1787 y FE(The)k(functions)g Fo(MPI)p 458 1787 V 16 w(GET)p 562 1787 V 16 w(SOURCE)p FE(,)h Fo(MPI)p 868 1787 V 16 w(GET)p 972 1787 V 16 w(T)l(A)o(G)p FE(,)f Fo(MPI)p 1189 1787 V 16 w(GET)p 1293 1787 V 16 w(LEN)p FE(,)f Fo(MPI)p 1507 1787 V 16 w(PROBE)p 1671 1787 V 18 w(LEN)p FE(,)g(and)75 1844 y Fo(MPI)p 160 1844 V 16 w(IS)p 214 1844 V 16 w(CANCELLED)c FE(are)g(primitiv)o(e.)166 1900 y(These)g(functions)h(are)f(simple)i(macros)d(that)h(access)g (records)g(in)h(a)f(structure.)75 2021 y Fq(4.14.7)49 b(send-receive)15 b(and)i(exchange)75 2107 y Fo(MPI)p 160 2107 V 16 w(SENDRECV\()22 b(send)p 531 2107 V 18 w(sta)o(rt,)g(send)p 753 2107 V 18 w(count,)h(send)p 995 2107 V 18 w(t)o(yp)q(e,)g(dest,)h(recv)p 1319 2107 V 16 w(sta)o(rt,)e(recv)p 1533 2107 V 17 w(count,)h(recv)p 1768 2107 V 16 w(t)o(yp)q(e,)75 2163 y(source,)15 b(tag,)g(comm,)e (status\))k FE(is)75 2264 y Fp(MPI_ISEND\(handle[0],)k(send_start,)h (send_count,)h(send_type,)f(dest,)i(tag,)f(comm\))75 2321 y(MPI_IRECV\(handle[1],)e(recv_start,)h(recv_count,)h(recv_type,)f (source,)h(tag,)h(comm\))75 2377 y(MPI_WAITALL\(2,)e(handle,)h (status_array\))75 2434 y(status)g(=)h(status_array[1])166 2534 y FE(The)13 b(non)o(blo)q(c)o(king)h(sends)f(and)g(receiv)o(es)h (can)f(b)q(e)g(replaced)h(b)o(y)f(their)g(primitiv)o(e)h(implemen)o (tation.)166 2591 y(The)f(function)i Fo(MPI)p 519 2591 V 15 w(EX)o(CHANGE)g FE(can)e(b)q(e)h(handled)h(in)f(a)f(similar)i (manner;)e(a)g(temp)q(orary)g(bu\013er)75 2647 y(need)20 b(b)q(e)h(allo)q(cated)f(to)f(replicate)i(the)e(send)h(and)g(receiv)o (e)h(bu\013er.)32 b(A)20 b(size)g(for)f(this)h(bu\013er)g(can)f(b)q(e) 75 2704 y(computed)d(using)g(the)f(function)h Fo(MPI)p 745 2704 V 16 w(TYPE)p 878 2704 V 16 w(EXTENT)p FE(.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 89 95 89 94 bop 75 -100 a Fx(4.14.)34 b(UNIVERSAL)17 b(COMMUNICA)l(TION)f (FUNCTIONS)633 b FE(89)75 45 y Fq(4.14.8)49 b(Derived)16 b(datat)o(yp)q(es)75 131 y FE(W)l(e)22 b(ha)o(v)o(e)f(outlined)i(in)f (Section)g(4.13.1)e(ho)o(w)h(eac)o(h)h(datat)o(yp)q(e)e(constructor)h (can)h(b)q(e)g(expressed)g(in)75 187 y(terms)d(of)g(the)g(next)g(one.) 32 b(Th)o(us)20 b(all)g(datat)o(yp)q(e)f(constructors)f(can)h(b)q(e)h (expressed)g(in)g(terms)f(of)g(the)75 244 y(constructor)e Fo(MPI)p 403 244 14 2 v 15 w(TYPE)p 535 244 V 17 w(STRUC)p FE(;)h(the)f(functions)h Fo(MPI)p 1088 244 V 16 w(TYPE)p 1221 244 V 17 w(FREE)p FE(,)f Fo(MPI)p 1459 244 V 16 w(TYPE)p 1592 244 V 17 w(EXTENT)h FE(and)75 300 y Fo(MPI)p 160 300 V 16 w(ADDRESS)e FE(are)f(primitiv)o(e.)75 422 y Fq(4.14.9)49 b(Summa)o(ry)75 508 y FE(In)13 b(order)e(to)g(implemen)o (t)j(the)e(MPI)f(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)g(one)f (needs)g(16)g(functions;)h(ab)q(out)75 564 y(half)j(of)e(those)h(are)g (trivial)h(macros:)131 671 y(1.)22 b Fo(MPI)p 274 671 V 15 w(COMM)p 431 671 V 17 w(INIT)131 764 y FE(2.)g Fo(MPI)p 274 764 V 15 w(ST)l(ART)131 858 y FE(3.)g Fo(MPI)p 274 858 V 15 w(W)l(AIT)l(ALL)131 952 y FE(4.)g Fo(MPI)p 274 952 V 15 w(TEST)l(ANY)131 1046 y FE(5.)g Fo(MPI)p 274 1046 V 15 w(PROBE)131 1140 y FE(6.)g Fo(MPI)p 274 1140 V 15 w(CANCEL)131 1233 y FE(7.)g Fo(MPI)p 274 1233 V 15 w(GET)p 377 1233 V 17 w(SOURCE)131 1327 y FE(8.)g Fo(MPI)p 274 1327 V 15 w(GET)p 377 1327 V 17 w(T)l(A)o(G)131 1421 y FE(9.)g Fo(MPI)p 274 1421 V 15 w(GET)p 377 1421 V 17 w(LEN)108 1515 y FE(10.)g Fo(MPI)p 274 1515 V 15 w(PROBE)p 437 1515 V 18 w(LEN)108 1609 y FE(11.)g Fo(MPI)p 274 1609 V 15 w(IS)p 327 1609 V 17 w(CANCELLED)108 1702 y FE(12.)g Fo(MPI)p 274 1702 V 15 w(TYPE)p 406 1702 V 17 w(STRUC)108 1796 y FE(13.)g Fo(MPI)p 274 1796 V 15 w(TYPE)p 406 1796 V 17 w(EXTENT)108 1890 y FE(14.)g Fo(MPI)p 274 1890 V 15 w(ADDRESS)108 1984 y FE(15.)g Fo(MPI)p 274 1984 V 15 w(TYPE)p 406 1984 V 17 w(FREE)108 2078 y FE(16.)g Fo(MPI)p 274 2078 V 15 w(COMM)p 431 2078 V 17 w(FREE)166 2184 y FE(On)17 b(the)f(other)g(hand,)h(w)o(e)f(exp)q (ect)h(most)f(implemen)o(tations)h(to)f(use)h(more)e(primitiv)o(es,)j (in)f(order)75 2240 y(to)e(reduce)h(comm)o(unication)g(o)o(v)o (erheads,)e(reduce)i(bu\013ering,)f(and)h(impro)o(v)o(e)f(error)f(rep)q (orting.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 90 96 90 95 bop 75 358 a Fz(Section)35 b(5)75 567 y FG(Collecti)q(v)m(e)42 b(Comm)m(unication)75 774 y Fw(1)75 941 y Fy(5.1)59 b(Intro)r(duction) 75 1045 y FE(Collectiv)o(e)13 b(comm)o(unication)f(is)g(de\014ned)h(to) e(b)q(e)h(comm)o(unication)g(that)f(in)o(v)o(olv)o(es)h(a)f(group)g(of) g(pro)q(cesses.)75 1102 y(The)k(functions)h(pro)o(vided)g(b)o(y)g(the)f (MPI)g(collectiv)o(e)i(comm)o(unication)f(include:)143 1201 y FA(\017)23 b FE(Broadcast)14 b(from)g(one)i(mem)o(b)q(er)f(to)g (all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 1300 y FA(\017)23 b FE(Barrier)15 b(across)f(all)j(group)d(mem)o(b)q(ers)143 1399 y FA(\017)23 b FE(Gather)14 b(data)h(from)f(all)i(group)f(mem)o(b) q(ers)g(to)g(one)g(mem)o(b)q(er.)143 1498 y FA(\017)23 b FE(Scatter)14 b(data)h(from)f(one)i(mem)o(b)q(er)f(to)g(all)h(mem)o (b)q(ers)f(of)g(a)g(group.)143 1597 y FA(\017)23 b FE(Global)15 b(op)q(erations)g(suc)o(h)h(as)e(sum,)h(max,)f(min,)i(etc.,)e(w)o(ere)h (the)g(result)g(is)h(kno)o(wn)e(b)o(y)h(all)h(group)189 1654 y(mem)o(b)q(ers)e(and)h(a)g(v)m(ariation)g(where)g(the)g(result)g (is)g(kno)o(wn)f(b)o(y)h(only)g(one)g(mem)o(b)q(er.)20 b(The)15 b(abilit)o(y)189 1710 y(to)f(ha)o(v)o(e)h(user)g(de\014ned)i (global)f(op)q(erations.)143 1809 y FA(\017)23 b FE(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g(called)h (parallel)h(pre\014x\).)143 1908 y FA(\017)23 b FE(Broadcast)14 b(from)g(all)j(mem)o(b)q(ers)e(to)f(all)j(mem)o(b)q(ers)e(of)g(a)g (group.)143 2007 y FA(\017)23 b FE(Scatter)c(\(or)g(Gather\))g(data)g (from)g(all)i(mem)o(b)q(ers)f(to)f(all)i(mem)o(b)q(ers)f(of)g(a)f (group)h(\(also)f(called)189 2064 y(complete)d(exc)o(hange)f(or)g (all-to-all\).)75 2163 y(While)j(v)o(endors)f(ma)o(y)f(optimize)i (certain)g(collectiv)o(e)g(routines)g(for)e(their)h(arc)o(hitectures,)h (a)e(complete)75 2219 y(library)d(of)f(the)g(collectiv)o(e)i(comm)o (unication)f(routines)g(can)g(b)q(e)g(written)f(en)o(tirely)h(using)g (p)q(oin)o(t-to-p)q(oin)o(t)75 2276 y(comm)o(unication)j(functions.)166 2334 y(The)d(syn)o(tax)f(and)i(seman)o(tics)f(of)f(the)h(collectiv)o(e) i(op)q(erations)e(are)g(de\014ned)h(so)f(as)g(to)f(b)q(e)i(consisten)o (t)75 2390 y(with)22 b(the)f(syn)o(tax)g(and)h(seman)o(tics)f(of)g(the) h(p)q(oin)o(t-to-p)q(oin)o(t)g(op)q(erations.)39 b(A)21 b(collectiv)o(e)j(op)q(eration)75 2447 y(is)f(executed)h(b)o(y)f(ha)o (ving)g(all)g(pro)q(cesses)h(in)f(the)g(group)g(call)g(the)g(comm)o (unication)h(routine,)h(with)75 2503 y(matc)o(hing)14 b(parameters.)19 b(One)14 b(of)g(the)g(k)o(ey)g(parameters)f(is)i(a)e (comm)o(unicator)h(that)f(de\014nes)i(the)f(group)75 2560 y(of)i(participating)h(pro)q(cesses)f(and)h(pro)o(vides)f(a)g(con) o(text)f(for)h(the)g(op)q(eration.)23 b(The)16 b(reader)g(is)h (referred)75 2616 y(to)12 b(c)o(hapter)g(4)g(for)g(information)g (concerning)i(comm)o(unication)f(bu\013ers)f(and)h(their)g (manipulations)h(and)p 75 2661 720 2 v 127 2688 a FC(1)144 2704 y Fs(V)m(ersion)g(of)f(August)g(9,)g(1993)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 91 97 91 96 bop 75 -100 a Fx(5.2.)34 b(COMMUNICA)l(TION)16 b(FUNCTIONS)942 b FE(91)75 45 y(t)o(yp)q(e)16 b(matc)o(hing)h(rules;)g (and)g(to)f(c)o(hapter)g(3)g(for)g(information)g(on)g(ho)o(w)g(to)g (de\014ne)i(groups)e(and)g(create)75 102 y(comm)o(unicators.)166 159 y(Collectiv)o(e)h(routines)g(can)f(\(but)g(are)g(not)g(required)h (to\))e(return)i(as)e(so)q(on)h(as)g(their)h(participation)75 215 y(in)i(the)g(collectiv)o(e)h(comm)o(unication)f(is)g(complete.)31 b(The)19 b(completion)g(of)f(a)h(call)g(indicates)h(that)e(the)75 272 y(caller)d(is)g(no)o(w)e(free)h(to)g(access)g(the)g(lo)q(cations)h (in)g(the)f(comm)o(unication)g(bu\013er,)g(or)g(an)o(y)g(other)f(lo)q (cation)75 328 y(that)g(can)g(b)q(e)i(referenced)f(b)o(y)g(the)f (collectiv)o(e)i(op)q(eration.)20 b(It)14 b(do)q(es)f(not)h(indicate)h (that)d(other)i(pro)q(cesses)75 385 y(in)g(the)f(group)f(ha)o(v)o(e)h (started)f(the)h(op)q(eration)g(\(unless)h(otherwise)f(indicated)i(in)f (the)f(description)h(of)f(the)75 441 y(op)q(eration\).)19 b(The)11 b(successful)i(completion)g(of)e(a)h(collectiv)o(e)h(comm)o (unication)f(call)h(ma)o(y)e(dep)q(end)i(on)f(the)75 498 y(execution)i(of)e(a)h(matc)o(hing)f(call)i(at)e(all)i(pro)q (cesses)f(in)h(the)f(group.)18 b(Th)o(us,)13 b(a)g(collectiv)o(e)h (comm)o(unication)75 554 y(call)h(ma)o(y)l(,)e(or)g(ma)o(y)g(not,)h(ha) o(v)o(e)f(the)h(e\013ect)g(of)f(sync)o(hronizing)i(all)g(calling)g(pro) q(cesses.)20 b(A)14 b(more)f(detailed)75 611 y(discussion)g(of)e(the)g (correct)g(use)g(of)g(the)g(collectiv)o(e)i(routines)f(can)f(b)q(e)h (found)f(at)g(the)g(end)h(of)f(this)h(c)o(hapter.)166 751 y Fv(Discussion:)33 b Fu(The)13 b(collectiv)o(e)g(op)q(erations)g (do)g(not)f(accept)j(a)d(message)h(tag)f(parameter.)17 b(The)d(rationale)75 807 y(for)j(not)g(using)g(tags)g(is)h(that)f(the)h (need)g(for)f(distinguishing)f(collectiv)o(e)h(op)q(erations)h(with)f (the)h(same)e(con)o(text)75 864 y(seldom)g(arises)i(\(since)g(the)g(op) q(erations)f(are)h(blo)q(c)o(king\);)f(the)h(tag)f(\014eld)g(can)h(b)q (e)f(used)i(b)o(y)e(the)g(p)q(oin)o(t-to-p)q(oin)o(t)75 920 y(messages)d(that)g(implemen)o(t)d(the)j(collectiv)o(e)g(comm)o (unication.)75 1152 y Fy(5.2)59 b(Communication)18 b(F)n(unctions)75 1255 y FE(The)d(k)o(ey)h(concept)f(of)g(the)g(collectiv)o(e)i (functions)f(is)g(to)f(ha)o(v)o(e)f(a)h(\\group")g(of)g(participating)h (pro)q(cesses.)75 1312 y(The)j(routines)f(do)g(not)g(ha)o(v)o(e)g(a)g (group)g(iden)o(ti\014er)i(as)e(an)g(explicit)j(parameter.)28 b(Instead,)19 b(there)f(is)h(a)75 1368 y(comm)o(unicator)f(parameter.) 29 b(In)19 b(this)g(c)o(hapter)g(a)f(comm)o(unicator)g(can)h(b)q(e)g (though)o(t)f(of)g(as)g(a)g(group)75 1425 y(iden)o(ti\014er)f(merged)e (with)h(a)e(con)o(text.)166 1558 y Fv(Discussion:)38 b Fu(The)16 b(last)e(prop)q(osal)h(con)o(tained)g(t)o(w)o(o)g(la)o(y)o (ers)g(of)f(functions)h(one)g(\(con)o(tiguous\))g(bu\013er)h(for)75 1608 y(the)e(\\con)o(tiguous")g(functions,)f(and)h(an)f(arra)o(y)h(of)f (bu\013ers)j(for)d(the)i(noncon)o(tiguous)e(ones.)166 1659 y(The)e(latest)g(pt2pt)g(c)o(hapter)h(no)f(longer)g(con)o(tains)f (a)h(bu\013er)h(descriptor)g(handle,)f(instead)g(there)h(is)f(a)g (general)75 1709 y(datat)o(yp)q(e)16 b(parameter)f(whic)o(h)h(can)g (describ)q(e)h(arbitrary)f(structures.)26 b(I)16 b(ha)o(v)o(e)f(incorp) q(orated)h(this)g(feature)h(in)o(to)75 1759 y(the)g(collectiv)o(e)f (functions,)g(since)g(w)o(e)h(ha)o(v)o(e)e(agreed)i(that)f(the)h (collectiv)o(e)f(routines)g(m)o(ust)f(b)q(e)i(build-able)e(from)75 1808 y(the)f(pt2pt)g(routines.)75 2040 y Fy(5.3)59 b(Ba)n(rrier)21 b(synchronization)75 2191 y Fo(MPI)p 160 2191 14 2 v 16 w(BARRIER\()16 b(comm)d(\))117 2269 y Fu(IN)155 b Fo(comm)470 b Fu(comm)o(unicator)11 b(handle)166 2395 y Fo(MPI)p 251 2395 V 16 w(BARRIER)k FE(blo)q(c)o(ks)g(the)f(caller)h (un)o(til)g(all)f(group)g(mem)o(b)q(ers)g(ha)o(v)o(e)f(called)j(it;)e (the)g(call)h(returns)75 2451 y(at)g(an)o(y)f(pro)q(cess)i(only)g (after)e(all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered)h(the)f (call.)75 2600 y Fy(5.4)59 b(Data)19 b(move)g(functions)75 2704 y FE(Figure)c(5.1)g(illustrates)h(the)f(the)g(di\013eren)o(t)h (collectiv)o(e)h(mo)o(v)o(e)d(functions)i(supp)q(orted)g(b)o(y)f(MPI.) -32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 92 98 92 97 bop 75 -100 a FE(92)724 b Fx(SECTION)16 b(5.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 2420 y @beginspecial @setspecial %%BeginDocument: coll-fig1.ps /arrowdict 13 dict def % Local storage for the procedure % ``arrow.'' /arrow % The procedure ``arrow'' adds an { arrowdict begin % arrow shape to the current path. /headlength exch def % It takes seven arguments: the x /halfheadthickness exch 2 div def % and y coordinates of the tail /halfthickness exch 2 div def % (imagine that a line has been /tipy exch def /tipx exch def % drawn down the center of the /taily exch def /tailx exch def % arrow from the tip to the tail, % then x and y lie on this line), % the x and y coordinates of the % tip of the arrow, the thickness % of the arrow in the tail % portion, the thickness of the % arrow at the widest part of the % arrowhead and the length of the % arrowhead. /dx tipx tailx sub def % Compute the differences in x and /dy tipy taily sub def % y for the tip and tail. These /arrowlength dx dx mul dy dy mul add % will be used to compute the sqrt def % length of the arrow and to /angle dy dx atan def % compute the angle of direction % that the arrow is facing with % respect to the current user % coordinate system origin. /base arrowlength headlength sub def % Compute where the base of the % arrowhead will be. /savematrix matrix currentmatrix def % Save the current user coordinate % system. We are using the same % strategy to localize the effect % of transformations as was used % in the program to draw an % ellipse. tailx taily translate % Translate to the starting point % of the tail. angle rotate % Rotate the x-axis to correspond % with the center line of the % arrow. 0 halfthickness neg moveto % Add the arrow shape to the % current path. base halfthickness neg lineto base halfheadthickness neg lineto arrowlength 0 lineto base halfheadthickness lineto base halfthickness lineto 0 halfthickness lineto closepath savematrix setmatrix % Restore the current user % coordinate system. end } def /Mydict 100 dict def Mydict begin /Box { /height exch def /length exch def length 0 rlineto 0 height rlineto length neg 0 rlineto closepath } def /Grid { /ny exch def /nx exch def /dely exch def /delx exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def 0 1 ny { pop x y moveto lenx 0 rlineto stroke /y y dely add def} for /y ypos def /x xpos def 0 1 nx { pop x y moveto 0 leny rlineto stroke /x x delx add def} for } def /GridLabels { /shift exch def /raise exch def /yoff exch def /xoff exch def /p1 exch def /p2 exch def /ny exch def /nx exch def /dely exch def /delx exch def /Darray exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def /dx3 delx 3 div def /dy3 dely 3 div def /ix -1 def /iy ny 1 sub def Darray{ aload pop /Subc exch def /Text exch def /ix ix 1 add def ix nx ge { /ix 0 def /iy iy 1 sub def} if /x xpos delx ix 0.5 add mul add /Helvetica findfont p1 scalefont setfont Text stringwidth pop /Helvetica findfont p2 scalefont setfont Subc stringwidth pop add xoff add 2 div sub shift add def /y ypos dely iy 0.5 add mul add raise add def x y moveto /Helvetica findfont p1 scalefont Text show xoff yoff rmoveto /Helvetica findfont p2 scalefont Subc show } forall } def 2 setlinecap 6.5 72 mul 320 sub 2 div 0 translate 0 150 moveto 20 20 6 6 Grid 0 150 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 225 200 225 12 24 18 arrow stroke 200 195 140 195 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (one-all scatter) dup stringwidth pop 170 exch 2 div sub 242 moveto show (one-all gather) dup stringwidth pop 170 exch 2 div sub 170 moveto show 220 150 moveto 20 20 6 6 Grid 220 150 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(1)] [()()] [()()] [()()] [()()] [()()] [(A)(2)] [()()] [()()] [()()] [()()] [()()] [(A)(3)] [()()] [()()] [()()] [()()] [()()] [(A)(4)] [()()] [()()] [()()] [()()] [()()] [(A)(5)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 0 moveto 20 20 6 6 Grid 0 0 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [(B)(0)] [(B)(1)] [(B)(2)] [(B)(3)] [(B)(4)] [(B)(5)] [(C)(0)] [(C)(1)] [(C)(2)] [(C)(3)] [(C)(4)] [(C)(5)] [(D)(0)] [(D)(1)] [(D)(2)] [(D)(3)] [(D)(4)] [(D)(5)] [(E)(0)] [(E)(1)] [(E)(2)] [(E)(3)] [(E)(4)] [(E)(5)] [(F)(0)] [(F)(1)] [(F)(2)] [(F)(3)] [(F)(4)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 220 0 moveto 20 20 6 6 Grid 220 0 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(1)] [(B)(1)] [(C)(1)] [(D)(1)] [(E)(1)] [(F)(1)] [(A)(2)] [(B)(2)] [(C)(2)] [(D)(2)] [(E)(2)] [(F)(2)] [(A)(3)] [(B)(3)] [(C)(3)] [(D)(3)] [(E)(3)] [(F)(3)] [(A)(4)] [(B)(4)] [(C)(4)] [(D)(4)] [(E)(4)] [(F)(4)] [(A)(5)] [(B)(5)] [(C)(5)] [(D)(5)] [(E)(5)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 60 200 60 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (all-all scatter) dup stringwidth pop 170 exch 2 div sub 77 moveto show 0 300 moveto 20 20 6 6 Grid 0 300 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(B)(0)] [()()] [()()] [()()] [()()] [()()] [(C)(0)] [()()] [()()] [()()] [()()] [()()] [(D)(0)] [()()] [()()] [()()] [()()] [()()] [(E)(0)] [()()] [()()] [()()] [()()] [()()] [(F)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 360 200 360 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (all-all broadcast) dup stringwidth pop 170 exch 2 div sub 377 moveto show 220 300 moveto 20 20 6 6 Grid 220 300 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 450 moveto 20 20 6 6 Grid 0 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 510 200 510 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont 0 575 moveto (data) show (data) stringwidth pop 4 add 575 4 add (data) stringwidth pop 33 add 575 4 add 1 4 5 arrow fill (one-all broadcast) dup stringwidth pop 170 exch 2 div sub 527 moveto show gsave 0 570 (processes) stringwidth pop sub translate 90 rotate 0 5 moveto (processes) show -4 8 -33 8 1 4 5 arrow fill grestore 220 450 moveto 20 20 6 6 Grid 220 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels %%EndDocument @endspecial 98 x FE(Figure)f(5.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2574 y(eac)o(h)g(ro)o(w)g(of)g(b)q(o)o(xes)g (represen)o(ts)g(data)g(lo)q(cations)h(in)g(one)g(pro)q(cess.)k(Th)o (us,)15 b(in)h(the)f(one-all)i(broadcast,)75 2631 y(initially)k(just)e (the)f(\014rst)h(pro)q(cess)g(con)o(tains)f(the)h(data)f FD(A)1085 2638 y Fw(0)1105 2631 y FE(,)h(but)g(after)e(the)i(broadcast) f(all)i(pro)q(cesses)75 2687 y(con)o(tain)15 b(it.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 93 99 93 98 bop 75 -100 a Fx(5.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FE(93)75 45 y Fq(5.4.1)49 b(Broadcast)90 204 y Fo(MPI)p 175 204 14 2 v 16 w(BCAST\()16 b(bu\013er,)f(cnt,)g(t)o (yp)q(e,)h(ro)q(ot,)f(comm)e(\))117 295 y(IN/OUT)38 b(bu\013er)478 b Fu(starting)14 b(address)h(of)f(bu\013er)117 397 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(bu\013er) 117 499 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)i(\(p)q(ossibly)f(general\))117 600 y(IN)171 b Fo(ro)q(ot)492 b Fu(rank)14 b(of)f(broadcast)i(ro)q(ot)117 702 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 840 y Fo(MPI)p 251 840 V 16 w(BCAST)g FE(broadcasts)e(a)h(message)g (from)g(the)g(pro)q(cess)h(with)f(rank)21 b Fo(ro)q(ot)10 b FE(to)f(all)j(other)e(pro)q(cesses)75 897 y(of)15 b(the)g(group.)20 b(It)15 b(is)h(called)h(b)o(y)e(all)h(mem)o(b)q(ers)g(of)f(group)g (using)h(the)f(same)g(argumen)o(ts)f(for)30 b Fo(cnt,)16 b(t)o(yp)q(e,)75 953 y(comm,)j(and)i(ro)q(ot)p FE(.)34 b(On)20 b(return)g(the)g(con)o(ten)o(ts)f(of)h(the)g(bu\013er)g(of)g (the)g(pro)q(cess)g(with)g(rank)40 b Fo(ro)q(ot)19 b FE(is)75 1010 y(con)o(tained)d(in)g(the)f(bu\013er)g(of)g(the)g (calling)i(pro)q(cess.)75 1208 y Fq(5.4.2)49 b(Gather)75 1367 y Fo(MPI)p 160 1367 V 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcnt,)h (sendt)o(yp)q(e,)f(recvbuf,)f(recvcnt,)g(recvt)o(yp)q(e,)f(ro)q(ot,)g (comm\))117 1457 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f(send)g(bu\013er)117 1559 y(IN)171 b Fo(sendcnt)429 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 1661 y(IN)171 b Fo(sendt)o(yp)q(e)408 b Fu(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 1763 y(OUT)124 b Fo(recvbuf)434 b Fu(address)15 b(of)f(receiv)o(e)h (bu\013er)g({)e(signi\014can)o(t)h(only)f(at)h(ro)q(ot)117 1865 y(IN)171 b Fo(recvcnt)435 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1967 y(IN)171 b Fo(recvt)o(yp)q(e)414 b Fu(data)13 b(t)o(yp)q(e)i(of)d (recv)j(bu\013er)g(elemen)o(ts)e({)g(signi\014can)o(t)g(only)g(at)905 2023 y(ro)q(ot)117 2125 y(IN)171 b Fo(ro)q(ot)492 b Fu(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 2227 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 2365 y FE(Eac)o(h)17 b(pro)q(cess)g(\(including)i(the)e(ro)q(ot)f(pro)q (cess\))h(sends)g(the)g(con)o(ten)o(ts)g(of)f(its)h(send)h(bu\013er)f (to)f(the)75 2421 y(ro)q(ot)g(pro)q(cess.)25 b(The)17 b(ro)q(ot)f(pro)q(cess)h(places)h(all)f(the)g(incoming)h(messages)f(in) g(the)g(lo)q(cations)h(sp)q(eci\014ed)75 2478 y(b)o(y)e(the)g(recvbuf)h (and)f(recvt)o(yp)q(e.)23 b(The)16 b(receiv)o(e)h(bu\013er)f(is)g (ignored)h(for)e(all)i(non-ro)q(ot)f(pro)q(cesses.)23 b(The)75 2534 y(receiv)o(e)d(bu\013er)f(of)f(the)h(ro)q(ot)f(pro)q (cess)h(is)h(assumed)f(con)o(tiguous)g(and)g(partitioned)g(in)o(to)g Fo(MPI)p 1739 2534 V 16 w(GSIZE)75 2591 y FE(consecutiv)o(e)e(blo)q(c)o (ks,)h(eac)o(h)e(consisting)i(of)32 b Fo(sendcnt)20 b FE(elemen)o(ts.)k(The)17 b(data)f(sen)o(t)g(from)g(pro)q(cess)h(with)75 2647 y(rank)e(i)h(is)f(stored)g(in)h(the)f Fp(i)p FE(-th)g(blo)q(c)o (k.)21 b(The)15 b(function)i(is)e(called)i(with)f(the)f(same)g(v)m (alues)h(for)30 b Fo(sendcnt,)75 2704 y(sendt)o(yp)q(e,)17 b(ro)q(ot)p FE(,)e(and)30 b Fo(comm)14 b FE(at)g(all)i(participating)h (pro)q(cesses.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 94 100 94 99 bop 75 -100 a FE(94)724 b Fx(SECTION)16 b(5.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fq(5.4.3)49 b(Scatter)75 188 y Fo(MPI)p 160 188 14 2 v 16 w(SCA)l(TTER\()16 b(sendbuf,)h(sendcnt,)h(sendt)o(yp)q(e,)f(recvbuf,)e(recvcnt,)h(recvt)o (yp)q(e,)g(ro)q(ot,)e(comm\))117 270 y Fu(IN)171 b Fo(sendbuf)428 b Fu(address)15 b(of)f(send)g(bu\013er)h({)f(signi\014can)o(t)f(only)g (at)h(ro)q(ot)117 356 y(IN)171 b Fo(sendcnt)429 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 441 y(IN)171 b Fo(sendt)o(yp)q(e)408 b Fu(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)117 527 y(OUT)124 b Fo(recvbuf)434 b Fu(address)15 b(of)f(receiv)o(e)h(bu\013er.)117 612 y(IN)171 b Fo(recvcnt)435 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 697 y(IN)171 b Fo(recvt)o(yp)q(e)414 b Fu(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)117 783 y(IN)171 b Fo(ro)q(ot)492 b Fu(rank)14 b(of)f(sending)h(pro)q(cess)i(\(in)o (teger\))117 868 y(IN)171 b Fo(group)463 b Fu(comm)o(unicator)11 b(handle)166 998 y FE(The)16 b(ro)q(ot)g(pro)q(cess)g(sends)h(the)f Fp(i)p FE(-th)g(p)q(ortion)h(of)e(its)i(send)g(bu\013er)f(to)f(the)i (pro)q(cess)f(with)h(rank)f Fp(i)p FE(;)75 1054 y(eac)o(h)f(pro)q(cess) h(\(including)i(the)e(ro)q(ot)e(pro)q(cess\))h(stores)g(the)h(incoming) g(message)f(in)h(its)g(receiv)o(e)g(bu\013er.)75 1111 y(The)f(send)h(bu\013er)e(of)h(the)g(ro)q(ot)f(pro)q(cess)h(is)h (assumed)f(con)o(tiguous)f(and)i(partitioned)f(in)o(to)g Fo(MPI)p 1739 1111 V 16 w(GSIZE)75 1167 y FE(consecutiv)o(e)d(blo)q(c)o (ks,)h(eac)o(h)e(consisting)i(of)22 b Fo(recvcnt)12 b FE(elemen)o(ts.)19 b(The)12 b Fp(i)p FE(-th)f(blo)q(c)o(k)h(is)g(sen)o (t)f(to)g(the)h(pro)q(cess)75 1224 y(with)g(rank)f(i)h(in)h(the)e (group)h(and)f(stored)h(in)g(its)g(receiv)o(e)g(bu\013er.)19 b(The)12 b(routine)g(is)g(called)h(b)o(y)e(all)i(mem)o(b)q(ers)75 1280 y(of)i(the)g(group)g(using)h(the)f(same)g(argumen)o(ts)f(for)30 b Fo(recvcnt,)16 b(recvt)o(yp)q(e,)g(ro)q(ot)p FE(,)e(and)31 b Fo(comm)p FE(.)166 1342 y(Note)15 b(that)29 b Fo(MPI)p 474 1342 V 16 w(SCA)l(TTER)17 b FE(is)e(the)h(rev)o(erse)f(op)q (eration)g(to)g Fo(MPI)p 1322 1342 V 15 w(GA)l(THER)p FE(.)75 1493 y Fq(5.4.4)49 b(All-to-all)19 b(b)o(roadcast)75 1636 y Fo(MPI)p 160 1636 V 16 w(ALLCAST\()c(sendbuf,)i(sendcnt,)g (sendt)o(yp)q(e,)h(recvbuf,)d(recvcnt,)h(recvt)o(yp)q(e,)g(comm\))117 1718 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f (send)g(bu\013er)117 1803 y(IN)171 b Fo(sendcnt)429 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in) o(teger\))117 1889 y(IN)171 b Fo(sendt)o(yp)q(e)408 b Fu(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 1974 y(OUT)124 b Fo(recvbuf)434 b Fu(address)15 b(of)f(receiv)o(e)h (bu\013er.)117 2059 y(IN)171 b Fo(recvcnt)435 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 2145 y(IN)171 b Fo(recvt)o(yp)q(e)414 b Fu(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)117 2230 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 2360 y FE(Eac)o(h)16 b(pro)q(cess)g(in)h(the)f(group)g(broadcasts)g(its)g (en)o(tire)h(send)f(bu\013er)g(to)g(all)h(pro)q(cesses)g(\(including)75 2416 y(itself)t(\);)f(All)h(send)f(bu\013ers)g(ha)o(v)o(e)f(the)h(same) g(n)o(um)o(b)q(er)g(of)f(elemen)o(ts.)23 b(Eac)o(h)15 b(pro)q(cess)h(concatenates)g(the)75 2473 y(incoming)f(messages,)f(in)h (the)f(order)g(of)g(the)g(senders')g(ranks,)g(and)g(stores)g(them)g(in) h(its)f(receiv)o(e)h(bu\013er.)75 2529 y(The)i(routine)g(is)g(called)h (b)o(y)f(all)h(mem)o(b)q(ers)e(of)h(the)f(group)h(using)g(the)g(same)f (argumen)o(ts)g(for)33 b Fo(sendcnt,)75 2586 y(sendt)o(yp)q(e)p FE(,)17 b(and)31 b Fo(comm)p FE(.)166 2647 y Fo(MPI)p 251 2647 V 16 w(ALLCAST)19 b FE(is)h(equiv)m(alen)o(t)h(to)d FD(n)i FE(executions)g(of)f Fo(MPI)p 1220 2647 V 16 w(BCAST)p FE(,)g(with)h(eac)o(h)f(pro)q(cess)h(once)75 2704 y(the)15 b(ro)q(ot.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 95 101 95 100 bop 75 -100 a Fx(5.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FE(95)75 45 y Fq(5.4.5)49 b(All-to-all)19 b(scatter-gather)75 180 y Fo(MPI)p 160 180 14 2 v 16 w(ALL)l(TO)o(ALL\(sendbuf,)d(sendcnt,) i(sendt)o(yp)q(e,)f(recvbuf,)f(recvcnt,)f(recvt)o(yp)q(e,)h(comm\))117 259 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f (send)g(bu\013er)117 336 y(IN)171 b Fo(sendcnt)429 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in) o(teger\))117 413 y(IN)171 b Fo(sendt)o(yp)q(e)408 b Fu(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)117 490 y(OUT)124 b Fo(recvbuf)434 b Fu(address)15 b(of)f(receiv)o(e)h (bu\013er.)117 568 y(IN)171 b Fo(recvcnt)435 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 645 y(IN)171 b Fo(recvt)o(yp)q(e)414 b Fu(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)117 722 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 848 y FE(Eac)o(h)j(pro)q(cess)h(in)h(the)e(group)g(sends)i(the)e Fp(i)p FE(-th)h(p)q(ortion)g(of)f(its)g(send)i(bu\013er)e(to)g(the)h (pro)q(cess)g(with)75 904 y(rank)21 b Fp(i)g FE(\(itself)h(included\).) 40 b(All)22 b(messages)f(sen)o(t)g(from)f(one)i(pro)q(cess)f(to)f (another)h(ha)o(v)o(e)g(the)g(same)75 961 y(length.)k(The)16 b(send)i(bu\013er)e(of)g(eac)o(h)h(pro)q(cess)g(is)g(partitioned)g(in)o (to)g Fo(MPI)p 1346 961 V 16 w(GSIZE)f FE(consecutiv)o(e)i(blo)q(c)o (ks,)75 1017 y(eac)o(h)13 b(consisting)g(of)25 b Fo(sendcnt)16 b FE(elemen)o(ts.)j(The)13 b Fp(i)p FE(-th)g(blo)q(c)o(k)g(is)g(sen)o (t)g(to)f(the)h Fp(i)p FE(-th)f(pro)q(cess)h(in)h(the)f(group.)75 1074 y(Eac)o(h)k(pro)q(cess)g(concatenates)f(the)h(incoming)h (messages,)e(in)i(the)f(order)f(of)h(the)f(senders')h(ranks,)g(and)75 1130 y(store)c(them)h(in)g(its)g(receiv)o(e)g(bu\013er.)20 b(The)14 b(routine)g(is)g(called)h(b)o(y)f(all)g(mem)o(b)q(ers)g(of)f (the)h(group)f(using)i(the)75 1186 y(same)g(argumen)o(ts)f(for)30 b Fo(sendcnt,)17 b(sendt)o(yp)q(e)p FE(,)h(and)30 b Fo(comm)p FE(.)166 1244 y(An)15 b(all-to-all)h(scatter-gather)d(is)i(the)g(equiv) m(alen)o(t)h(of)e FD(n)h FE(scatters)e(\(or)h FD(n)h FE(gathers\))e(executed)j(with)75 1300 y(eac)o(h)f(pro)q(cess)h(b)q (eing)g(once)g(the)f(ro)q(ot.)166 1441 y Fv(Discussion:)30 b Fu(Do)10 b(w)o(e)h(w)o(an)o(t)f(to)h(ha)o(v)o(e)f(completely)f (general)i(v)o(ersions)g(of)f(the)h(ab)q(o)o(v)o(e)g(data)f(mo)o(v)o(e) f(routines?)75 1497 y(Without)14 b(bu\013er)h(descriptors)i(this)d(is)h (nearly)f(imp)q(ossible)f(in)h(fortran)h(b)q(ecause)h(of)e(v)n(ariable) g(length)g(argumen)o(t)75 1554 y(list)j(or)h(arra)o(y)p 300 1554 13 2 v 15 w(of)p 349 1554 V 14 w(addresses)i(requiremen)o(ts.) 30 b(In)18 b(C)g(it)f(is)h(merely)f(ugly)m(.)29 b(F)m(or)17 b(example)g(the)h(general)g(Alltoall)75 1610 y(w)o(ould)d(b)q(e:)23 b Fr(MPI)p 352 1610 V 15 w(ALL)m(TO)o(ALL\(a)o(rra)o(y)p 667 1610 V 15 w(of)p 716 1610 V 15 w(sendadr,)17 b(a)o(rra)o(y)p 976 1610 V 16 w(of)p 1026 1610 V 14 w(sendcnt,)h(a)o(rra)o(y)p 1285 1610 V 15 w(of)p 1334 1610 V 15 w(sendt)o(yp)q(e,)g(a)o(rra)o(y)p 1613 1610 V 15 w(of)p 1662 1610 V 14 w(recvadr,)g(a)o(r-)75 1666 y(ra)o(y)p 129 1666 V 15 w(of)p 178 1666 V 14 w(recvcnt,)f(a)o (rra)o(y)p 429 1666 V 15 w(of)p 478 1666 V 15 w(recvt)o(yp)q(e,)f (comm\))d Fu(It)i(is)g(not)g(clear)g(if)f(w)o(e)h(w)o(ould)f(w)o(an)o (t)g(to)h(use)h(an)e(arra)o(y)p 1666 1666 V 15 w(of)p 1715 1666 V 14 w(t)o(yp)q(es)i(or)75 1723 y(a)e(more)e(complex)h(datat) o(yp)q(e)h(with)g(end)p 711 1723 V 15 w(of)p 760 1723 V 15 w(bu\013er)h(mark)o(ers)e(inserted.)75 1955 y Fy(5.5)59 b(Global)20 b(Compute)e(Op)r(erations)75 2059 y FE(The)13 b(functions)g(in)h(this)f(section)h(p)q(erform)e(one)h(of)f(the)h (follo)o(wing)g(op)q(erations)g(across)f(all)i(the)f(mem)o(b)q(ers)75 2115 y(of)i(a)g(group:)189 2213 y(global)g(max)g(on)g(in)o(teger)h(and) f(\015oating)g(p)q(oin)o(t)h(data)e(t)o(yp)q(es)189 2311 y(global)h(min)h(on)g(in)o(teger)f(and)g(\015oating)g(p)q(oin)o(t)h (data)f(t)o(yp)q(es)189 2409 y(global)g(sum)h(on)f(in)o(teger)g(and)h (\015oating)f(p)q(oin)o(t)g(data)g(t)o(yp)q(es)189 2507 y(global)g(pro)q(duct)h(on)f(in)o(teger)h(and)f(\015oating)g(p)q(oin)o (t)h(data)e(t)o(yp)q(es)189 2605 y(global)h(AND)h(on)f(logical)h(and)f (in)o(teger)h(data)e(t)o(yp)q(es)189 2704 y(global)h(OR)h(on)g(logical) g(and)f(in)o(teger)h(data)e(t)o(yp)q(es)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 96 102 96 101 bop 75 -100 a FE(96)724 b Fx(SECTION)16 b(5.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y FE(global)f(X)o(OR)h(on)f (logical)i(and)e(in)o(teger)h(data)e(t)o(yp)q(es)189 142 y(global)h(max)g(and)h(who)f(\(rank\))f(has)h(it)189 239 y(global)g(min)h(and)g(who)f(\(rank\))f(has)h(it)189 336 y(user)g(de\014ned)i(\(asso)q(ciativ)o(e\))d(op)q(eration)189 434 y(user)h(de\014ned)i(\(asso)q(ciativ)o(e)e(and)g(comm)o(utativ)o (e\))f(op)q(eration)75 560 y Fq(5.5.1)49 b(Reduce)75 695 y Fo(MPI)p 160 695 14 2 v 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d (cnt,)h(t)o(yp)q(e,)g(op,)f(ro)q(ot,)f(comm\))117 773 y Fu(IN)171 b Fo(sendbuf)428 b Fu(address)15 b(of)f(send)g(bu\013er)117 850 y(OUT)124 b Fo(recvbuf)434 b Fu(address)15 b(of)f(receiv)o(e)h (bu\013er)g({)e(signi\014can)o(t)h(only)f(at)h(ro)q(ot)117 926 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (in)h(input)g(bu\013er)g(\(in)o(teger\))117 1003 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f (input)h(bu\013er)117 1080 y(IN)171 b Fo(op)525 b Fu(op)q(eration)117 1157 y(IN)171 b Fo(ro)q(ot)492 b Fu(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 1233 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 1359 y FE(Com)o(bines)19 b(the)f(v)m(alues)i(pro)o(vided)f(in)g(the)g(send)g(bu\013er)f(of)g (eac)o(h)g(pro)q(cess)h(in)g(the)g(group,)f(using)75 1415 y(the)d(op)q(eration)30 b Fo(op)p FE(,)15 b(and)g(returns)g(the)g (com)o(bined)h(v)m(alue)h(in)f(the)f(receiv)o(e)h(bu\013er)f(of)f(the)h (pro)q(cess)h(with)75 1472 y(rank)32 b Fo(ro)q(ot)p FE(.)21 b(Eac)o(h)16 b(pro)q(cess)g(can)g(pro)o(vide)h(one)f(v)m(alue,)h(or)e (a)h(sequence)h(of)e(v)m(alues,)i(in)g(whic)o(h)g(case)f(the)75 1528 y(com)o(bine)j(op)q(eration)g(is)g(executed)g(p)q(oin)o(t-wise)h (on)e(eac)o(h)g(en)o(try)g(of)g(the)h(sequence.)30 b(F)l(or)18 b(example,)h(if)75 1585 y(the)g(op)q(eration)f(is)h Fr(MPI)p 489 1585 13 2 v 15 w(MAX)f FE(and)h(the)f(send)h(bu\013er)g(con)o (tains)g(t)o(w)o(o)e(\015oating)h(p)q(oin)o(t)h(n)o(um)o(b)q(ers,)h (then)75 1641 y(recvbuf\(1\))d(=)h(global)h(max\(sendbuf\(1\)\))d(and)i (recvbuf\(2\))g(=)g(global)g(max\(sendbuf\(2\)\).)26 b(All)19 b(send)75 1697 y(bu\013ers)d(should)i(de\014ne)g(sequences)f (of)f(equal)i(length)f(of)f(en)o(tries)h(all)g(of)f(the)h(same)f(data)g (t)o(yp)q(e,)h(where)75 1754 y(the)d(t)o(yp)q(e)g(is)h(one)f(of)f (those)h(allo)o(w)o(ed)g(for)g(op)q(erands)g(of)28 b Fo(op)p FE(.)19 b(F)l(or)14 b(all)h(op)q(erations)f(except)27 b Fr(MPI)p 1706 1754 V 14 w(MINLOC)75 1810 y FE(and)j Fr(MPI)p 255 1810 V 14 w(MAXLOC)15 b FE(the)h(n)o(um)o(b)q(er)g(and)f (t)o(yp)q(e)h(of)f(elemen)o(ts)h(in)h(the)e(send)i(bu\013er)e(are)g (the)h(same)f(as)h(for)75 1867 y(the)e(receiv)o(e)h(bu\013ers.)k(F)l (or)26 b Fr(MPI)p 635 1867 V 14 w(MINLOC)14 b FE(and)26 b Fr(MPI)p 994 1867 V 15 w(MAXLOC)p FE(,)12 b(the)i(receiv)o(e)h (bu\013er)f(will)h(con)o(tain)28 b Fo(cnt)75 1923 y FE(elemen)o(ts)20 b(of)f(the)h(same)f(t)o(yp)q(e)h(as)f(the)h(elemen)o(ts)g(in)g(the)g (input)g(bu\013er,)g(follo)o(w)o(ed)g(b)o(y)39 b Fo(cnt)21 b FE(in)o(tegers)75 1980 y(\(ranks\).)166 2037 y(The)15 b(op)q(eration)g(de\014ned)h(b)o(y)30 b Fo(op)15 b FE(is)g(asso)q (ciativ)o(e)g(and)g(comm)o(utativ)o(e,)f(and)h(the)g(implemen)o(tation) 75 2093 y(can)h(tak)o(e)f(adv)m(an)o(tage)g(of)h(asso)q(ciativit)o(y)g (and)g(comm)o(utativit)o(y)f(in)i(order)f(to)f(c)o(hange)h(order)f(of)h (ev)m(alua-)75 2150 y(tion.)23 b(The)17 b(routine)f(is)h(called)h(b)o (y)e(all)h(group)f(mem)o(b)q(ers)g(using)h(the)f(same)g(argumen)o(ts)f (for)32 b Fo(cnt,)17 b(t)o(yp)q(e,)75 2206 y(op,)e(ro)q(ot)g FE(and)30 b Fo(comm)p FE(.)166 2264 y(W)l(e)15 b(list)h(b)q(elo)o(w)g (the)f(supp)q(orted)h(options)f(for)30 b Fo(op)p FE(.)117 2367 y Fr(MPI)p 194 2367 V 14 w(MAX)649 b FE(maxim)o(um)117 2423 y Fr(MPI)p 194 2423 V 14 w(MIN)664 b FE(minim)o(um)117 2479 y Fr(MPI)p 194 2479 V 14 w(SUM)653 b FE(sum)117 2535 y Fr(MPI)p 194 2535 V 14 w(PROD)626 b FE(pro)q(duct)117 2592 y Fr(MPI)p 194 2592 V 14 w(AND)654 b FE(and)15 b(\(logical)h(or)f (bit-wise)h(in)o(teger\))117 2648 y Fr(MPI)p 194 2648 V 14 w(OR)683 b FE(or)14 b(\(logical)j(or)d(bit-wise)j(in)o(teger\))117 2704 y Fr(MPI)p 194 2704 V 14 w(X)o(OR)656 b FE(xor)14 b(\(logical)i(or)f(bit-wise)i(in)o(teger\))1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 97 103 97 102 bop 75 -100 a Fx(5.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FE(97)117 45 y Fr(MPI)p 194 45 13 2 v 14 w(MAXLOC)569 b FE(maxim)o(um)14 b(v)m(alue)h(and)f(rank)g(of)g(pro)q(cess)g(with)g (max-)905 102 y(im)o(um)h(v)m(alue)i(\(rank)d(of)g(\014rst)h(pro)q (cess)g(with)g(maxim)o(um)905 158 y(v)m(alue,)h(in)g(case)g(of)e (ties\))117 211 y Fr(MPI)p 194 211 V 14 w(MINLOC)584 b FE(minim)o(um)17 b(v)m(alue)h(and)f(rank)f(of)g(pro)q(cess)h(with)g (min-)905 267 y(im)o(um)g(v)m(alue)g(\(rank)f(of)f(\014rst)h(pro)q (cess)g(with)h(minim)o(um)905 324 y(v)m(alue,)f(in)g(case)g(of)e (ties\))166 407 y(All)19 b(op)q(erations,)f(with)g(the)f(exception)i (of)33 b Fr(MPI)p 1019 407 V 15 w(MAXLOC)16 b FE(and)34 b Fr(MPI)p 1404 407 V 15 w(MINLOC)17 b FE(return)g(a)h(v)m(alue)75 463 y(whic)o(h)11 b(has)f(the)g(same)g(datat)o(yp)q(e)g(as)f(the)i(op)q (erands.)18 b(Eac)o(h)10 b(op)q(erand)h(of)19 b Fr(MPI)p 1389 463 V 14 w(MAXLOC)9 b FE(and)20 b Fr(MPI)p 1752 463 V 14 w(MINLOC)75 520 y FE(can)15 b(b)q(e)h(though)o(t)e(as)g(a)h (pair)g Fp(\(v,)24 b(i\))p FE(:)19 b Fp(i)c FE(is)h(the)f(rank)f(of)h (the)g(calling)i(pro)q(cess)e(that)f(is)h(passed)h(implic-)75 576 y(itly)l(,)f(and)f Fp(v)f FE(is)h(the)g(v)m(alue)h(that)e(is)h (explicitly)j(passed)d(to)f(the)h(call.)38 b Fr(MPI)p 1332 576 V 15 w(MAXLOC)12 b FE(and)27 b Fr(MPI)p 1706 576 V 14 w(MINLOC)75 633 y FE(return)15 b(\(explicitly\))i(a)e(pair)h Fp(\(value,)23 b(rank\))p FE(.)166 689 y(When)33 b Fr(MPI)p 393 689 V 15 w(MINLOC)17 b FE(or)33 b Fr(MPI)p 731 689 V 14 w(MAXLOC)16 b FE(are)h(in)o(v)o(ok)o(ed,)h(the)g(input)g(bu\013er) f(should)i(con)o(tain)e FD(m)75 746 y FE(elemen)o(ts)h(of)f(the)h(same) f(t)o(yp)q(e)h(to)f(whic)o(h)h(the)g(op)q(eration)34 b Fr(MPI)p 1189 746 V 14 w(MIN)18 b FE(or)33 b Fr(MPI)p 1447 746 V 14 w(MAX)17 b FE(can)h(b)q(e)g(applied.)75 802 y(The)f(op)q(eration)f(returns)g(at)g(the)g(ro)q(ot)g FD(m)g FE(elemen)o(ts)h(of)f(the)g(same)g(t)o(yp)q(e)h(as)f(the)g (inputs,)h(follo)o(w)o(ed)g(b)o(y)75 859 y FD(m)e FE(in)o(tegers)g (\(ranks\).)k(The)d(output)f(bu\013er)g(should)h(b)q(e)g(de\014ned)g (accordingly)l(.)166 915 y(The)f(op)q(eration)h(that)e(de\014nes)30 b Fr(MPI)p 801 915 V 15 w(MAXLOC)14 b FE(is)721 967 y Fe( )774 1011 y FD(u)779 1067 y(i)821 967 y Fe(!)864 1039 y FA(\016)897 967 y Fe( )950 1011 y FD(v)952 1067 y(j)995 967 y Fe(!)1040 1039 y FE(=)1088 967 y Fe( )1142 1011 y FD(w)1146 1067 y(k)1196 967 y Fe(!)75 1163 y FE(where)833 1220 y FD(w)f FE(=)g(max\()p FD(u;)8 b(v)r FE(\))75 1299 y(and)710 1402 y FD(k)13 b FE(=)795 1303 y Fe(8)795 1341 y(>)795 1353 y(<)795 1428 y(>)795 1440 y(:)853 1345 y FD(i)194 b FE(if)15 b FD(u)e(>)g(v)853 1402 y FE(min)q(\()p FD(i;)8 b(j)s FE(\))39 b(if)15 b FD(u)e FE(=)g FD(v)853 1458 y(j)191 b FE(if)15 b FD(u)e(<)g(v)166 1536 y FE(Note)i(that)f (this)i(op)q(eration)f(is)h(asso)q(ciativ)o(e)g(and)f(comm)o(utativ)o (e.)166 1593 y(A)g(similar)i(de\014nition)g(can)e(b)q(e)h(giv)o(en)f (for)29 b Fr(MPI)p 991 1593 V 14 w(MINLOC)p FE(.)166 1725 y Fv(Discussion:)166 1775 y Fu(W)m(e)19 b(de\014ne)40 b Fr(MPI)p 465 1775 V 14 w(MINLOC)19 b Fu(to)g(return)i(a)e(v)o(ector)h (of)f(v)n(alues,)h(follo)o(w)o(ed)e(b)o(y)h(a)g(v)o(ector)h(of)f (ranks.)35 b(The)75 1825 y(alternativ)o(e)13 b(is)g(for)27 b Fr(MPI)p 477 1825 V 14 w(MINLOC)13 b Fu(to)g(return)i(a)e(v)o(ector)h (of)f(\(v)n(alue,)g(rank\))g(pairs,)g(i.e.,)f(a)h(v)o(ector)h(of)f (structures.)75 1875 y(This)18 b(second)i(c)o(hoice)f(is)g(less)g(con)o (v)o(enien)o(t)g(for)f(F)m(ortran.)32 b(Another)20 b(alternativ)o(e)e (is)g(to)h(ha)o(v)o(e)37 b Fr(MPI)p 1706 1875 V 14 w(MINLOC)75 1925 y Fu(return)15 b(t)o(w)o(o)e(output)i(bu\013ers,)f(but)g(then)h (it)e(need)i(b)q(e)g(in)o(v)o(ok)o(ed)e(di\013eren)o(tly)h(than)g(the)h (other)f(op)q(erations.)166 1974 y(The)f(computation)d(can)i(still)g(b) q(e)g(pip)q(elined,)g(pro)o(vided)g(that)g(the)h(lo)q(cation)e(of)h (the)h(\014rst)g(rank)f(en)o(try)h(in)e(the)75 2024 y(output)j (bu\013er)h(can)f(b)q(e)h(computed)e(up)h(fron)o(t.)166 2239 y Fv(Implemen)o(tati)o(on)e(note:)166 2289 y Fu(The)i(op)q (erations)g(can)g(b)q(e)g(applied)g(to)f(op)q(erands)i(of)e(di\013eren) o(t)i(t)o(yp)q(es,)f(in)f(di\013eren)o(t)i(calls:)i(e.g.,)c Fr(MPI)p 1775 2289 V 14 w(SUM)75 2339 y Fu(ma)o(y)h(require)k(an)e(in)o (teger)g(sum)g(in)g(one)g(call,)g(and)g(a)g(complex)f(sum)g(in)h (another.)26 b(Since)17 b(w)o(e)g(require)g(that)f(all)75 2389 y(elemen)o(ts)f(b)q(e)h(of)f(the)h(same)f(datat)o(yp)q(e,)g(it)h (is)f(not)g(necessary)j(to)d(store)h(a)g(full)e(signature)i(with)f(eac) o(h)h(bu\013er:)22 b(It)75 2439 y(is)14 b(only)e(necessary)k(to)e (store)g(the)h(datat)o(yp)q(e)f(of)f(the)h(elemen)o(ts)g(when)g(all)e (elemen)o(ts)i(are)g(of)f(the)h(same)f(t)o(yp)q(e,)h(and)75 2488 y(store)h(a)e(\015ag)h(indicating)f(that)g(the)i(bu\013er)g(is)f (not)f(homogeneous,)g(otherwise.)166 2704 y Fv(Missing:)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 98 104 98 103 bop 75 -100 a FE(98)724 b Fx(SECTION)16 b(5.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Fu(Need)j(to)f(de\014ne)h (the)g(t)o(yp)q(es)g(compatible)d(with)i(eac)o(h)h(op)q(eration.)30 b(This)18 b(includes)h Fr(MPI)p 1611 45 13 2 v 14 w(BYTE)f Fu(for)f(the)75 95 y(logical)12 b(op)q(erations,)i(and)g(whatev)o(er)g (F)m(ortran/C)g(allo)o(w)e(for)h(all)g(op)q(erations.)75 281 y Fo(MPI)p 160 281 14 2 v 16 w(USER)p 288 281 V 17 w(REDUCE\()j(sendbuf,)h(recvbuf,)f(cnt,)f(t)o(yp)q(e,)h(function,)h(ro) q(ot,)d(comm\))117 359 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f(send)g(bu\013er)117 433 y(OUT)124 b Fo(recvbuf)434 b Fu(starting)15 b(address)g(of)f(receiv)o (e)i(bu\013er)f({)f(signi\014can)o(t)g(only)g(at)905 490 y(ro)q(ot)117 565 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g(\(in)o(teger\))117 639 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(input)h(bu\013er)117 714 y(IN)171 b Fo(function)418 b Fu(user)15 b(de\014ned)g(function)117 789 y(IN)171 b Fo(ro)q(ot)492 b Fu(rank)14 b(of)f(ro)q(ot)h(pro)q(cess) i(\(in)o(teger\))117 864 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 988 y FE(Similar)20 b(to)f(the)g(reduce)h(op)q(eration)f (function)h(ab)q(o)o(v)o(e)f(except)g(that)g(a)f(user)i(supplied)h (function)75 1045 y(is)f(used.)64 b Fo(function)20 b FE(is)g(a)f(function)h(with)f(three)g(argumen)o(ts.)31 b(A)19 b(C)g(protot)o(yp)q(e)f(for)h(suc)o(h)g(function)75 1101 y(is)g Fp(f\()k(invec,)g(inoutvec,)g(*len\))p FE(.)k(Both)36 b Fo(invec)19 b FE(and)36 b Fo(inoutvec)19 b FE(are)f(arra)o(ys)e(with) 37 b Fo(*len)18 b FE(en)o(tries.)75 1158 y(The)h(function)g(computes)g (p)q(oin)o(t-wise)h(a)e(comm)o(utativ)o(e)g(and)h(asso)q(ciativ)o(e)g (op)q(eration)f(on)h(eac)o(h)g(pair)75 1214 y(of)e(en)o(tries)h(and)g (returns)f(the)h(result)g(in)36 b Fo(inoutvec)p FE(.)28 b(A)17 b(pseudo-co)q(de)i(for)35 b Fo(function)19 b FE(is)f(giv)o(en)g (b)q(elo)o(w,)75 1270 y(where)d Fp(op)g FE(is)h(the)f(comm)o(utativ)o (e)g(and)g(asso)q(ciativ)o(e)h(op)q(eration)f(de\014ned)i(b)o(y)30 b Fo(function)p FE(.)361 1374 y Fp(for\(i=0;)23 b(i)h(<)g(*len;)f (i++\))g({)552 1431 y(inoutvec[i])g(op=)g(invec[i])361 1487 y(})166 1591 y FE(The)16 b(t)o(yp)q(e)h(of)f(the)g(elemen)o(ts)h (of)32 b Fo(invec)17 b FE(and)g(of)32 b Fo(inoutvec)18 b FE(matc)o(h)d(the)i(t)o(yp)q(e)f(of)g(the)g(elemen)o(ts)h(of)75 1648 y(the)e(send)h(bu\013ers)f(and)h(the)f(receiv)o(e)h(bu\013er.)75 1751 y Fo(MPI)p 160 1751 V 16 w(USER)p 288 1751 V 17 w(REDUCEA\()g(sendbuf,)h(recvbuf,)f(cnt,)g(t)o(yp)q(e,)g(function,)g (ro)q(ot,)e(comm\))117 1829 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f(send)g(bu\013er)117 1903 y(OUT)124 b Fo(recvbuf)434 b Fu(starting)15 b(address)g(of)f (receiv)o(e)i(bu\013er)f({)f(signi\014can)o(t)g(only)g(at)905 1960 y(ro)q(ot)117 2035 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g(\(in)o(teger\))117 2109 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(input)h(bu\013er)117 2184 y(IN)171 b Fo(function)418 b Fu(user)15 b(de\014ned)g(function)117 2259 y(IN)171 b Fo(ro)q(ot)492 b Fu(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 2334 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 2458 y FE(Iden)o(tical)16 b(to)f Fo(MPI)p 493 2458 V 16 w(USER)p 621 2458 V 17 w(REDUCE)p FE(,)g(except)g(that)g(the)g(op)q(eration)g(de\014ned)h(b)o (y)30 b Fo(function)16 b FE(is)f(not)75 2515 y(required)j(to)e(b)q(e)h (comm)o(utativ)o(e,)f(but)g(only)h(asso)q(ciativ)o(e.)25 b(Th)o(us,)16 b(the)h(order)f(of)g(computation)h(can)f(b)q(e)75 2571 y(mo)q(di\014ed)h(only)e(using)h(asso)q(ciativit)o(y)l(.)166 2704 y Fv(Implemen)o(tati)o(on)c(note:)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 99 105 99 104 bop 75 -100 a Fx(5.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FE(99)166 45 y Fu(The)19 b(co)q(de)h(for)e Fr(MPI)p 502 45 13 2 v 14 w(USER)p 620 45 V 14 w(REDUCEA)f Fu(can)i(b)q(e)g (used)h(to)e(pro)o(vide)h(an)f(iden)o(tical)g(implemen)o(tation)d(for) 75 102 y Fr(MPI)p 152 102 V 14 w(USER)p 270 102 V 14 w(REDUCE)p Fu(.)166 353 y Fv(Discussion:)166 446 y Fu(The)c(addition)f (of)h(the)g(third)g(parameter,)22 b Fr(*len)12 b Fu(in)21 b Fr(function)12 b Fu(allo)o(w)e(the)h(system)g(to)g(a)o(v)o(oid)e (calling)21 b Fr(function)75 503 y Fu(for)15 b(eac)o(h)g(elemen)o(t)g (in)f(the)i(input)e(bu\013er;)i(rather,)g(the)g(system)e(can)h(c)o(ho)q (ose)h(to)f(apply)29 b Fr(function)16 b Fu(to)f(c)o(h)o(unks)h(of)75 559 y(inputs,)g(where)h(the)f(size)h(of)e(the)h(c)o(h)o(unk)g(is)g(c)o (hosen)h(b)o(y)e(the)i(system)e(so)h(as)g(to)g(optimize)e(comm)o (unicatio)o(n)f(and)75 616 y(computation)f(pip)q(elining.)17 b(E.g.,)27 b Fr(*len)14 b Fu(could)g(b)q(e)h(set)g(to)f(b)q(e)g(the)h (t)o(ypical)e(pac)o(k)o(et)h(size)h(in)f(the)g(comm)o(unication)75 672 y(subsystem.)166 924 y Fv(Missing:)166 1010 y Fu(The)k(last)g (appro)o(v)o(ed)g(draft)f(has)h(an)g(additional)34 b Fr(unitsize)19 b Fu(parameter)e(in)g Fr(MPI)p 1464 1010 V 15 w(USER)p 1583 1010 V 14 w(REDUCE)p Fu(:)f(Eac)o(h)75 1060 y(elemen)o(t)g(of)31 b Fr(invec)17 b Fu(or)32 b Fr(inoutvec)18 b Fu(corresp)q(onds)g(to)32 b Fr(unitsize)17 b Fu(elemen)o(ts)f(of)f(the)i(input)f(\(output\))h(bu\013er.)25 b(This)75 1110 y(allo)o(ws,)16 b(for)i(example,)e(to)h(pass)h(to)f Fr(MPI)p 739 1110 V 15 w(USER)p 858 1110 V 14 w(REDUCE)e Fu(a)j(v)o(ector)g(of)e(real)i(n)o(um)o(b)q(ers,)f(and)g(ha)o(v)o(e)35 b Fr(function)75 1160 y Fu(treat)16 b(eac)o(h)g(pair)e(of)h(real)g(n)o (um)o(b)q(ers)g(as)g(one)g(complex)f(n)o(um)o(b)q(er.)22 b(I)15 b(am)e(not)i(sure)i(w)o(e)e(need)h(this)f(if)g(w)o(e)g(appro)o (v)o(e)75 1209 y(the)g(prop)q(osal)f(to)g(use)i(bu\013er)f(t)o(yp)q (es:)20 b(It)14 b(w)o(e)h(do)f(so)h(it)f(will)f(b)q(e)i(easy)f(to)h (de\014ne)g(the)g(bu\013er)h(to)e(consists)h(of)f(pairs)75 1259 y(of)f(real)h(n)o(um)o(b)q(ers,)f(sa)o(y)m(.)166 1435 y FE(MPI)j(also)f(includes)j(v)m(arian)o(ts)e(of)f(eac)o(h)h(of)f (the)h(reduce)g(op)q(erations)g(where)g(the)g(result)g(is)g(kno)o(wn)75 1491 y(to)f(all)h(pro)q(cesses)f(in)h(the)g(group)f(on)g(return.)75 1631 y Fo(MPI)p 160 1631 14 2 v 16 w(ALLREDUCE\()g(sendbuf,)i(recvbuf,) f(cnt,)g(t)o(yp)q(e,)f(op,)g(comm\))117 1745 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f(send)g(bu\013er)117 1893 y(OUT)124 b Fo(recvbuf)434 b Fu(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)117 2042 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 2190 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 2338 y(IN)171 b Fo(op)525 b Fu(op)q(eration)117 2486 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 2647 y FE(Same)19 b(as)h(the)f Fo(MPI)p 519 2647 V 16 w(REDUCE)h FE(op)q(eration)g(function)g(except)g(that)f(the)h(result)g (app)q(ears)f(in)i(the)75 2704 y(receiv)o(e)16 b(bu\013er)f(of)g(all)h (the)f(group)g(mem)o(b)q(ers.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 100 106 100 105 bop 75 -100 a FE(100)701 b Fx(SECTION)16 b(5.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(USER)p 288 45 V 17 w(ALLREDUCE\()f(sendbuf,)j(recvbuf,)d (cnt,)h(t)o(yp)q(e,)g(function,)g(comm\))117 123 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f(send)g(bu\013er)117 198 y(OUT)124 b Fo(recvbuf)434 b Fu(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)117 274 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 350 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 426 y(IN)171 b Fo(function)418 b Fu(user)15 b(de\014ned)g(function) 117 501 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 626 y FE(Same)k(as)g(the)g Fo(MPI)p 506 626 V 16 w(USER)p 634 626 V 18 w(REDUCE)h FE(op)q(eration)f(function)h (except)g(that)f(the)g(result)h(app)q(ears)f(in)75 683 y(the)g(receiv)o(e)h(bu\013er)g(of)e(all)i(the)g(group)f(mem)o(b)q (ers.)75 787 y Fo(MPI)p 160 787 V 16 w(USER)p 288 787 V 17 w(ALLREDUCEA\()h(sendbuf,)h(recvbuf,)e(cnt,)h(t)o(yp)q(e,)g (function,)g(comm\))117 864 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f(send)g(bu\013er)117 940 y(OUT)124 b Fo(recvbuf)434 b Fu(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)117 1016 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 1091 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 1167 y(IN)171 b Fo(function)418 b Fu(user)15 b(de\014ned)g (function)117 1243 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 1367 y FE(Same)16 b(as)f Fo(MPI)p 429 1367 V 16 w(USER)p 557 1367 V 18 w(REDUCEA)p FE(,)h(except)g(that)g(the)g (result)g(app)q(ears)g(in)h(the)f(receiv)o(e)h(bu\013er)e(of)75 1424 y(all)h(the)f(group)g(mem)o(b)q(ers.)166 1557 y Fv(Implemen)o(tati)o(on)d(note:)166 1613 y Fu(The)j(allreduce)h(op)q (erations)f(can)g(b)q(e)h(implem)o(en)o(ted)d(as)i(a)g(reduce,)h(follo) o(w)o(ed)e(b)o(y)h(a)f(broadcast.)22 b(Ho)o(w)o(ev)o(er,)75 1670 y(a)14 b(direct)g(implemen)o(tation)d(can)j(lead)f(to)h(b)q(etter) i(p)q(erformance.)75 1876 y Fq(5.5.2)49 b(Scan)75 2010 y Fo(MPI)p 160 2010 V 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d(cnt,)h(t)o (yp)q(e,)g(op,)f(comm)e(\))117 2087 y Fu(IN)171 b Fo(sendbuf)428 b Fu(starting)14 b(address)h(of)f(send)g(bu\013er)117 2163 y(IN)171 b Fo(recvbuf)434 b Fu(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)117 2239 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 2314 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er) 117 2390 y(IN)171 b Fo(op)525 b Fu(op)q(eration)117 2466 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 2591 y Fo(MPI)p 251 2591 V 16 w(SCAN)23 b FE(is)g(used)h(to)e(p)q (erform)g(a)h(parallel)h(pre\014x)f(with)g(resp)q(ect)g(to)f(an)h(asso) q(ciativ)o(e)g(and)75 2647 y(comm)o(utativ)o(e)e(reduction)i(op)q (eration)g(on)f(data)f(distributed)j(across)d(the)h(group.)40 b(The)23 b(op)q(eration)75 2704 y(returns)17 b(in)h(the)f(receiv)o(e)i (bu\013er)e(of)g(the)g(pro)q(cess)g(with)h(rank)f Fp(i)g FE(the)g(reduction)h(of)f(the)g(v)m(alues)i(in)f(the)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 101 107 101 106 bop 75 -100 a Fx(5.6.)29 b(CORRECTNESS)1285 b FE(101)75 45 y(send)22 b(bu\013ers)f(of)f(pro)q(cesses)i(with)g(ranks)e Fp(0,...,i)p FE(.)37 b(The)21 b(t)o(yp)q(e)h(of)e(op)q(erations)i(supp) q(orted,)g(their)75 102 y(seman)o(tics,)15 b(and)g(the)h(constrain)o (ts)e(on)h(send)h(and)g(receiv)o(e)g(bu\013ers)f(are)g(as)g(for)f Fo(MPI)p 1531 102 14 2 v 16 w(REDUCE)p FE(.)75 205 y Fo(MPI)p 160 205 V 16 w(USER)p 288 205 V 17 w(SCAN\()i(sendbuf,)h (recvbuf,)f(cnt,)f(t)o(yp)q(e,)h(function,)h(comm\))117 282 y Fu(IN)171 b Fo(sendbuf)428 b Fu(address)15 b(of)f(input)f (bu\013er)117 357 y(OUT)124 b Fo(recvbuf)434 b Fu(address)15 b(of)f(output)g(bu\013er)117 432 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input)h(and)g(output) g(bu\013er)h(\(in)o(te-)905 489 y(ger\))117 564 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)117 639 y(IN)171 b Fo(function)418 b Fu(user)15 b(pro)o(vided)f(function) 117 714 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 838 y FE(Same)h(as)g(the)h Fo(MPI)p 498 838 V 15 w(SCAN)g FE(op)q(eration)g(function)g(except)g(that)f(a)g (user)g(supplied)j(function)e(is)g(used.)75 894 y Fo(function)k FE(is)f(an)g(asso)q(ciativ)o(e)f(and)h(comm)o(utativ)o(e)f(function)h (with)g(an)g(input)g(v)o(ector,)f(an)g(inout)h(v)o(ector,)75 951 y(and)g(a)g(length)g(argumen)o(t.)21 b(The)16 b(t)o(yp)q(es)g(of)g (the)g(t)o(w)o(o)e(v)o(ectors)h(and)h(of)g(the)g(returned)g(v)m(alues)h (all)g(agree.)75 1007 y(See)f Fo(MPI)p 241 1007 V 16 w(USER)p 369 1007 V 17 w(REDUCE)g FE(for)e(more)h(details.)75 1111 y Fo(MPI)p 160 1111 V 16 w(USER)p 288 1111 V 17 w(SCANA\()h(sendbuf,)h(recvbuf,)f(cnt,)g(t)o(yp)q(e,)g(function,)g (comm\))117 1188 y Fu(IN)171 b Fo(sendbuf)428 b Fu(address)15 b(of)f(input)f(bu\013er)117 1263 y(OUT)124 b Fo(recvbuf)434 b Fu(address)15 b(of)f(output)g(bu\013er)117 1338 y(IN)171 b Fo(cnt)512 b Fu(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input)h (and)g(output)g(bu\013er)h(\(in)o(te-)905 1395 y(ger\))117 1470 y(IN)171 b Fo(t)o(yp)q(e)491 b Fu(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)117 1545 y(IN)171 b Fo(function)418 b Fu(user)15 b(de\014ned)g(function)117 1619 y(IN)171 b Fo(comm)454 b Fu(comm)o(unicator)11 b(handle)166 1744 y FE(Same)i(as)g Fo(MPI)p 424 1744 V 16 w(USER)p 552 1744 V 17 w(SCAN)p FE(,)h(except)f(that)g(the)g(user-de\014ned)i(op)q(eration)e(need)h (not)f(b)q(e)h(comm)o(u-)75 1800 y(tativ)o(e.)166 1933 y Fv(Implemen)o(tati)o(on)e(note:)166 1989 y Fr(MPI)p 243 1989 13 2 v 14 w(USER)p 361 1989 V 14 w(SCAN)i Fu(can)g(b)q(e)g (implemen)o(ted)e(as)i Fr(MPI)p 1000 1989 V 14 w(USER)p 1118 1989 V 15 w(SCANA)p Fu(.)75 2215 y Fy(5.6)59 b(Co)n(rrectness)75 2316 y FE(A)13 b(correct)f(program)g(should)i(in)o(v)o(ok)o(e)f (collectiv)o(e)h(comm)o(unications)g(so)e(that)g(deadlo)q(c)o(k)i(will) g(not)f(o)q(ccur,)75 2373 y(whether)22 b(collectiv)o(e)h(comm)o (unication)f(is)g(sync)o(hronizing)h(or)d(not.)38 b(The)22 b(follo)o(wing)g(t)o(w)o(o)e(examples)75 2429 y(illustrate)c(dangerous) f(use)h(of)f(collectiv)o(e)i(routines.)j(The)15 b(\014rst)g(example)h (is)g(erroneous.)75 2534 y Fp(/*)24 b(Example)e(A)i(*/)75 2591 y(switch\(MPI_rank\(comm,rank\))o(;)d(rank\))147 2647 y({)147 2704 y(case)i(0:)g({)h(MPI_bcast\(var1,)e(cnt,)h(type,)h (0,)f(comm\);)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 102 108 102 107 bop 75 -100 a FE(102)706 b Fx(SECTION)16 b(5.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)385 45 y Fp(MPI_send\(var2,)22 b(cnt,)h(type,)h(1,)f(tag,)h(comm\);)385 102 y(break;)337 158 y(})147 214 y(case)f(1:)g({)h(MPI_recv\(var2,)e(cnt,)h(type,)h(0,)f (tag,)h(comm\);)385 271 y(MPI_bcast\(var1,)e(cnt,)h(type,)h(0,)f (comm\);)385 327 y(break;)337 384 y(})147 440 y(})166 552 y FE(Pro)q(cess)16 b(zero)f(executes)h(a)g(broadcast,)f(follo)o(w)o (ed)h(b)o(y)f(a)h(blo)q(c)o(king)h(send)f(op)q(eration;)g(pro)q(cess)g (one)75 609 y(\014rst)h(executes)h(a)g(matc)o(hing)f(blo)q(c)o(king)i (receiv)o(e,)g(follo)o(w)o(ed)f(b)o(y)g(the)f(matc)o(hing)h(broadcast)f (call.)28 b(This)75 665 y(program)18 b(ma)o(y)g(deadlo)q(c)o(k.)33 b(The)20 b(broadcast)e(call)i(on)f(pro)q(cess)h(zero)f(ma)o(y)f(blo)q (c)o(k)i(un)o(til)g(pro)q(cess)g(one)75 722 y(executes)c(the)f(matc)o (hing)g(broadcast)g(call,)h(so)f(that)f(the)i(send)f(is)h(not)f (executed.)21 b(Pro)q(cess)15 b(one)g(blo)q(c)o(ks)75 778 y(on)g(the)g(receiv)o(e)i(and)e(nev)o(er)g(executes)h(the)f (broadcast.)166 836 y(The)g(follo)o(wing)h(example)g(is)g(correct,)e (but)i(nondeterministic:)75 949 y Fp(/*)24 b(Example)e(B)i(*/)75 1005 y(switch\(MPI_rank\(comm,rank\))o(;)d(rank\))147 1062 y({)170 1118 y(case)j(0:)f({)h(MPI_bcast\(var1,)e(cnt,)h(type,)g (0,)h(comm\);)409 1175 y(MPI_send\(var2,)e(cnt,)h(type,)h(1,)f(tag,)g (comm\);)409 1231 y(break;)361 1288 y(})170 1344 y(case)h(1:)f({)h (MPI_recv\(var2,)e(cnt,)h(type,)h(MPI_SOURCE_ANY,)d(tag,)j(comm\);)409 1401 y(MPI_bcast\(var1,)e(cnt,)h(type,)g(0,)h(comm\);)409 1457 y(MPI_recv\(var2,)e(cnt,)h(type,)h(MPI_SOURCE_ANY,)d(tag,)j (comm\);)409 1514 y(break;)361 1570 y(})170 1626 y(case)g(2:)f({)h (MPI_send\(var2,)e(cnt,)h(type,)h(1,)f(tag,)g(comm\);)409 1683 y(MPI_bcast\(var1,)f(cnt,)h(type,)g(0,)h(comm\);)409 1739 y(break;)361 1796 y(})170 1852 y(})166 1964 y FE(All)17 b(three)e(pro)q(cesses)h(participate)g(in)g(a)f(broadcast.)20 b(Pro)q(cess)15 b(0)g(sends)h(a)f(message)g(to)g(pro)q(cess)g(1)75 2021 y(after)d(the)h(broadcast,)f(and)g(pro)q(cess)h(2)g(sends)g(a)f (message)g(to)g(pro)q(cess)h(1)f(after)g(the)h(broadcast.)18 b(Pro)q(cess)75 2077 y(1)d(receiv)o(es)h(b)q(efore)f(and)h(after)e(the) i(broadcast,)e(with)h(a)g(wildcard)h(source)g(parameter.)166 2135 y(Tw)o(o)e(p)q(ossible)i(executions,)g(with)f(di\013eren)o(t)g (matc)o(hings)g(of)f(sends)h(and)g(receiv)o(es)h(are)f(illustrated)75 2191 y(b)q(elo)o(w.)337 2305 y Fp(First)24 b(Execution)170 2418 y(0)311 b(1)357 b(2)648 2474 y(/-----)47 b(send)457 2530 y(recv)23 b(<-/)75 2587 y(broadcast)118 b(broadcast)166 b(broadcast)123 2643 y(send)23 b(---\\)337 2700 y(\\-->)h(recv)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 103 109 103 108 bop 75 -100 a Fx(5.6.)34 b(CORRECTNESS)1280 b FE(103)337 45 y Fp(Second)24 b(Execution)147 158 y(0)334 b(1)357 b(2)75 214 y(broadcast)123 271 y(send)23 b(---\\)337 327 y(\\-->)48 b(recv)433 384 y(broadcast)166 b(broadcast)719 440 y(/---)47 b(send)481 497 y(recv)23 b(<---/)166 610 y FE(Note)16 b(that)g(the)h(second)g(execution)h(has)f(the)f(p)q (eculiar)j(e\013ect)e(that)f(a)g(send)h(executed)h(after)e(the)75 666 y(broadcast)e(is)i(receiv)o(ed)h(at)d(another)h(no)q(de)h(b)q (efore)f(the)g(broadcast.)166 800 y Fv(Discussion:)166 852 y Fu(An)d(alternativ)o(e)g(design)h(is)f(to)g(require)h(that)g(all) e(collectiv)o(e)h(comm)o(unication)d(calls)j(are)g(sync)o(hronizing.)18 b(In)75 901 y(this)11 b(case,)h(the)g(second)g(program)d(is)i (deterministic)g(and)f(only)h(the)g(\014rst)h(execution)g(ma)o(y)d(o)q (ccur.)18 b(This)11 b(will)e(mak)o(e)75 951 y(a)14 b(di\013erence)h (only)e(for)h(collectiv)o(e)g(op)q(erations)g(where)h(not)e(all)g(pro)q (cesses)j(b)q(oth)e(send)h(and)f(receiv)o(e)h(\(broadcast,)75 1001 y(reduce,)g(scatter,)g(gather\).)166 1142 y FE(It)21 b(is)g(the)g(user's)f(resp)q(onsibilit)o(y)j(to)d(mak)o(e)g(sure)h (that)f(there)h(are)f(no)h(t)o(w)o(o)e(concurren)o(tly)i(exe-)75 1198 y(cuting)g(collectiv)o(e)i(calls)e(that)f(use)h(the)g(same)f(comm) o(unicator)g(on)h(the)g(same)f(pro)q(cess.)36 b(\(Since)22 b(all)75 1254 y(collectiv)o(e)d(comm)o(unication)f(calls)h(are)e(blo)q (c)o(king)i(this)e(restriction)h(only)g(a\013ects)f(m)o(ultithreaded)h (im-)75 1311 y(plemen)o(tations.\))i(On)14 b(the)g(other)g(hand,)g(it)g (is)g(legitimate)h(for)e(one)h(pro)q(cess)h(to)e(start)g(a)g(new)h (collectiv)o(e)75 1367 y(comm)o(unication)j(call)g(ev)o(en)f(though)g (a)f(previous)i(call)g(that)e(uses)h(the)g(same)g(comm)o(unicator)g (has)f(not)75 1424 y(y)o(et)g(terminated)g(on)g(another)g(pro)q(cess.) 20 b(As)15 b(illustrated)i(in)f(the)f(follo)o(wing)h(example:)75 1538 y Fp(/*)24 b(Example)e(C)i(*/)99 1595 y(MPI_bcast\(var1,)e(cnt,)h (type,)g(0,)h(comm\);)99 1651 y(MPI_bcast\(var2,)e(cnt,)h(type,)g(1,)h (comm\);)166 1765 y FE(In)17 b(a)e(nonsync)o(hronizing)j(implemen)o (tation)g(of)d(broadcast,)g(pro)q(cess)i(zero)f(ma)o(y)f(start)g (executing)75 1821 y(the)h(second)g(broadcast)f(b)q(efore)h(pro)q(cess) g(one)g(terminated)g(the)g(\014rst)f(broadcast.)21 b(Both)15 b(pro)q(cess)h(zero)75 1877 y(and)h(one)f(ma)o(y)g(terminate)g(their)h (t)o(w)o(o)e(broadcast)g(calls)j(b)q(efore)e(other)g(pro)q(cesses)h(ha) o(v)o(e)f(started)f(their)75 1934 y(calls.)21 b(It)15 b(is)h(the)f(implemen)o(tor's)h(resp)q(onsibilit)o(y)h(to)e(ensure)h (this)f(will)i(not)e(cause)h(an)o(y)e(error.)166 2068 y Fv(Implemen)o(tati)o(on)e(note:)166 2119 y Fu(Assume)e(that)g (broadcast)h(is)f(implemen)o(ted)e(using)i(p)q(oin)o(t-to-p)q(oin)o(t)e (MPI)j(comm)o(unicati)o(on.)j(The)d(follo)o(wing)75 2169 y(t)o(w)o(o)i(rules)i(are)f(satis\014ed:)134 2248 y(1.)22 b(All)13 b(receiv)o(es)i(sp)q(ecify)g(their)f(source)h(explicitly)e (\(no)h(wildcards\).)134 2328 y(2.)22 b(Eac)o(h)12 b(pro)q(cess)i (sends)g(all)d(messages)h(that)h(p)q(ertain)f(to)h(one)f(collectiv)o(e) g(call)g(b)q(efore)h(sending)f(an)o(y)g(message)189 2378 y(that)i(p)q(ertain)g(to)g(a)f(subsequen)o(t)j(collectiv)o(e)e(call.) 166 2457 y(Then)i(messages)g(b)q(elonging)f(to)h(successiv)o(e)i (broadcasts)f(cannot)f(b)q(e)h(confused,)g(as)f(the)h(order)f(of)g(p)q (oin)o(t-)75 2507 y(to-p)q(oin)o(t)d(messages)h(is)g(preserv)o(ed.)20 b(This)14 b(is)f(true,)i(in)e(general,)h(for)f(an)o(y)h(collectiv)o(e)g (library)m(.)166 2647 y FE(A)k(collectiv)o(e)h(comm)o(unication)f(ma)o (y)f(execute)i(in)f(a)g(con)o(text)f(while)i(p)q(oin)o(t-to-p)q(oin)o (t)f(comm)o(uni-)75 2704 y(cations)g(that)g(use)g(the)h(same)f(con)o (text)f(are)h(p)q(ending,)j(or)c(o)q(ccur)i(concurren)o(tly)l(.)30 b(This)19 b(is)g(illustrated)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 104 110 104 109 bop 75 -100 a FE(104)701 b Fx(SECTION)16 b(5.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FE(in)g(example)g(B)g(ab)q (o)o(v)o(e,)f(the)g(\014rst)g(pro)q(cess)h(ma)o(y)f(receiv)o(e)h(a)f (message)g(sen)o(t)g(with)h(the)f(con)o(text)g(of)g(com-)75 102 y(m)o(unicator)32 b Fo(comm)14 b FE(while)j(it)f(is)g(executing)h (a)f(broadcast)f(with)h(the)g(same)g(comm)o(unicator.)22 b(It)15 b(is)i(the)75 158 y(implemen)o(ter)f(resp)q(onsibili)q(t)o(y)h (to)e(ensure)h(this)f(will)i(not)e(cause)g(an)o(y)g(confusion.)166 291 y Fv(Implemen)o(tati)o(on)21 b(note:)65 b Fu(Assume)22 b(that)f(collectiv)o(e)g(comm)o(unications)d(are)k(implemen)o(ted)d (using)75 340 y(p)q(oin)o(t-to-p)q(oin)o(t)e(MPI)i(comm)o(unication.)29 b(Then,)20 b(in)e(order)i(to)e(a)o(v)o(oid)g(confusion,)h(whenev)o(er)h (a)e(comm)o(unica-)75 390 y(tor)g(is)f(created,)i(a)e(\\hidden)h(comm)o (unicator")c(need)19 b(b)q(e)f(created)h(for)e(collectiv)o(e)g(comm)o (unication.)26 b(A)17 b(direct)75 440 y(implemen)o(tatio)o(n)11 b(of)i(MPI)h(collectiv)o(e)g(comm)o(unicatio)o(n)d(can)j(ac)o(hiev)o(e) g(a)g(similar)d(e\013ect)k(more)e(c)o(heaply)m(,)g(e.g.,)f(b)o(y)75 490 y(using)j(a)g(hidden)h(tag)f(or)g(con)o(text)h(bit)g(to)f(indicate) g(whether)i(the)f(comm)o(unicator)d(is)i(used)h(for)f(p)q(oin)o(t-to-p) q(oin)o(t)75 540 y(or)f(collectiv)o(e)g(comm)o(unicatio)o(n.)166 589 y(An)g(alternativ)o(e)h(c)o(hoice)g(is)f(to)g(require)h(that)g(a)f (comm)o(unicator)e(is)i(quiescen)o(t)i(when)f(used)g(in)f(a)g (collectiv)o(e)75 639 y(comm)o(unication:)h(No)f(messages)h(using)f(a)f (con)o(text)i(can)g(b)q(e)g(p)q(ending)f(at)g(a)g(pro)q(cess)i(when)f (this)f(pro)q(cess)i(starts)75 689 y(executing)e(a)f(collectiv)o(e)h (comm)o(unicati)o(on)d(with)i(this)g(con)o(text,)h(nor)f(can)h(an)o(y)f (new)h(message)f(with)g(this)g(con)o(text)75 739 y(arriv)o(e)j(during)h (the)g(execution)g(of)f(this)g(collectiv)o(e)h(comm)o(unicatio)o(n,)d (unless)j(they)g(w)o(ere)h(sen)o(t)f(as)g(part)f(of)g(the)75 789 y(execution)f(of)e(the)h(collectiv)o(e)g(call)f(itself.)166 838 y(This)j(approac)o(h)h(has)f(the)h(adv)n(an)o(tage)f(of)f (simplifying)e(the)k(la)o(y)o(ering)f(of)f(collectiv)o(e)i(comm)o (unicatio)o(ns)d(on)75 888 y(top)d(of)f(p)q(oin)o(t-to-p)q(oin)o(t)f (comm)o(unicatio)o(n)f(\(no)j(need)g(for)g(hidden)f(con)o(texts\).)19 b(Also,)10 b(it)g(imp)q(oses)g(on)h(the)g(collectiv)o(e)75 938 y(collectiv)o(e)k(comm)o(unicatio)o(n)d(library)i(the)i(same)d (restrictions)j(that)f(hold)f(for)h(an)o(y)f(other)i(collectiv)o(e)e (library)m(.)20 b(It)75 988 y(has)14 b(the)h(disadv)n(an)o(tage)e(of)g (restricting)i(the)f(use)h(of)e(collectiv)o(e)h(comm)o(unicatio)o(ns.) 166 1038 y(The)i(question)h(is)f(whether)h(w)o(e)g(w)o(an)o(t)e(to)h (view)g(the)h(collectiv)o(e)f(comm)o(unication)d(op)q(erations)j(as)g (part)h(of)75 1088 y(the)f(basic)g(comm)o(unicati)o(on)d(services)k(of) e(MPI,)g(or)g(whether)i(w)o(e)f(w)o(an)o(t)f(to)g(see)i(them)d(as)i(a)f (library)g(la)o(y)o(ered)g(on)75 1137 y(top)f(of)f(these)i(basic)f (services.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 105 111 105 110 bop 75 391 a Fz(Section)35 b(6)75 633 y FG(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)75 908 y Fy(6.1)59 b(Intro)r(duction)75 1076 y FE(A)14 b(pro)q(cess)f(group)g(in)i(MPI)e(is)h(a)f(collection)j (of)d(n)h(pro)q(cesses.)19 b(Eac)o(h)14 b(pro)q(cess)f(in)i(the)e (group)g(is)h(assigned)75 1133 y(a)g(rank)g(b)q(et)o(w)o(een)h(0)g(and) f(n-1.)20 b(In)15 b(the)g(ma)s(jorit)o(y)e(of)h(parallel)i (applications)g(this)f(linear)h(ordering)f(do)q(es)75 1189 y(not)e(re\015ect)g(the)g(comm)o(unication)h(structure)e(of)h(the) g(pro)q(cesses)g(whic)o(h)h(is)f(usually)i(determined)f(b)o(y)f(the)75 1246 y(underlying)20 b(problem)e(geometry)f(and)h(the)f(n)o(umerical)i (algorithm)f(used.)28 b(Often)18 b(the)f(pro)q(cesses)h(are)75 1302 y(arranged)d(in)h(top)q(ological)g(patterns)e(as)h(simple)i(as)e (a)g(t)o(w)o(o{)e(or)i(three{dimensional)i(grid.)166 1393 y(F)l(or)12 b(the)h(user)g(who)g(programs)f(an)h(application)h(on) f(suc)o(h)g(a)g(pro)q(cess)g(structure)g(the)g(most)f(natural)75 1449 y(w)o(a)o(y)k(of)g(referring)i(to)e(individual)k(pro)q(cesses)d (is)h(to)e(sp)q(ecify)i(their)g(co)q(ordinates)f(in)h(the)f(pro)q(cess) g(grid.)75 1506 y(Usually)f(a)e(pro)q(cess)g(do)q(es)h(not)f(ha)o(v)o (e)g(to)g(kno)o(w)g(the)g(iden)o(ti\014ers)i(of)e Ft(al)r(l)g FE(pro)q(cesses)h(in)g(the)f(grid,)h(b)q(ecause)75 1562 y(comm)o(unication)e(only)f(tak)o(es)g(place)g(with)h(pro)q(cesses)f (in)h(a)f(certain)g(neigh)o(b)q(orho)q(o)q(d,)i(and)e(p)q(erhaps)h (with)75 1619 y(a)18 b(few)g(distan)o(t)f(pro)q(cesses.)29 b(Non{scalable)19 b(features)f(lik)o(e)h(the)f(explicit)i(listing)f(of) f(all)h(participating)75 1675 y(pro)q(cesses)f(in)g(the)f(application)i (could,)f(therefore,)f(b)q(e)h(a)o(v)o(oided,)f(if)h(there)f(w)o(ere)g (a)g(means)g(to)g(sp)q(ecify)75 1732 y(the)e(information)h(whic)o(h)g (is)f(really)i(required.)166 1822 y(Additionally)l(,)g(the)e (user{de\014ned)h(pro)q(cess)f(structure)g(can)g(b)q(e)h(exploited)g(b) o(y)f(the)g(system)f(in)i(the)75 1879 y(assignmen)o(t)d(of)g(pro)q (cesses)h(to)f(hardw)o(are)f(pro)q(cessors,)h(if)h(this)g(helps)h(to)d (impro)o(v)o(e)i(the)f(comm)o(unication)75 1935 y(p)q(erformance)g(on)h (a)f(giv)o(en)h(mac)o(hine.)20 b(Ho)o(w)12 b(this)i(mapping)g(is)g (done)g(in)g(detail)g(is)g(outside)g(the)g(scop)q(e)f(of)75 1992 y(this)k(standardization)h(prop)q(osal,)f(since)h(it)f(is)h(to)q (o)e(m)o(uc)o(h)h(dep)q(enden)o(t)i(on)d(the)h(particular)h(hardw)o (are.)75 2048 y(The)d(description)i(of)d(a)h(\\virtual)g(pro)q(cess)g (top)q(ology",)f(ho)o(w)o(ev)o(er,)g(only)h(dep)q(ends)i(on)e(the)g (application,)75 2105 y(and)k(is,)g(therefore,)g(mac)o(hine{indep)q (endent.)32 b(The)19 b(functions)g(whic)o(h)h(are)e(prop)q(osed)h(in)g (this)g(pap)q(er)75 2161 y(only)d(deal)g(with)f(this)h(side)g(of)f(the) g(mapping)h(problem.)166 2252 y(There)e(are)g(w)o(ell{kno)o(wn)h(tec)o (hniques)g(for)f(mapping)h(grid/torus)e(structures)h(to)g(hardw)o(are)f (top)q(o-)75 2308 y(logies)18 b(suc)o(h)f(as)g(h)o(yp)q(ercub)q(es)h (or)f(grids.)26 b(But)17 b(ev)o(en)g(for)g(more)f(complicated)j(graph)d (structures)h(go)q(o)q(d)75 2365 y(heuristics)i(often)e(yield)i (near{optimal)e(results)h([13)o(].)26 b(On)18 b(the)f(other)g(hand,)h (if)g(there)f(is)h(no)f(w)o(a)o(y)g(for)75 2421 y(the)c(user)g(to)f(sp) q(ecify)j(the)e(logical)h(pro)q(cess)f(arrangemen)o(t)f(as)g(a)h (\\virtual)g(top)q(ology",)f(this)i(information)75 2478 y(is)k(most)f(di\016cult)j(to)d(reco)o(v)o(er)g(later.)28 b(A)18 b(random)g(mapping)g(is)g(most)g(lik)o(ely)h(the)f(consequence.) 29 b(On)75 2534 y(some)15 b(mac)o(hines)h(this)g(will)h(lead)f(to)f (unnecessary)h(con)o(ten)o(tion)g(in)g(the)f(in)o(terconnect)i(net)o(w) o(ork.)i(Some)75 2591 y(details)g(ab)q(out)e(predicted)i(and)f (measured)g(p)q(erformance)f(impro)o(v)o(emen)o(ts)h(whic)o(h)g(result) g(from)f(go)q(o)q(d)75 2647 y(pro)q(cess{to{pro)q(cessor)e(mapping)h (on)f(mo)q(dern)h(w)o(ormhole{routing)f(arc)o(hitectures)h(can)g(b)q(e) g(found)g(in)75 2704 y([5)o(,)f(4].)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 106 112 106 111 bop 75 -100 a FE(106)903 b Fx(SECTION)16 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fy(6.2)59 b(Virtual)20 b(T)-5 b(op)r(ologies)75 148 y FE(In)17 b(the)g(most)e(general)i(case)g (the)f(connection)i(structure)e(of)g(a)g(set)h(of)f(pro)q(cesses)h(can) f(b)q(e)h(represen)o(ted)75 205 y(b)o(y)c(a)g(graph.)19 b(The)14 b(no)q(des)f(stand)h(for)e(the)i(pro)q(cesses,)f(and)h(the)f (edges)h(connect)f(pro)q(cesses)h(whic)o(h)g(com-)75 261 y(m)o(unicate)h(with)g(eac)o(h)g(other.)20 b(MPI)14 b(pro)o(vides)i(message{passing)e(b)q(et)o(w)o(een)h(an)o(y)g(pair)g (of)f(pro)q(cesses)h(in)75 318 y(a)i(group.)27 b(There)18 b(is)g(no)g(requiremen)o(t)g(for)f(op)q(ening)i(a)e(c)o(hannel)i (explicitly)l(.)30 b(Therefore,)18 b(a)f(\\missing)75 374 y(link")g(in)f(the)f(user{de\014ned)i(pro)q(cess)f(graph)f(do)q(es) h(not)f(prev)o(en)o(t)g(the)g(corresp)q(onding)i(pro)q(cesses)f(from)75 431 y(exc)o(hanging)h(messages.)k(It)16 b(only)g(means)g(that)f(this)i (connection)g(is)f(neglected)h(in)g(the)f(virtual)h(top)q(ol-)75 487 y(ogy)l(.)24 b(As)16 b(a)g(consequence,)i(an)f(automatic)f(mapping) h(to)q(ol)g(will)h(not)e(try)g(to)g(assign)h(the)g(pro)q(cesses)g(to)75 544 y(neigh)o(b)q(oring)g(pro)q(cessors.)166 601 y(The)i(graph)g (de\014nition)i(function)g Fo(MPI)p 870 601 14 2 v 15 w(GRAPH)g FE(presen)o(ted)f(in)g(this)f(pap)q(er)h(do)q(es)g(not)f (include)75 657 y(an)o(y)c(w)o(eigh)o(ting)g(of)f(the)h(comm)o (unication)h(v)o(olume)f(along)g(the)g(di\013eren)o(t)g(edges.)20 b(Pro)q(cesses)15 b(are)f(either)75 714 y(neigh)o(b)q(ors)22 b(or)e(not.)37 b(Exp)q(erience)23 b(with)f(similar)g(tec)o(hniques)g (in)g(P)l(ARMA)o(CS)f([10)o(,)g(11])f(sho)o(w)h(that)75 770 y(this)h(information)g(is)g(usually)h(su\016cien)o(t)g(for)e(a)g (go)q(o)q(d)h(mapping.)40 b(Additionally)l(,)25 b(a)d(more)f(precise)75 827 y(sp)q(eci\014cation)16 b(w)o(ould)f(b)q(e)g(more)e(di\016cult)j (for)e(the)g(user)g(to)g(set)g(up,)g(and)h(it)f(w)o(ould)h(mak)o(e)e (the)i(in)o(terface)75 883 y(functions)h(more)f(complicated.)166 941 y(In)i(principle)i(a)c(function)i(for)f(the)g(de\014nition)i(of)e (suc)o(h)g(a)g(graph)g(structure)g(w)o(ould)g(b)q(e)h(su\016cien)o(t)75 997 y(for)c(all)i(applications.)21 b(Ho)o(w)o(ev)o(er,)13 b(in)h(man)o(y)g(applications)h(the)f(graph)g(structure)f(is)h(v)o(ery) g(regular,)g(and)75 1054 y(the)e(detailed)i(set{up)f(of)e(the)i(graph)f (w)o(ould)h(b)q(e)g(incon)o(v)o(enien)o(t)g(for)f(the)g(user)h(and)f (resource{consuming)75 1110 y(during)g(run)g(time.)19 b(A)12 b(large)f(fraction)g(of)g(all)i(parallel)g(applications)g(use)f (pro)q(cess)g(top)q(ologies)f(lik)o(e)i(rings,)75 1167 y(or)g(t)o(w)o(o{)f(or)h(higher{dimensional)k(grids)c(or)h(tori.)19 b(These)14 b(structures)f(are)g(completely)i(de\014ned)g(b)o(y)f(the)75 1223 y(n)o(um)o(b)q(er)g(of)g(dimensions)i(and)e(the)g(n)o(um)o(b)q (ers)h(of)f(pro)q(cesses)g(in)h(eac)o(h)f(co)q(ordinate)h(direction.)21 b(Also,)14 b(the)75 1279 y(mapping)h(of)e(grids)h(and)h(tori)e(is)i (generally)g(an)f(easier)g(problem)h(then)f(that)f(of)h(general)g (graphs.)19 b(Th)o(us,)75 1336 y(it)14 b(is)h(desirable)h(to)d(do)h (the)h(mapping)g(as)e(grids)i(and)f(tori)g(rather)g(than)g(obscuring)h (the)f(information)g(in)75 1392 y(a)h(graph.)166 1450 y(Therefore,)j(in)g(addition)h(to)f(the)f(general)i(graph)e(Cartesian)h (top)q(ologies)g(are)f(supp)q(orted.)29 b(The)75 1506 y(top)q(ology)11 b(de\014nition)j(function)e Fo(MPI)p 716 1506 V 16 w(CART)g FE(can)g(b)q(e)g(used)g(to)f(describ)q(e)j (Cartesian)d(structures)g(of)g(arbi-)75 1563 y(trary)i(dimension.)22 b(F)l(or)13 b(eac)o(h)i(co)q(ordinate)f(direction)i(it)f(is)f(sp)q (eci\014ed)j(whether)d(the)h(pro)q(cess)f(structure)75 1619 y(is)h(p)q(erio)q(dic)h(or)e(not.)19 b(Similar)d(functions)f(are)f (con)o(tained)h(in)g(EXPRESS)g([14)o(])f(and)g(P)l(ARMA)o(CS.)g(Note)75 1676 y(that)e(an)h(n{dimensional)h(h)o(yp)q(ercub)q(e)h(is)e(an)g (n{dimensional)i(torus)d(with)h(2)f(pro)q(cesses)h(p)q(er)h(co)q (ordinate)75 1732 y(direction.)21 b(Th)o(us,)15 b(sp)q(ecial)i(supp)q (ort)e(for)g(h)o(yp)q(ercub)q(e)i(structures)e(is)g(not)g(necessary)l (.)166 1789 y(Based)f(on)g(the)f(p)q(ositions)i(in)g(the)e(virtual)i (top)q(ology)e(a)h(pro)q(cess)g(can)g(lo)q(ok)g(up)g(the)g(ranks)f(of)g (other)75 1846 y(pro)q(cesses)f(b)o(y)g(calling)i Fo(MPI)p 554 1846 V 16 w(INQRANK)p FE(.)e(The)g(pro)q(cesses)h(are)e(sp)q (eci\014ed)j(b)o(y)e(their)h(rectangular)f(pro)q(cess)75 1902 y(co)q(ordinates.)20 b(Th)o(us,)15 b(it)g(is)h(p)q(ossible)h(to)d (inquire)j(for)e(the)g(ranks)g(of)f(only)i(those)f(\(few\))f(pro)q (cesses)i(with)75 1959 y(whic)o(h)e(messages)g(ha)o(v)o(e)f(to)g(b)q(e) h(exc)o(hanged.)20 b(The)14 b(in)o(v)o(erse)g(function)g Fo(MPI)p 1351 1959 V 16 w(INQLOC)g FE(translates)f(a)g(rank)75 2015 y(in)o(to)i(the)g(corresp)q(onding)i(Cartesian)e(co)q(ordinates.) 75 2164 y Fy(6.3)59 b(Emb)r(edding)18 b(in)i(MPI)75 2267 y FE(The)d(supp)q(ort)g(for)f(virtual)i(top)q(ologies)f(has)f(to)h(b)q (e)g(fully)h(in)o(tegrated)f(in)g(the)g(framew)o(ork)f(of)g(MPI.)g(In) 75 2324 y(particular,)21 b(the)f(virtual)g(top)q(ology)f(functions)i (do)e(not)h(in)o(terfere)g(with)g(the)f(functions)i(for)e(pro)q(cess)75 2380 y(group)i(creation)g(and)g(with)g(those)f(for)h(collectiv)o(e)h (comm)o(unication.)38 b(The)21 b(curren)o(t)g(design)g(of)g(the)75 2437 y(pro)q(cess)15 b(top)q(ology)g(functions)h(is)g(based)g(on)f(the) g(follo)o(wing)h(assumptions:)143 2534 y FA(\017)23 b FE(MPI)15 b(is)h(free)g(to)f(c)o(ho)q(ose)h(an)o(y)f(pro)q(cess)h(in)h (a)e(group)g(for)g(an)o(y)h(lo)q(cation)g(in)h(the)f(top)q(ology)l(.)21 b(There)189 2591 y(are)16 b(no)g(sp)q(ecial)i(pro)q(cesses)f(whic)o(h)g (ha)o(v)o(e)f(to)g(b)q(e)h(aligned)g(with)g(top)q(ology)f (substructures.)23 b(This)189 2647 y(assumption)d(\014ts)g(b)q(est)h (to)e(a)h(\\lo)q(cal)h(SPMD)f(mo)q(del")h(in)g(whic)o(h)g(all)g(pro)q (cesses)g(in)g(the)f(same)189 2704 y(group)15 b(execute)g(the)h(same)f (program.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 107 113 107 112 bop 75 -100 a Fx(6.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)513 b FE(107)143 45 y FA(\017)23 b FE(One)g(imp)q(ortan)o(t)f(feature)g(of)g(MPI)g(are)g (the)h(pro)q(cess)f(groups.)42 b(The)22 b(treatmen)o(t)f(of)h(virtual) 189 102 y(top)q(ologies)f(has)g(to)g(tak)o(e)g(this)g(concept)h(in)o (to)f(accoun)o(t.)38 b(The)22 b(functions)g(presen)o(ted)f(in)i(this) 189 158 y(pap)q(er)16 b(are)g(not)g(only)h(compatible)h(with)e(the)h (pro)q(cess)f(groups,)g(they)g(ev)o(en)h(tak)o(e)f(adv)m(an)o(tage)f (of)189 214 y(them)c(for)g(the)g(de\014nition)i(of)e(mapping)h(o)o(v)o (erla)o(ys)e(and)i(dynamic)g(re{mapping)g(during)g(execution)189 271 y(time.)29 b(Virtual)19 b(pro)q(cess)f(top)q(ologies)h(help)h(in)f (the)f(sub)q(division)j(of)d(pro)q(cess)g(groups)g(whic)o(h)h(in)189 327 y(man)o(y)e(applications)j(has)e(to)f(b)q(e)i(done)g(along)f(pro)q (cess)g(co)q(ordinate)h(lines.)30 b(F)l(urthermore,)18 b(the)189 384 y(de\014nition)24 b(of)f(collectiv)o(e)h(gather{t)o(yp)q (e)e(comm)o(unication)i(calls)f(b)q(ene\014ts)h(from)e(the)h(natural) 189 440 y(ordering)15 b(of)g(pro)q(cesses)h(in)g(a)e(virtual)i(top)q (ology)l(.)75 536 y(The)h(virtual)g(top)q(ology)g(handling)h(follo)o (ws)f(the)g(general)g(MPI)g(rule)g(of)g(functional)h(la)o(y)o(ering.)25 b(A)o(t)16 b(the)75 593 y(b)q(ottom)j(it)g(is)h(based)g(on)f(the)h (general)g(pro)q(cess)f(group)g(creation)h(function)g Fo(MPI)p 1522 593 14 2 v 16 w(CREA)l(TE)p 1707 593 V 18 w(GROUP)75 649 y FE(plus)f(a)f(few)g(top)q(ology)g(sp)q(eci\014c)i (mapping)e(functions.)30 b(In)18 b(principle)j(this)e(is)f(su\016cien)o (t)h(for)e(de\014ning)75 706 y(top)q(ologies)f(and)f(use)h(them)f(in)h (applications.)166 763 y(A)f(n)o(um)o(b)q(er)g(of)f(higher{lev)o(el)j (functions)e(pro)o(vide)g(an)g(in)o(terface)g(whic)o(h)h(is)f(similar)h (to)e(what)g(users)75 819 y(of)19 b(pac)o(k)m(ages)g(lik)o(e)h(Express) f(and)g(P)l(ARMA)o(CS)h(are)f(familiar)h(with.)32 b(This)19 b(lev)o(el)i(can)e(b)q(e)h(written)f(in)75 875 y(a)j(p)q(ortable)h(w)o (a)o(y)e(in)i(terms)f(of)g(the)g(lo)o(w{lev)o(el)h(functions.)42 b(Implemen)o(tations)24 b(of)e(the)g(high{lev)o(el)75 932 y(functions)e(whic)o(h)h(b)o(ypass)e(the)g(lo)o(w)h(lev)o(el)h(ma)o (y)d(b)q(e)j(adv)m(an)o(tageous)d(on)i(some)f(systems)g(in)h(terms)f (of)75 988 y(memory)c(and)g(cpu)h(time)f(usage,)g(but)g(this)h(is)g (not)f(a)f(requiremen)o(t.)75 1135 y Fy(6.4)59 b(Overview)19 b(of)h(the)f(p)n(rop)r(osed)g(MPI)h(functions)75 1237 y FE(This)g(c)o(hapter)f(assumes)f(the)i(a)o(v)m(ailabilit)o(y)h(of)d (the)i(follo)o(wing)f(MPI)g(features)g(whic)o(h)h(are)f(de\014ned)h(in) 75 1294 y(other)15 b(c)o(hapters:)131 1390 y(1.)22 b(An)11 b(existing)g(pro)q(cess)g(group)f(\(e.g.,)g(the)h(ALL)h(group\))e(can)g (b)q(e)i(partitioned)f(and/or)f(ren)o(um)o(b)q(ered)189 1446 y(using)189 1569 y Fo(MPI)p 274 1569 V 15 w(CREA)l(TE)p 458 1569 V 18 w(GROUP\(gid)p 703 1569 V 17 w(old,)15 b(gid)p 862 1569 V 17 w(new,)g(group)p 1089 1569 V 17 w(k)o(ey)l(,)g(o)o(rder)p 1289 1569 V 15 w(k)o(ey\))230 1666 y Fu(IN)171 b Fo(gid)p 507 1666 V 17 w(old)441 b Fu(group)14 b(id)f(of)g(paren)o(t)i(group)230 1732 y(OUT)124 b Fo(gid)p 507 1732 V 17 w(new)424 b Fu(group)10 b(id)h(of)f(newly)g (created)j(subgroup)e(of)f(whic)o(h)g(the)1019 1788 y(calling)i(pro)q (cess)k(is)e(a)f(mem)o(b)q(er)230 1853 y(IN)171 b Fo(group)p 558 1853 V 17 w(k)o(ey)385 b Fu(in)o(teger)14 b(v)n(alue)f(used)i(for)f (the)g(partitioning)230 1919 y(IN)171 b Fo(o)o(rder)p 547 1919 V 16 w(k)o(ey)397 b Fu(in)o(teger)15 b(v)n(alue)f(used)h(for)f (determining)g(the)h(ranks)f(in)1019 1975 y(the)g(new)h(group)189 2119 y FE(This)22 b(function)g(is)f(called)i(b)o(y)e(all)i(pro)q (cesses)e(in)i(group)d Ft(gid)p 1265 2119 V 17 w(old)i FE(lo)q(osely)g(sync)o(hronously)l(.)39 b(It)189 2176 y(creates)18 b(as)g(man)o(y)g(di\013eren)o(t)g(subgroups)h(as)f(there)h (are)f(di\013eren)o(t)g(v)m(alues)i(of)e Ft(gr)n(oup)p 1676 2176 V 17 w(key)p FE(.)30 b(The)189 2232 y(ranking)16 b(in)h(the)g(newly)g(created)f(groups)g(is)h(determined)h(b)o(y)e(the)h Ft(or)n(der)p 1464 2232 V 17 w(key)f FE(v)m(alues,)h(with)g(the)189 2289 y(pro)q(cess)e(ranks)g(in)g(the)g(paren)o(t)g(group)g(for)f(tie)h (breaking.)21 b(If)15 b(there)g(is)h(a)e(standardized)i(in)o(ternal)189 2345 y(information)21 b(cac)o(hing)g(facilit)o(y)l(,)j(it)d(can)g(b)q (e)h(used)g(b)o(y)e(MPI)h(routines)h(to)e(store)h(and)g(retriev)o(e)189 2402 y(information)f(related)g(to)f(a)g(pro)q(cess)i(group.)33 b(This)20 b(facilit)o(y)h(w)o(as)e(prop)q(osed)h(in)h(the)f(Con)o(text) 189 2458 y(Sub)q(committee.)f(High{lev)o(el)12 b(top)q(ology)f (functions)g(store)f(structure)h(information)f(for)g(a)h(pro)q(cess)189 2514 y(group,)j(whic)o(h)i(is)g(then)f(later)h(used)f(b)o(y)h(top)q (ology)e(inquiry)j(functions.)189 2591 y(In)12 b(case)f(the)g(cac)o (hing)h(mec)o(hanism)g(do)q(es)f(not)g(b)q(ecome)h(part)f(of)g(MPI,)g (the)g(top)q(ology)g(information)189 2647 y(can)i(b)q(e)g(retriev)o(ed) g(b)o(y)g(group)f(id)i(in)g(an)e(implemen)o(tation{dep)q(enden)o(t)k(w) o(a)o(y)l(,)c(whic)o(h)h(eac)o(h)g(v)o(endor)189 2704 y(can)i(do)g(as)g(is)h(most)e(con)o(v)o(enien)o(t.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 108 114 108 113 bop 75 -100 a FE(108)903 b Fx(SECTION)16 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)166 45 y FE(There)i(has)g(b)q(een)h(a)e (general)i(decision)g(in)g(MPI)f(to)f(start)f(all)j(n)o(um)o(b)q(ering) g(with)f(0)g(instead)g(of)g(1.)75 102 y(Th)o(us,)f(co)q(ordinates)g(in) g(a)g(Cartesian)g(structure)f(are)h(0,1,)e(...,)h(dim-1.)22 b(In)17 b(the)f(same)f(w)o(a)o(y)l(,)g(no)q(des)i(in)f(a)75 158 y(general)g(graph)f(are)f(n)o(um)o(b)q(ered)i(zero)g(up.)75 279 y Fq(6.4.1)49 b(Lo)o(w{level)18 b(top)q(ology)g(functions)75 365 y FE(On)13 b(the)g(lo)o(w)o(est)f(lev)o(el)i(groups)e(are)h (created)f(b)o(y)h(the)g(general)g(function)g Fo(MPI)p 1384 365 14 2 v 16 w(CREA)l(TE)p 1569 365 V 18 w(GROUP)p FE(.)g(Using)75 422 y(the)k(input)i(argumen)o(t)d Ft(newr)n(ank)h FE(the)g(caller)i(can)e(c)o(hange)g(the)h(pro)q(cess)f(ranks)g(in)h (the)g(new)f(group)g(as)75 478 y(compared)e(to)e(the)i(paren)o(t)f (group.)19 b(This)d(feature)e(can)g(b)q(e)i(used)f(to)e(e\016cien)o (tly)j(map)f(the)f(pro)q(cesses)h(in)75 535 y(the)h(group)g(to)g(the)g (p)q(ositions)h(in)g(the)f(top)q(ology)l(.)23 b(F)l(or)15 b(eac)o(h)i(class)f(of)g(top)q(ologies,)g(therefore,)g(there)g(is)75 591 y(a)g(function)h(whic)o(h)g(computes)f(the)h(rank)f(of)g(the)g (calling)i(pro)q(cess)e(in)h(the)g(new)f(group,)g(based)h(on)f(the)75 648 y(size)g(and)g(structure)g(of)f(the)g(top)q(ology)l(.)21 b(Note)16 b(that)e(it)i(is)g(a)g(v)m(alid)h(implemen)o(tation)g(if)f (the)f(new)h(ranks)75 704 y(are)f(iden)o(tical)j(to)c(the)i(old)g (ranks.)k(So,)c(if)f(for)g(a)h(giv)o(en)g(mac)o(hine)g(the)f(placemen)o (t)i(of)e(pro)q(cesses)h(mak)o(es)75 760 y(no)f(di\013erence,)h(the)f (implemen)o(tation)i(of)e(the)g(rank)g(computation)g(functions)h(is)g (trivial.)166 817 y(The)h(functions)h(in)g(this)f(section)h(do)e(not)h (c)o(hange)g(the)g(state)f(of)h(the)g(calling)h(pro)q(cess,)g(but)f (they)75 873 y(pro)o(vide)f(the)f(information)g(used)h(in)g(a)f(follo)o (wing)h(group)f(creation)g(for)g(pro)q(cess)g(reordering.)166 930 y(F)l(or)f(Cartesian)h(structures)g(the)h(function)g(is)75 1033 y Fo(MPI)p 160 1033 V 16 w(MAP)p 275 1033 V 16 w(CART\(gid)p 485 1033 V 17 w(old,)f(ndims,)g(dims,)g(p)q(erio)q(d,)g(newrank\))117 1111 y Fu(IN)171 b Fo(gid)p 394 1111 V 16 w(old)441 b Fu(group)14 b(id)f(of)h(paren)o(t)g(group)117 1186 y(IN)171 b Fo(ndims)460 b Fu(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i (Cartesian)g(structure)117 1260 y(IN)171 b Fo(dims\(*\))424 b Fu(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(eac)o(h)g(co)q (ordinate)g(direction)117 1335 y(IN)155 b Fo(p)q(erio)q(d\(*\))410 b Fu(p)q(erio)q(dicit)o(y)14 b(sp)q(eci\014cation)h(in)e(eac)o(h)h(co)q (ordinate)h(direction)117 1410 y(OUT)124 b Fo(newrank)414 b Fu(reordered)16 b(rank)e(of)f(the)i(calling)d(pro)q(cess)166 1534 y FE(The)j(corresp)q(onding)i(function)f(for)e(general)i(graph)f (structures)g(is)75 1638 y Fo(MPI)p 160 1638 V 16 w(MAP)p 275 1638 V 16 w(GRAPH\(gid)p 516 1638 V 18 w(old,)f(no)q(des,)j(index,) f(nglist,)f(newrank\))117 1715 y Fu(IN)171 b Fo(gid)p 394 1715 V 16 w(old)441 b Fu(group)14 b(id)f(of)h(paren)o(t)g(group)117 1790 y(IN)171 b Fo(no)q(des)464 b Fu(n)o(um)o(b)q(er)13 b(of)h(graph)f(no)q(des)117 1865 y(IN)171 b Fo(index\(*\))414 b Fu(de\014ne)15 b(graph)f(structure,)h(see)h(text)117 1940 y(IN)171 b Fo(nglist\(*\))411 b Fu(de\014ne)15 b(graph)f (structure,)h(see)h(text)117 2015 y(OUT)124 b Fo(newrank)414 b Fu(reordered)16 b(rank)e(of)f(the)i(calling)d(pro)q(cess)166 2139 y FE(De\014nition)k(of)f(graph)g(structure:)k(for)c(eac)o(h)g (graph)g(no)q(de)g(its)g(set)g(of)g(neigh)o(b)q(ors)h(is)f(en)o(tered,) g(i.)h(e.,)75 2195 y(the)g(set)f(of)g(pro)q(cesses)h(with)f(whic)o(h)i (the)e(no)q(de)h(pro)q(cess)g(is)g(going)f(to)g(comm)o(unicate)h(most)f (frequen)o(tly)l(.)75 2252 y(Since)j(there)e(is)h(no)g(w)o(eigh)o(ting) f(of)g(no)q(de)h(connections,)h(it)e(migh)o(t)g(b)q(e)h(sensible)i(to)d (sp)q(ecify)h(only)g(those)75 2308 y(neigh)o(b)q(ors)f(with)f(whic)o(h) i(there)e(is)h(a)e(considerable)j(amoun)o(t)e(of)g(comm)o(unication.) 166 2365 y(F)l(or)21 b(eac)o(h)h(no)q(de)h FD(i)38 b FE(\(0)23 b FA(\024)i FD(i)e FA(\024)i FD(nodes)15 b FA(\000)g FE(1\))21 b(the)h(list)h(of)e(the)h(neigh)o(b)q(or)h(no)q (des)g(is)f(stored)g(at)75 2421 y FD(ng)r(l)q(ist)p FE(\()p FD(j)s FE(\))p FD(;)28 b(index)p FE(\()p FD(i)p FE(\))12 b FA(\024)h FD(j)i FA(\024)e FD(index)p FE(\()p FD(i)d FE(+)g(1\))g FA(\000)g FE(1.)20 b(F)l(or)15 b(the)g(last)g(pro)q(cess)h FD(index)p FE(\()p FD(i)9 b FE(+)i(1\))j(p)q(oin)o(ts)i(to)f(the)75 2478 y(\014rst)i(free)h(p)q(osition)h(in)f(v)o(ector)f(nglist.)28 b(Note)17 b(that)g(the)h(matrix)f(is)i(symmetric,)f(i.)f(e.,)h(if)g(no) q(de)h Ft(i)e FE(is)h(a)75 2534 y(neigh)o(b)q(or)e(of)f(no)q(de)h Ft(j)p FE(,)f(than)g(no)q(de)h Ft(j)f FE(is)h(also)f(a)g(neigh)o(b)q (or)h(of)f(no)q(de)h Ft(i)p FE(.)k(This)15 b(prop)q(ert)o(y)g(can)h(b)q (e)g(used)g(for)75 2591 y(an)f(in)o(ternal)h(c)o(hec)o(king)g(of)f(the) g(input)i(argumen)o(ts.)166 2647 y(The)i(de\014nitions)h(of)e(the)h (argumen)o(ts)f Fp(nodes)p FE(,)g Fp(index)p FE(,)h(and)g Fp(nglist)f FE(are)g(illustrated)i(with)f(the)75 2704 y(follo)o(wing)d(simple)h(example:)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 109 115 109 114 bop 75 -100 a Fx(6.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)513 b FE(109)166 45 y(Assume)15 b(there)h(are)f(four)f(pro)q(cesses)i(0,)e(1,)h(2,)f(3)h (with)h(the)f(follo)o(wing)h(adjacency)g(matrix:)100 98 y(pro)q(cess)50 b(neigh)o(b)q(ors)100 154 y(0)170 b(1,)15 b(3)100 211 y(1)170 b(0)100 267 y(2)g(3)100 323 y(3)g(0,)15 b(2)166 384 y(Then,)g(the)h(input)g(argumen)o(ts)e(are:)100 437 y(no)q(des)i(=)50 b(4)100 493 y(index)16 b(=)56 b(0,)14 b(2,)h(3,)f(4,)h(6)100 550 y(nglist)h(=)54 b(1,)14 b(3,)h(0,)f(3,)h(0,) f(2)166 610 y(If)i(the)g(top)q(ology)g(con)o(tains)g(few)o(er)f(pro)q (cesses)i(than)e(there)h(are)g(pro)q(cesses)g(in)h(the)f(paren)o(t)g (group,)75 667 y(the)g(newrank)h(v)m(alues)g(for)f(the)h(remaining)g (pro)q(cesses)g(are)f(set)g(to)g Fr(MPI)p 1320 667 13 2 v 14 w(PROCNULL)p FE(.)f(It)i(is)g(an)f(error)f(if)75 723 y(the)g(top)q(ology)g(de\014nes)h(more)f(pro)q(cesses)h(than)f(are) g(con)o(tained)h(in)g(the)f(paren)o(t)g(group.)166 864 y Fv(Discussion:)21 b Fu(It)c(cannot)g(b)q(e)h(ruled)f(out)g (completely)f(that)h(top)q(ology)e(de\014nition)i(functions)g(in)o (ternally)75 913 y(require)e(comm)o(unication.)i(This)e(could)f(b)q (ecome)h(a)f(necessit)o(y)i(if)e(a)g(parallel)f(algorithm)f(is)j(used)g (for)g(doing)e(the)75 963 y(mapping.)18 b(If)d(so,)f(then)h(it)g(migh)o (t)d(b)q(e)k(necessary)g(to)f(pass)g(them)f(a)g(comm)o(unicator)e (instead)j(of/in)f(addition)f(to)75 1013 y(the)h(group)g(id.)166 1078 y(In)k(this)g(case)h(w)o(e)f(also)f(ha)o(v)o(e)h(to)g(sa)o(y)g (that)g(a)f(lo)o(w{lev)o(el)g(top)q(ology)f(de\014nition)i(function)g (is)f(a)h(collectiv)o(e)75 1134 y(op)q(eration)f(on)f(the)i(group.)26 b(If)17 b(it)f(is)h(not)g(called)f(b)o(y)h(all)f(group)g(pro)q(cesses)k (lo)q(osely)c(sync)o(hronously)m(,)h(deadlo)q(c)o(k)75 1191 y(migh)o(t)12 b(o)q(ccur.)75 1441 y Fq(6.4.2)49 b(High{level)18 b(top)q(ology)h(functions)75 1542 y FE(In)g(principle,) i(all)e(that)e(is)i(needed)g(to)f(create)g(an)g(MPI)g(pro)q(cess)g (group)g(with)h(top)q(ological)g(ordering)75 1599 y(are)13 b(the)h(t)o(w)o(o)f(mapping)h(functions)g(and)g(the)g(general)g(group)g (creation)f(routine.)20 b(Ho)o(w)o(ev)o(er,)13 b(this)h(is)g(not)75 1655 y(what)19 b(the)g(applications)i(programmer)d(exp)q(ects)i(to)f (see)h(on)f(his)h(lev)o(el)h(of)e(abstraction.)31 b(Therefore,)75 1712 y(another)13 b(la)o(y)o(er)h(of)f(functions)h(is)g(de\014ned)i(on) d(top)g(of)h(the)f(lo)o(w{lev)o(el)i(functions.)20 b(This)14 b(higher)h(lev)o(el)g(can)75 1768 y(b)q(e)g(p)q(ortably)f(implemen)o (ted)i(in)f(terms)f(of)g(the)g(lev)o(el)i(b)q(elo)o(w.)k(Of)14 b(course)h(MPI)f(implemen)o(tors)h(are)f(free)75 1825 y(to)h(b)o(ypass)g(the)g(lo)o(w)o(er)g(lev)o(el)h(for)f(the)g(sak)o(e)g (of)g(b)q(etter)g(p)q(erformance)g(or)g(resource)g(usage.)166 1889 y(F)l(or)h(Cartesian)g(and)h(graph)f(structures)h(the)f(functions) i Fo(MPI)p 1252 1889 14 2 v 16 w(CART)f FE(and)g Fo(MPI)p 1576 1889 V 15 w(GRAPH)h FE(create)75 1946 y(a)c(pro)q(cess)g(group)g (with)h(top)q(ological)g(structure.)k(They)14 b(com)o(bine)h(the)f (call)i(to)d(the)h(mapping)h(function)75 2002 y(with)f(the)g(call)h(to) e(the)g(group)h(creation)g(routine.)20 b(The)14 b(user)f(can)h(decide)i (whether)e(the)f(ranking)h(in)h(the)75 2058 y(new)i(group)g(ma)o(y)f(b) q(e)i(reordered)f(\(for)f(a)g(more)h(e\016cien)o(t)g(mapping\))h(or)e (not.)25 b(In)17 b(the)g(latter)g(case)g(the)75 2115 y(rank)e(of)g(the)g(calling)i(pro)q(cess)e(is)h(copied)g(from)f(the)g (paren)o(t)g(group.)166 2179 y(The)h(option)g(of)f(not)g(reordering)h (the)g(pro)q(cess)f(ranks)h(can)f(b)q(e)i(useful)f(in)h(the)e(case)h (of)f(an)h(o)o(v)o(erla)o(y)75 2236 y(of)f(di\013eren)o(t)h(top)q (ologies.)21 b(The)16 b(user)f(can)h(then)g(fully)h(con)o(trol)e(whic)o (h)h(lo)q(cation)g(in)h(the)e(\014rst)g(top)q(ology)75 2292 y(coincides)i(with)e(whic)o(h)h(lo)q(cation)g(in)f(the)g(second)h (one.)k(The)15 b(price)h(whic)o(h)f(one)h(ob)o(viously)f(has)g(to)f(pa) o(y)75 2349 y(in)i(this)g(case)f(is)h(that)e(for)h(only)g(one)h(top)q (ology)f(the)g(pro)q(cess)g(arrangemen)o(t)f(can)i(b)q(e)g(optimized.) 166 2413 y(As)11 b(sp)q(eci\014ed)j(for)d(the)h(lo)o(w{lev)o(el)g (functions)h(it)f(is)g(an)f(error)g(if)h(one)g(attempts)e(to)h (de\014ne)i(a)e(top)q(ology)75 2470 y(with)18 b(more)g(lo)q(cations)g (than)g(there)g(are)g(pro)q(cesses)g(in)h(the)f(paren)o(t)f(pro)q(cess) i(group.)27 b(If)19 b(the)f(n)o(um)o(b)q(er)75 2526 y(of)f(lo)q (cations)h(is)f(less)h(than)f(the)g(n)o(um)o(b)q(er)h(of)f(a)o(v)m (ailable)h(pro)q(cesses,)g(the)f(remaining)i(pro)q(cesses)e(of)g(the)75 2583 y(paren)o(t)f(group)g(will)h(not)f(b)q(e)h(mem)o(b)q(ers)f(of)g (the)g(new)g(group.)23 b(F)l(or)15 b(those)h(pro)q(cesses)g(the)h(rank) e(inquiry)75 2639 y(function)h Fo(MPI)p 338 2639 V 16 w(RANK)f FE(returns)g Fr(MPI)p 723 2639 13 2 v 15 w(PROCNULL)f FE(if)i(called)h(with)e(the)h(iden)o(ti\014er)g(of)f(the)g(new)h (group.)166 2704 y Fo(MPI)p 251 2704 14 2 v 16 w(CART)f FE(is)h(de\014ned)h(as)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 110 116 110 115 bop 75 -100 a FE(110)903 b Fx(SECTION)16 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(CART\(gid)p 370 45 V 17 w(old,)f(gid)p 528 45 V 17 w(ca)o(rt,)g(ndims,)h(dims,)g(p)q(erio)q(d,)h(reo)o(rder\))117 128 y Fu(IN)171 b Fo(gid)p 394 128 V 16 w(old)441 b Fu(group)14 b(id)f(of)h(paren)o(t)g(group)117 215 y(OUT)124 b Fo(gid)p 394 215 V 16 w(ca)o(rt)425 b Fu(group)14 b(id)f(of)h(Cartesian)g(group) 117 302 y(IN)171 b Fo(ndims,)15 b(dims\(*\),)e(p)q(erio)q(d\(*\))82 b Fu(de\014nition)17 b(of)g(Cartesian)h(structure,)h(\(for)f(details)f (see)h(the)905 358 y(de\014nition)c(of)f Fr(MPI)p 1214 358 13 2 v 14 w(MAP)p 1319 358 V 15 w(CART)p Fu(\))117 445 y(IN)171 b Fo(reo)o(rder)438 b Fu(ranking)13 b(ma)o(y)f(b)q(e)j (reordered)h(or)e(not)166 575 y FE(Apart)h(from)f(some)h(lo)q(cal)h (computations,)f Fo(MPI)p 1008 575 14 2 v 16 w(CART)h FE(is)f(based)h(on)f(the)g(t)o(w)o(o)f(function)i(calls)166 637 y Fo(MPI)p 251 637 V 16 w(MAP)p 366 637 V 16 w(CART)g(\(gid)p 592 637 V 16 w(old,)f(ndims,)g(dims,)g(p)q(erio)q(d,)g(newrank\))166 699 y FE(and)166 762 y Fo(MPI)p 251 762 V 16 w(CREA)l(TE)p 436 762 V 17 w(GROUP)i(\(gid)p 697 762 V 16 w(old,)e(gid)p 855 762 V 16 w(ca)o(rt,)g(0,)f(newrank\))p FE(.)166 883 y(F)l(or)g(graph)h(structures)g(the)h(corresp)q(onding)g(routine)f(is) 75 992 y Fo(MPI)p 160 992 V 16 w(GRAPH\(gid)p 401 992 V 17 w(old,)g(gid)p 560 992 V 16 w(graph,)g(no)q(des,)i(index,)e (nglist,)h(reo)o(rder\))117 1075 y Fu(IN)171 b Fo(gid)p 394 1075 V 16 w(old)441 b Fu(group)14 b(id)f(of)h(paren)o(t)g(group)117 1162 y(OUT)124 b Fo(gid)p 394 1162 V 16 w(graph)391 b Fu(group)14 b(id)f(of)h(graph)f(group)117 1249 y(IN)171 b Fo(no)q(des,)16 b(index\(*\),)f(nglist\(*\))90 b Fu(de\014nition)14 b(of)g(graph)g(structure,)h(\(for)f(details)g(see)i(the)f(de\014-)905 1305 y(nition)e(of)g(MPI)p 1155 1305 13 2 v 16 w(MAP)p 1268 1305 V 15 w(GRAPH\))117 1392 y(IN)171 b Fo(reo)o(rder)438 b Fu(ranking)13 b(ma)o(y)f(b)q(e)j(reordered)h(or)e(not)166 1581 y FE(The)i(follo)o(wing)h(top)q(ology)f(information)h(is)g(lik)o (ely)h(to)e(b)q(e)g(stored)g(with)h(the)f(pro)q(cess)h(group)f(b)o(y)g (a)75 1637 y(high{lev)o(el)h(top)q(ology)e(de\014nition)i(function:)131 1754 y(1.)22 b(T)o(yp)q(e)15 b(of)g(top)q(ology)g(\(Cartesian/graph\)) 131 1871 y(2.)22 b(F)l(or)14 b(a)h(Cartesian)g(top)q(ology:)243 1988 y FA(\017)23 b FE(ndims)16 b(\(n)o(um)o(b)q(er)f(of)g (dimensions\),)243 2073 y FA(\017)23 b FE(dims\(*\))15 b(\(n)o(um)o(b)q(ers)g(of)f(pro)q(cesses)i(p)q(er)g(co)q(ordinate)f (direction\),)243 2157 y FA(\017)23 b FE(p)q(erio)q(d\(*\))15 b(\(p)q(erio)q(dicit)o(y)i(information\),)243 2242 y FA(\017)23 b FE(o)o(wn)p 372 2242 14 2 v 16 w(p)q(osition\(*\))c(\(o)o (wn)f(p)q(osition)i(in)g(grid,)h(could)f(also)f(b)q(e)h(computed)f (from)g(rank)g(and)289 2299 y(dims\(*\)\))75 2416 y(F)l(or)d(a)g(graph) g(structure)g(no)g(information)h(has)f(to)g(b)q(e)h(stored,)f(b)q (ecause)h(the)g(graph)f(no)q(de)h(n)o(um)o(b)q(er)f(is)75 2472 y(the)h(rank)f(in)i(the)f(graph)f(group,)h(and)g(the)g(size)g(of)g (the)g(graph)f(is)i(the)e(n)o(um)o(b)q(er)i(of)e(pro)q(cesses)h(in)h (that)75 2528 y(group.)166 2591 y(The)d(actual)g(set)g(of)g(top)q (ology)f(data)h(stored)f(with)i(a)f(pro)q(cess)g(group)g(is)g(implemen) o(tation)h(dep)q(en-)75 2647 y(den)o(t.)k(It)13 b(can)h(con)o(tain)g (more)g(information)g(than)g(giv)o(en)g(ab)q(o)o(v)o(e,)f(whereas)h(in) h(other)e(cases)h(some)g(items)75 2704 y(migh)o(t)h(b)q(e)h(redundan)o (t.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 111 117 111 116 bop 75 -100 a Fx(6.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)513 b FE(111)75 45 y Fq(6.4.3)49 b(T)l(op)q(ology)19 b(inquiry)e(functions)75 162 y FE(If)j(a)g(top)q(ology)g(has)f(b)q(een)j(de\014ned)f(with)f(one) g(of)g(the)g(ab)q(o)o(v)o(e)g(high{lev)o(el)i(functions,)f(the)f(top)q (ology)75 218 y(information)c(can)h(b)q(e)f(lo)q(ok)o(ed)h(up)g(using)f (inquiry)i(functions.)23 b(They)17 b(all)g(are)f(lo)q(cal)h(calls,)g (and)f(do)g(not)75 275 y(require)g(an)o(y)f(comm)o(unication.)166 347 y(The)g(function)h Fo(MPI)p 522 347 14 2 v 16 w(INQMAP)f FE(returns)f(the)h(t)o(yp)q(e)g(of)g(mapping)g(whic)o(h)h(is)f (assigned)h(to)e(a)h(group.)75 466 y Fo(MPI)p 160 466 V 16 w(INQMAP\(gid,)g(map\))117 560 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)117 666 y(OUT)108 b Fo(map)505 b Fu(top)q(ology)13 b(t)o(yp)q(e)h(of)g(group)f(with)h(iden)o(ti\014er)g Fn(gid)166 807 y FE(The)h(output)g(v)m(alue)i Fp(map)d FE(is)i(one)g(of)e(the)i(follo)o(wing:)117 910 y Fr(MPI)p 194 910 13 2 v 14 w(GRAPHTOP)516 b FE(graph)15 b(top)q(ology)117 996 y Fr(MPI)p 194 996 V 14 w(CARTTOP)545 b FE(Cartesian)15 b(top)q(ology)117 1083 y Fr(MPI)p 194 1083 V 14 w(NOTOP)595 b FE(no)15 b(top)q(ology)166 1181 y(F)l(or)i(a)h(pro)q(cess)g(group)f (with)i(a)e(graph)h(structure)f(whic)o(h)i(has)f(b)q(een)h(created)f(b) o(y)f Fo(MPI)p 1698 1181 14 2 v 16 w(GRAPH)p FE(,)75 1238 y(functions)f Fo(MPI)p 356 1238 V 16 w(GRAPHDIMS)g FE(and)f Fo(MPI)p 819 1238 V 16 w(INQGRAPH)h FE(lo)q(ok)g(up)g(top)q (ology)e(information:)75 1357 y Fo(MPI)p 160 1357 V 16 w(GRAPHDIMS\(gid,)h(indexlen,)h(nglistlen\))117 1450 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)f(of)h(group)f(with)h (graph)g(structure)117 1557 y(OUT)124 b Fo(indexlen)419 b Fu(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(v)o(ector)i Fn(index)117 1664 y Fu(OUT)124 b Fo(nglistlen)416 b Fu(n)o(um)o(b)q(er) 13 b(of)h(en)o(tries)h(in)e(v)o(ector)i Fn(nglist)166 1804 y FE(The)h(information)g(pro)o(vided)h(b)o(y)f Fo(MPI)p 844 1804 V 15 w(GRAPHDIMS)h FE(can)f(b)q(e)g(used)h(to)e(dimension)j (the)e(v)o(ectors)75 1861 y Fp(index)i FE(and)i Fp(nglist)e FE(long)h(enough)h(in)g(the)f(follo)o(wing)h(call)g(to)f Fo(MPI)p 1298 1861 V 15 w(INQGRAPH)p FE(.)i(The)e(n)o(um)o(b)q(er)g(of) 75 1917 y(no)q(des)d(in)g(the)f(graph)g(is)h Fp(indexlen)22 b(-)i(1)p FE(.)75 2037 y Fo(MPI)p 160 2037 V 16 w(INQGRAPH\(gid,)16 b(indexdim,)f(nglistdim,)g(index,)h(nglist\))117 2130 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)f(of)h(group)f(with)h (graph)g(structure)117 2237 y(IN)171 b Fo(indexdim)402 b Fu(length)14 b(of)f(v)o(ector)i(index\(*\))f(in)f(the)i(calling)d (program)117 2344 y(IN)171 b Fo(nglistdim)399 b Fu(length)14 b(of)f(v)o(ector)i(nglist\(*\))e(in)h(the)g(calling)f(program)117 2450 y(OUT)124 b Fo(index\(*\),)14 b(nglist\(*\))227 b Fu(v)o(ectors)15 b(con)o(taining)d(the)h(graph)g(structure)j(\(for)d (details)g(see)905 2507 y(the)i(de\014nition)e(of)g(MPI)p 1291 2507 13 2 v 16 w(MAP)p 1404 2507 V 15 w(GRAPH\))166 2647 y FE(F)l(or)18 b(a)h(pro)q(cess)g(group)f(with)h(Cartesian)g (ordering)g(whic)o(h)h(has)e(b)q(een)i(created)f(b)o(y)g Fo(MPI)p 1730 2647 14 2 v 16 w(CART)p FE(,)75 2704 y(functions)d Fo(MPI)p 356 2704 V 16 w(INQDIM)e FE(and)i Fo(MPI)p 721 2704 V 16 w(INQCART)g FE(lo)q(ok)f(up)h(top)q(ology)f(information:)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 112 118 112 117 bop 75 -100 a FE(112)903 b Fx(SECTION)16 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(INQDIM\(gid,)f(dim\))117 123 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g (structure)117 199 y(OUT)124 b Fo(dim)500 b Fu(dimension)12 b(of)i(the)g(Cartesian)g(structure)75 371 y Fo(MPI)p 160 371 V 16 w(INQCART\(gid,)h(maxdim,)e(size,)i(p)q(erio)q(d,)h(p)q (os\))117 449 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g(structure)117 525 y(IN)171 b Fo(maxdim)420 b Fu(length)14 b(of)g(v)o(ectors)h (size\(*\),)g(p)q(erio)q(d\(*\),)f(and)g(p)q(os\(*\))h(in)e(the)905 581 y(calling)g(program)117 657 y(OUT)124 b Fo(size\(*\))444 b Fu(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(for)e(eac)o(h)g(co)q (ordinate)g(direction)117 733 y(OUT)124 b Fo(p)q(erio)q(d\(*\))394 b Fu(p)q(erio)q(dicit)o(y)14 b(y)o(es/no)g(for)f(eac)o(h)i(co)q (ordinate)f(direction)117 809 y(OUT)124 b Fo(p)q(os\(*\))448 b Fu(p)q(osition)13 b(of)h(calling)e(pro)q(cess)k(in)d(Cartesian)h (structure)166 934 y FE(The)20 b(inquiry)g(for)f(Cartesian)g(top)q (ology)h(information)f(is)h(th)o(us)f(split)i(up)f(in)o(to)f(t)o(w)o(o) f(calls.)34 b(The)75 991 y(reason)21 b(is)h(that)f(otherwise)h(the)f (user)h(migh)o(t)f(not)g(kno)o(w)g(a)h(sa)o(v)o(e)e(v)m(alue)j(for)e (the)h(dimensioning)h(of)75 1047 y(v)o(ectors)14 b Fp(size,)24 b(period)14 b FE(and)h Fp(pos)g FE(b)q(efore)h(calling)g Fo(MPI)p 1071 1047 V 16 w(INQCART)g FE(\(input)g(argumen)o(t)f Fp(maxdim)p FE(\).)166 1104 y(F)l(or)i(a)g(pro)q(cess)g(group)h(with)f (Cartesian)g(structure,)h(the)f(function)i Fo(MPI)p 1451 1104 V 15 w(INQRANK)f FE(translates)75 1160 y(the)j(logical)h(pro)q (cess)f(co)q(ordinates)g(to)f(pro)q(cess)h(ranks)f(as)g(they)h(are)f (used)i(b)o(y)e(the)h(p)q(oin)o(t{to{p)q(oin)o(t)75 1217 y(routines:)75 1321 y Fo(MPI)p 160 1321 V 16 w(INQRANK\(gid,)15 b(lo)q(cation,)g(rank\))117 1399 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g(structure)117 1475 y(IN)171 b Fo(lo)q(cation)421 b Fu(co)q(ordinate)14 b(v)o(ector)h(of)e(pro)q(cess)117 1551 y(OUT)124 b Fo(rank)488 b Fu(pro)q(cess)16 b(rank)166 1676 y FE(In)g(a)f(sense,)g(the)g(in)o(v) o(erse)h(op)q(eration)f(to)g Fo(MPI)p 963 1676 V 16 w(INQRANK)g FE(is)h(pro)o(vided)g(b)o(y)f(the)h(function:)75 1780 y Fo(MPI)p 160 1780 V 16 w(INQLOC\(gid,)f(rank,)g(lo)q(cation,)g (maxdim\))117 1857 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g(structure)117 1933 y(IN)171 b Fo(rank)488 b Fu(rank)14 b(of)f(a)h(pro)q(cess)i (within)d(group)g Fn(gid)117 2009 y Fu(OUT)124 b Fo(lo)q(cation\(*\)) 362 b Fu(co)q(ordinate)14 b(v)o(ector)h(of)e(pro)q(cess)117 2085 y(IN)171 b Fo(maxdim)420 b Fu(length)11 b(of)e(v)o(ector)i Fn(location)e Fu(in)h(the)h(calling)e(program)g(\(only)905 2142 y(for)14 b(error)h(c)o(hec)o(king\))75 2267 y Fo(MPI)p 160 2267 V 16 w(INQLOC)g FE(translates)g(a)g(rank)g(in)h(the)f(group)g (in)o(to)g(a)g(top)q(ology)g(lo)q(cation.)75 2391 y Fq(6.4.4)49 b(P)o(a)o(rtitioning)17 b(of)g(Ca)o(rtesian)e(structures)75 2478 y FE(If)g(a)f(group)h(has)f(b)q(een)i(created)f(with)g Fo(MPI)p 820 2478 V 15 w(CART)p FE(,)g(the)g(function)g Fo(MPI)p 1319 2478 V 16 w(P)l(ARTC)h FE(can)f(b)q(e)g(used)g(to)f(par-) 75 2534 y(tition)i(the)g(group)f(in)o(to)h(subgroups)f(whic)o(h)i(form) e(h)o(yp)q(er{planes)h(of)f(the)h(original)h(pro)q(cess)f(structure.)75 2591 y(As)i(an)g(example,)h(assume)f(that)g Fo(MPI)p 752 2591 V 15 w(CART)h FE(has)f(de\014ned)h(a)f(t)o(w)o(o{dimensional)h (grid)f(structure)g(on)75 2647 y(a)f(pro)q(cess)g(group.)25 b(With)17 b(a)g(call)h(to)f Fo(MPI)p 814 2647 V 15 w(P)l(ARTC)h FE(this)g(group)e(can)i(then)f(b)q(e)h(split)g(up)f(in)o(to)g(disjoin)o (t)75 2704 y(subgroups)e(whic)o(h)h(represen)o(t)g(ro)o(ws)e(or)g (columns)i(of)f(the)g(original)i(group.)i(The)d(function)g(syn)o(tax)e (is:)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 113 119 113 118 bop 75 -100 a Fx(6.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)513 b FE(113)75 45 y Fo(MPI)p 160 45 14 2 v 16 w(P)l(ARTC\(gid,)15 b(lostdim,)g(gid)p 646 45 V 16 w(new\))117 122 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)f(of)h(group)f(with)h(Cartesian)g(structure)117 198 y(IN)171 b Fo(lostdim)433 b Fu(direction)14 b(p)q(erp)q(endicular)h (to)f(resulting)g(h)o(yp)q(er{planes)117 273 y(OUT)124 b Fo(gid)p 394 273 V 16 w(new)424 b Fu(group)12 b(iden)o(ti\014er)g(of) f(the)h(h)o(yp)q(er{plane)g(group)g(in)f(whic)o(h)g(the)905 330 y(calling)i(pro)q(cess)i(is)f(con)o(tained)166 454 y Fo(MPI)p 251 454 V 16 w(P)l(ARTC)22 b FE(uses)f(the)g(function)h Fo(MPI)p 885 454 V 16 w(CREA)l(TE)p 1070 454 V 17 w(GROUP)h FE(to)d(partition)i(the)f(paren)o(t)g(group,)75 511 y(using)h(the)f (pro)q(cess)g(co)q(ordinate)h(in)f(direction)i Fp(lostdim)d FE(as)g(the)i(k)o(ey)e(v)m(alue.)39 b(After)21 b(a)f(group)h(has)75 567 y(b)q(een)16 b(partitioned)f(b)o(y)g Fo(MPI)p 564 567 V 15 w(P)l(ARTC)p FE(,)g(the)g(inquiry)h(functions)f Fo(MPI)p 1262 567 V 16 w(INQCART)h FE(and)e Fo(MPI)p 1660 567 V 16 w(INQRANK)75 624 y FE(consisten)o(tly)i(refer)f(to)g(the) g(grid)g(sections)h(if)g(applied)h(to)d(a)h(subgroup.)166 680 y(If)f(the)h(pro)q(cess)f(top)q(ology)g(is)h(a)e(Cartesian)h (structure,)g(a)g(collectiv)o(e)i(shift)e(op)q(eration)h(most)e(lik)o (ely)75 737 y(is)21 b(used)h(along)e(a)h(co)q(ordinate)g(direction.)37 b(In)22 b(MPI)e(shift)h(op)q(erations)g(are)g(handled)h(b)o(y)f(the)f (more)75 793 y(general)f(send{receiv)o(e)h(functions.)31 b(Those)19 b(tak)o(e)f(as)g(input)i(the)f(rank)f(of)h(a)f(source)h(pro) q(cess)g(for)f(the)75 849 y(receiv)o(e,)e(and)f(the)g(rank)g(of)g(a)g (destination)h(pro)q(cess)g(for)e(the)h(send.)166 906 y(If)d(the)f(function)h Fo(MPI)p 541 906 V 16 w(SHIFT)g FE(is)g(called)h(for)d(a)i(Cartesian)f(pro)q(cess)g(group,)h(it)g(pro)o (vides)g(the)f(calling)75 962 y(pro)q(cess)17 b(with)f(the)h(ab)q(o)o (v)o(e)f(iden)o(ti\014ers,)i(whic)o(h)f(then)g(can)g(b)q(e)g(passed)f (to)g(the)h(send{receiv)o(e)g(function.)75 1019 y(The)e(user)h(sp)q (eci\014es)h(the)e(co)q(ordinate)h(direction)g(for)f(the)g(shift)h(and) f(the)g(step)g(length:)75 1123 y Fo(MPI)p 160 1123 V 16 w(SHIFT)p 303 1123 V 16 w(ID\(gid,)f(direction,)i(steps,)g(rank)p 864 1123 V 17 w(source,)f(rank)p 1111 1123 V 17 w(dest\))117 1200 y Fu(IN)171 b Fo(gid)514 b Fu(group)14 b(id)f(of)h(group)f(with)h (Cartesian)g(structure)117 1275 y(IN)171 b Fo(direction)408 b Fu(co)q(ordinate)14 b(direction)h(of)e(shift)117 1351 y(IN)171 b Fo(steps)478 b Fu(step)22 b(length)f(\()p Fm(>)i Fu(0:)32 b(up)o(w)o(ards)21 b(shift,)h Fm(<)h Fu(0:)32 b(do)o(wn)o(w)o(ards)905 1407 y(shift\))117 1483 y(OUT)124 b Fo(rank)p 420 1483 V 16 w(source)353 b Fu(rank)14 b(of)f(source)j(pro)q(cess)117 1558 y(OUT)124 b Fo(rank)p 420 1558 V 16 w(dest)396 b Fu(rank)14 b(of)f(destination)h (pro)q(cess)166 1683 y FE(Dep)q(ending)k(of)d(the)h(p)q(erio)q(dicit)o (y)j(of)c(the)h(Cartesian)g(group)g(in)h(the)f(sp)q(eci\014ed)i(co)q (ordinate)f(direc-)75 1739 y(tion,)h Fo(MPI)p 269 1739 V 16 w(SHIFT)p 412 1739 V 16 w(ID)f FE(pro)o(vides)i(the)e(iden)o (ti\014ers)j(for)d(a)g(circular)i(or)e(an)h(end{o\013)f(shift.)28 b(In)18 b(the)g(case)75 1795 y(of)d(an)h(end{o\013)f(shift,)g(at)g(the) h(b)q(oundary)g(pro)q(cess)g(one)g(of)f(the)g(iden)o(ti\014ers)i(is)f (set)g(to)f Fr(MPI)p 1637 1795 13 2 v 14 w(PROCNULL)p FE(,)75 1852 y(indicating)i(that)e(there)g(is)h(no)f(pro)q(cess)g(in)h (that)f(direction)h(to)f(comm)o(unicate)g(with.)166 1908 y(In)22 b(a)g(p)q(ortable)g(program)f(the)h(n)o(um)o(b)q(er)g(of)f(a)o (v)m(ailable)j(pro)q(cessors)d(usually)i(is)g(not)e(kno)o(wn)h(at)75 1965 y(compile)h(time.)38 b(In)21 b(the)h(case)f(of)f(a)h(Cartesian)g (top)q(ology)g(the)g(function)h Fo(MPI)p 1494 1965 14 2 v 16 w(MAKDIM)e FE(helps)i(in)75 2021 y(setting)f(the)g(n)o(um)o(b)q (ers)h(of)f(pro)q(cesses)g(p)q(er)h(co)q(ordinate)f(direction,)j(dep)q (ending)g(on)d(the)g(n)o(um)o(b)q(er)g(of)75 2078 y(pro)q(cesses)16 b(in)g(the)f(group)g(and)g(optional)h(constrain)o(ts)f(whic)o(h)h(can)f (b)q(e)h(sp)q(eci\014ed)h(b)o(y)e(the)h(user:)75 2182 y Fo(MPI)p 160 2182 V 16 w(MAKDIM\(np)o(ro)q(cs,)e(ndims,)h(dims\))117 2259 y Fu(IN)171 b Fo(np)o(ro)q(cs)449 b Fu(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)117 2334 y(IN)171 b Fo(ndims)460 b Fu(n)o(um)o(b)q(er)13 b(of)h(Cartesian)g(dimensions) 117 2410 y(INOUT)78 b Fo(dims)483 b Fu(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(eac)o(h)g(co)q(ordinate)g(direction)166 2534 y FE(The)h Fp(nprocs)e FE(pro)q(cesses)i(are)f(divided)j(as)d(ev)o (enly)i(as)e(p)q(ossible)i(among)e Fp(ndims)g FE(dimensions,)h(sub-)75 2591 y(ject)20 b(to)g(the)h(constrain)o(t)e(that)h(all)h(pro)q(cesses)g (are)f(used.)36 b(The)21 b(caller)g(ma)o(y)f(further)g(constrain)h(the) 75 2647 y(op)q(eration)h(of)g(this)h(routine)f(b)o(y)g(sp)q(ecifying)i (elemen)o(ts)f(of)f(arra)o(y)f Fp(dims)p FE(.)40 b(If)22 b Fp(dims\(i\))f FE(is)i(set)f(to)f(a)75 2704 y(p)q(ositiv)o(e)d(n)o (um)o(b)q(er,)g(the)g(routine)f(will)i(\014x)f(the)f(n)o(um)o(b)q(er)h (of)f(pro)q(cesses)h(in)g(co)q(ordinate)g(direction)g Fp(i)g FE(to)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 114 120 114 119 bop 75 -100 a FE(114)903 b Fx(SECTION)16 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fp(dims\(i\))p FE(.)27 b(Pro)q(cesses)17 b(will)j(then)e(b)q(e)g(distributed)i(among)d (directions)i(where)f Fp(dims\(i\))f FE(is)h(set)g(to)f(0,)75 102 y(sub)s(ject)e(to)g(the)g(ab)q(o)o(v)o(e)g(constrain)o(ts.)166 158 y(Note)g(that)f(an)h(error)g(will)i(o)q(ccur)e(if)h Fp(nprocs)e FE(is)i(not)f(a)g(m)o(ultiple)i(of)1417 118 y Fe(Y)1352 212 y Fi(i;dims)p Fw(\()p Fi(i)p Fw(\))p Fj(6)p Fw(=0)1543 158 y FD(dims)p FE(\()p FD(i)p FE(\).)166 260 y(Negativ)o(e)i(input)h(v)m(alues)h(of)e Fp(dims\(i\))f FE(will)j(also)e(cause)g(an)g(error.)32 b(All)20 b(elemen)o(ts)g(of)f Fp(dims\(*\))75 316 y FE(m)o(ust)c(b)q(e)g(set)g(to)g(0)g(for)f (completely)j(automatic)e(pro)q(cess)g(distribution.)166 373 y(When)c(the)g(function)h(returns,)f Fp(dims\(i\))f FE(will)j(b)q(e)e(set)g(to)f(the)h(n)o(um)o(b)q(er)g(of)g(pro)q(cesses) g(in)h(co)q(ordinate)75 429 y(direction)20 b Fp(i)p FE(.)29 b(F)l(or)18 b Fp(dims\(i\))g FE(not)g(set)g(b)o(y)h(the)f(user,)h Fp(dims\(i\))f FE(will)i(b)q(e)f(ordered)g(\(MAXIMUM,)e(...,)75 486 y(MINIMUM\).)d(Arra)o(y)h Fp(dims\(*\))f FE(is)i(suitable)g(for)f (use)g(as)g(input)h(to)f(routine)g Fo(MPI)p 1496 486 14 2 v 16 w(CART)p FE(.)75 629 y Fy(6.5)59 b(A)20 b(simple)f(example)75 730 y FE(The)12 b(example)h(in)g(\014gure)f(6.1)f(sho)o(ws)g(ho)o(w)h (the)g(grid)g(de\014nition)i(and)e(inquiry)i(functions)f(can)f(b)q(e)h (used)f(in)75 787 y(an)k(application)h(program.)k(A)16 b(partial)g(di\013eren)o(tial)h(equation,)f(for)f(example)h(the)g(P)o (oisson)g(equation,)75 843 y(is)g(to)f(b)q(e)h(solv)o(ed)f(on)h(a)f (rectangular)g(domain.)21 b(First)15 b(the)g(pro)q(cesses)h(organize)f (themselv)o(es)h(in)h(a)e(t)o(w)o(o{)75 900 y(dimensional)h(structure.) k(Eac)o(h)14 b(pro)q(cess)h(then)f(inquires)j(for)c(the)i(ranks)f(of)g (its)g(neigh)o(b)q(ors)i(in)f(the)f(four)75 956 y(directions.)29 b(The)18 b(n)o(umerical)h(problem)g(is)f(solv)o(ed)h(b)o(y)f(an)f (iterativ)o(e)h(metho)q(d,)h(the)f(details)h(of)e(whic)o(h)75 1013 y(are)e(hidden)i(in)f(the)f(subroutine)h Fp(relax)p FE(.)166 1069 y(In)e(eac)o(h)f(relaxation)g(step)g(eac)o(h)g(pro)q (cess)h(computes)f(new)g(v)m(alues)h(for)f(the)g(solution)h(grid)f (function)75 1126 y(at)19 b(all)i(p)q(oin)o(ts)f(o)o(wned)f(b)o(y)h (the)g(pro)q(cess.)33 b(Then)21 b(the)e(v)m(alues)i(at)e(in)o(terpro)q (cess)h(b)q(oundaries)h(ha)o(v)o(e)f(to)75 1182 y(b)q(e)h(exc)o(hanged) h(with)f(neigh)o(b)q(oring)h(pro)q(cesses.)37 b(F)l(or)20 b(example,)j(the)d(exc)o(hange)h(subroutine)h(migh)o(t)75 1238 y(con)o(tain)14 b(a)f(call)i(lik)o(e)g Fo(MPI)p 519 1238 V 15 w(SENDC\(...,neigh)p 850 1238 V 16 w(rank\(1\),...\))i FE(to)c(send)h(up)q(dated)h(v)m(alues)g(to)e(the)g(left{hand)75 1295 y(neigh)o(b)q(or)j(\(i-1,j\).)j(Note)c(that)g(there)g(is)h(only)f (lo)q(cal)i(comm)o(unication)f(in)g(this)f(exc)o(hange)h(op)q(eration.) 1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 115 121 115 120 bop 75 -100 a Fx(6.5.)34 b(A)15 b(SIMPLE)h(EXAMPLE)1157 b FE(115)252 191 y(in)o(teger)15 b(ndims,)h(n)o(um)p 639 191 14 2 v 16 w(neigh)252 247 y(logical)g(reorder)252 304 y(parameter)e(\(ndims=2,)i(n)o(um)p 782 304 V 16 w(neigh=4,)g(reorder=.true.\))252 360 y(in)o(teger)f(gid,)h(gid)p 554 360 V 17 w(cart,)e(dims\(ndims\),)h(neigh)p 1057 360 V 18 w(def\(2\),)f(neigh)p 1325 360 V 17 w(rank\(n)o(um)p 1537 360 V 16 w(neigh\))252 417 y(in)o(teger)h(o)o(wn)p 485 417 V 16 w(p)q(osition\(ndims\),)h(p)q(erio)q(d\(ndims\),)h(i,)e (j,)g(rank)252 473 y(real)p FA(\003)p FE(8)g(u\(0:101,0:101)o(\),)d (f\(0:101,0:10)o(1\))252 530 y(data)i(dims)i(/)f(ndims)h FA(\003)f FE(0)g(/)75 586 y(C)89 b(Create)15 b(a)g(grid)g(structure)g (in)h(ALL)g(group)f(and)h(inquire)g(ab)q(out)g(o)o(wn)e(p)q(osition)252 642 y(gid)i(=)f(MPI)p 470 642 V 16 w(ALL)75 699 y(C)89 b(Set)15 b(pro)q(cess)h(grid)f(size)i(and)e(p)q(erio)q(dicit)o(y)252 755 y(call)h(MPI)p 427 755 V 16 w(MAKDIM\(gid,)f(ndims,)h(dims\))252 812 y(p)q(erio)q(d\(1\))f(=)h(MPI)p 594 812 V 16 w(PER)252 868 y(p)q(erio)q(d\(2\))f(=)h(MPI)p 594 868 V 16 w(PER)252 925 y(call)g(MPI)p 427 925 V 16 w(CAR)l(T)g(\(gid,)f(gid)p 754 925 V 17 w(cart,)f(ndims,)i(dims,)f(p)q(erio)q(d,)i(reorder,)d (rank\))252 981 y(call)i(MPI)p 427 981 V 16 w(INQCAR)l(T)h(\(gid)p 752 981 V 17 w(cart,)d(ndims,)h(dims,)h(p)q(erio)q(d,)g(o)o(wn)p 1377 981 V 16 w(p)q(osition\))75 1038 y(C)89 b(Lo)q(ok)15 b(up)h(the)f(ranks)g(for)g(the)g(neigh)o(b)q(ors.)21 b(Own)15 b(pro)q(cess)h(co)q(ordinates)f(are)g(\(i,j\).)75 1094 y(C)89 b(Neigh)o(b)q(ors)16 b(are)f(\(i{1,j\),)f(\(i+1,j\),)g (\(i,j{1\),)g(\(i,j+1\))252 1151 y(i)i(=)f(o)o(wn)p 413 1151 V 16 w(p)q(osition\(1\))252 1207 y(j)g(=)g(o)o(wn)p 414 1207 V 16 w(p)q(osition\(2\))252 1263 y(neigh)p 360 1263 V 17 w(def\(1\))g(=)g(i{1)252 1320 y(neigh)p 360 1320 V 17 w(def\(2\))g(=)g(j)252 1376 y(call)h(MPI)p 427 1376 V 16 w(INQRANK)h(\(gid,)f(neigh)p 893 1376 V 17 w(def,)f(neigh)p 1102 1376 V 18 w(rank\(1\)\))252 1433 y(neigh)p 360 1433 V 17 w(def\(1\))g(=)g(i+1)252 1489 y(neigh)p 360 1489 V 17 w(def\(2\))g(=)g(j)252 1546 y(call)h(MPI)p 427 1546 V 16 w(INQRANK)h(\(gid,)f(neigh)p 893 1546 V 17 w(def,)f(neigh)p 1102 1546 V 18 w(rank\(2\)\))252 1602 y(neigh)p 360 1602 V 17 w(def\(1\))g(=)g(i)252 1659 y(neigh)p 360 1659 V 17 w(def\(2\))g(=)g(j{1)252 1715 y(call)h(MPI)p 427 1715 V 16 w(INQRANK)h(\(gid,)f(neigh)p 893 1715 V 17 w(def,)f(neigh)p 1102 1715 V 18 w(rank\(3\)\))252 1772 y(neigh)p 360 1772 V 17 w(def\(1\))g(=)g(i)252 1828 y(neigh)p 360 1828 V 17 w(def\(2\))g(=)g(j+1)252 1884 y(call)h(MPI)p 427 1884 V 16 w(INQRANK)h(\(gid,)f(neigh)p 893 1884 V 17 w(def,)f(neigh)p 1102 1884 V 18 w(rank\(4\)\))75 1941 y(C)89 b(Initialize)18 b(the)e(grid)f(functions)h(and)g(start)d (the)j(iteration)252 1997 y(call)g(init)g(\(u,)f(f)t(\))252 2054 y(do)g(10)g(it=1,100)322 2110 y(call)i(relax)e(\(u,)g(f)t(\))75 2167 y(C)89 b(Exc)o(hange)15 b(data)g(with)g(neigh)o(b)q(or)h(pro)q (cesses)322 2223 y(call)h(exc)o(hange)e(\(u,)g(neigh)p 779 2223 V 17 w(rank,)g(n)o(um)p 1001 2223 V 16 w(neigh\))75 2280 y(10)131 b(con)o(tin)o(ue)252 2336 y(call)16 b(output)f(\(u\))252 2393 y(end)147 2597 y(Figure)g(6.1:)k(Set{up)d(of)e(pro)q(cess)i (structure)f(for)f(t)o(w)o(o{dimensional)i(parallel)h(P)o(oisson)e (solv)o(er.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 116 122 116 121 bop 75 358 a Fz(Section)35 b(7)75 568 y FG(Language)41 b(Binding)75 811 y Fy(7.1)59 b(Intro)r(duction)75 916 y FE(This)21 b(c)o(hapter)g(de\014nes)h(the)f(rules)g(for)g(MPI)f (language)h(binding)i(in)f(general)f(and)g(for)f(F)l(ortran)g(77,)75 972 y(F)l(ortran)15 b(90,)g(ANSI)i(C,)e(and)i(ANSI)f(C++)h(in)g (particular.)23 b(De\014ned)17 b(here)f(are)g(v)m(arious)g(ob)s(ject)g (repre-)75 1029 y(sen)o(tations,)d(as)h(w)o(ell)g(as)f(the)h(naming)g (con)o(v)o(en)o(tions)g(used)g(for)f(expressing)i(this)f(standard.)19 b(The)14 b(actual)75 1085 y(calling)j(sequences)f(are)f(de\014ned)i (elsewhere.)166 1144 y(It)23 b(is)g(exp)q(ected)h(that)e(an)o(y)g(F)l (ortran)g(90)g(and)h(C++)g(implemen)o(tations)h(use)f(the)g(F)l(ortran) f(77)75 1200 y(and)f(ANSI)h(C)e(bindings,)k(resp)q(ectiv)o(ely)l(.)39 b(Although)22 b(w)o(e)e(consider)i(it)f(premature)g(to)f(de\014ne)i (other)75 1257 y(bindings)e(to)d(F)l(ortran)g(90)g(and)h(C++,)h(the)f (curren)o(t)f(bindings)j(are)e(designed)h(to)e(encourage,)i(rather)75 1313 y(than)c(discourage,)g(exp)q(erimen)o(tation)i(with)e(b)q(etter)g (bindings)i(whic)o(h)f(migh)o(t)f(b)q(e)h(adopted)g(later.)166 1371 y(There)i(are)g(sev)o(eral)g(imp)q(ortan)o(t)f(language)h(binding) i(issues)f(not)e(addressed)i(b)o(y)e(this)i(standard.)75 1428 y(It)f(is)g(b)q(ey)o(ond)h(the)f(scop)q(e)g(of)g(this)g(standard)g (to)f(discuss)i(the)f(in)o(terop)q(erabilit)o(y)i(of)d(message)h (passing)75 1484 y(b)q(et)o(w)o(een)c(languages.)19 b(Although)c(it)e (w)o(ould)i(b)q(e)f(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)h(to)e (guaran)o(tee)g(that)f(messages)75 1541 y(sen)o(t)k(b)q(et)o(w)o(een)h (languages)f(w)o(ere)h(prop)q(erly)g(in)o(terpreted,)g(the)f(matc)o (hing)h(of)f(t)o(yp)q(es)g(across)g(languages)75 1597 y(prev)o(en)o(ts)f(the)h(inclusion)i(of)d(suc)o(h)h(a)f(guaran)o(tee)g (in)i(this)f(standard.)k(It)c(is)g(fully)h(exp)q(ected)g(that)d(man)o (y)75 1654 y(implemen)o(tations)i(will)g(ha)o(v)o(e)f(suc)o(h)g (features,)f(and)h(that)f(suc)o(h)h(features)g(are)f(a)h(sign)g(of)f (the)h(qualit)o(y)g(of)75 1710 y(the)g(implemen)o(tation.)166 1851 y Fv(Discussion:)f Fu(There)f(w)o(as)e(some)g(informal)d (discussion)k(of)f(this)g(issue)h(at)g(the)g(last)f(meeting.)16 b(Is)c(it)f(p)q(ossible)75 1908 y(to)j(sa)o(y)f(something)g(stronger?) 75 2145 y Fy(7.2)59 b(Data)19 b(T)-5 b(yp)r(es)19 b(and)g(Naming)g (Conventions)75 2250 y FE(All)c(named)f(constan)o(ts,)f(routine)h (names,)g(and)g(v)m(ariable)h(names)f(will)h(b)q(egin)g(with)f(the)g (letters)g(\\MPI".)75 2306 y(In)21 b(the)f(C)g(binding,)j(these)d(will) i(b)q(e)f(upp)q(er)g(case,)g(with)f(all)h(remaining)h(letters)e(in)h (lo)o(w)o(er)f(case.)34 b(In)75 2363 y(F)l(ortran,)14 b(the)h(en)o(tire)h(name)f(will)i(b)q(e)f(lo)o(w)o(er)e(case.)166 2421 y(Handles)e(are)g(represen)o(ted)f(as)h(in)o(tegers)f(in)h(F)l (ortran)f(and)g(as)g(p)q(oin)o(ters)h(to)f(structures)g(in)i(C.)d(T)o (yp)q(ed)75 2478 y(comm)o(unication)20 b(routines)g(supp)q(ort)g FF(string)p FE(s,)g FF(double)j(precision)p FE(,)e FF(real)p FE(,)f(and)g FF(in)o(teger)f FE(data)g(in)75 2534 y(F)l(ortran.)34 b(Null-terminated)23 b FF(string)p FE(s,)e FF(double)p FE(,)h FF(real)p FE(,)g FF(short)p FE(,)f FF(c)o(har)p FE(,)g(and)g FF(long)g FE(are)f(supp)q(orted)75 2591 y(in)g(C.)e(In)i(addition,)g(a)f(sp)q(ecial)h FF(b)o(yte)f FE(t)o(yp)q(e)g(is)g(supp)q(orted)g(in)h(b)q(oth)f(languages.)31 b(This)20 b(t)o(yp)q(e)f(allo)o(ws)75 2647 y(comm)o(unication)h(to)f(b) q(e)h(p)q(erformed)g(with)g(no)f(t)o(yp)q(e)h(con)o(v)o(ersion)g(at)f (all.)33 b(In)21 b(C,)d(the)i(de\014nitions)h(of)75 2704 y(named)14 b(constan)o(ts,)f(function)i(protot)o(yp)q(es,)f(and)g (aggregate)f(t)o(yp)q(e)h(information)g(m)o(ust)g(b)q(e)h(supplied)h (in)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 117 123 117 122 bop 75 -100 a Fx(7.3.)34 b(F)o(OR)l(TRAN)16 b(77)f(BINDING)g (ISSUES)948 b FE(117)290 81 y Fp(double)23 b(precision)f(a)290 138 y(call)h(foo\(a\))385 194 y(.)385 251 y(.)385 307 y(.)290 363 y(subroutine)f(foo\(b\))290 420 y(integer)h(b)75 568 y FE(Figure)e(7.1:)29 b(This)21 b(is)g(an)f(example)h(of)f(calling) i(a)e(routine)h(with)g(mismatc)o(hed)f(formal)g(and)h(actual)75 624 y(parameters.)75 765 y(an)15 b(include)j(\014le)e Fp(mpi.h)p FE(.)75 931 y Fy(7.3)59 b(F)n(o)n(rtran)21 b(77)e(Binding)h(Issues)75 1040 y FE(There)d(are)g(sev)o(eral)g(p)q (oin)o(ts)h(w)o(ere)f(this)g(standard)g(div)o(erges)h(from)e(the)h (ANSI)h(F)l(ortran)e(77)h(standard.)75 1097 y(Unless)j(explicitly)i (stated,)d(the)h(MPI)f(F77)f(binding)j(is)f(consisten)o(t)f(with)g (ANSI)h(standard)f(F)l(ortran)75 1153 y(77.)26 b(These)18 b(exceptions)g(are)g(consisten)o(t)f(with)h(common)f(practice)h(in)h (the)e(F)l(ortran)f(comm)o(unit)o(y)l(.)27 b(In)75 1210 y(particular:)143 1320 y FA(\017)c FE(Iden)o(ti\014ers)16 b(are)f(limited)i(to)e(31,)f(not)h(six,)g(signi\014can)o(t)h(c)o (haracters.)143 1429 y FA(\017)23 b FE(Iden)o(ti\014ers)16 b(ma)o(y)f(con)o(tain)g(underscores)h(after)e(the)h(\014rst)g(c)o (haracter.)143 1539 y FA(\017)23 b FE(The)14 b(t)o(yp)q(es)f(of)g(the)h (formal)f(parameter)g(list)i(ma)o(y)e(not)g(matc)o(h)g(the)h(calling)h (parameter)e(list.)20 b(\(An)189 1595 y(example)c(is)f(giv)o(en)h(in)g (\014gure)f(7.3.\))143 1705 y FA(\017)23 b FE(Although)16 b(not)g(required,)g(it)h(is)f(strongly)g(suggested)f(that)h(named)g (constan)o(ts)f(b)q(e)h(pro)o(vided)h(in)189 1761 y(an)e(include)i (\014le.)75 1977 y Fy(7.4)59 b(C)20 b(Binding)g(Issues)75 2086 y FE(W)l(e)j(use)g(the)f(ANSI)i(C)e(declaration)h(format.)41 b(All)24 b(MPI)f(names)f(ha)o(v)o(e)g(an)h Fp(MPI)p 1548 2086 15 2 v 39 w FE(pre\014x,)i(de\014ned)75 2142 y(constan)o(ts)12 b(are)h(in)h(all)h(capital)f(letters,)f(and)g(de\014ned)i(t)o(yp)q(es)e (ha)o(v)o(e)g(one)h(capital)g(letter)f(after)g(the)g(pre\014x.)75 2308 y Fy(7.5)59 b(Sp)r(eci\014c)19 b(Bindings)75 2500 y Fv(Discussion:)13 b Fu(What)c(w)o(ould)g(b)q(e)h(the)g(b)q(est)h(w)o (a)o(y)e(to)g(presen)o(t)i(the)f(follo)o(wing)d(information?)14 b(One)c(big)f(alphab)q(etical)75 2556 y(list?)16 b(In)10 b(order)h(of)e(presen)o(tation)i(in)e(the)i(individual)d(c)o(hapters?) 18 b(By)10 b(c)o(hapter)h(but)f(alphab)q(etical)f(within)g(c)o(hapter?) 75 2613 y(C)14 b(and)g(F)m(ortran)f(mixed)g(or)g(separate?)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 118 124 118 123 bop 75 -100 a FE(118)953 b Fx(SECTION)16 b(7.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fq(7.5.1)49 b(C)16 b(bindings)i(fo)o(r)e(P)o(oint-to-P)o(oint)h(Communication)75 138 y FE(These)f(are)e(in)i(alphab)q(etical)i(order.)75 199 y Fp(int)23 b(MPI)p 245 199 15 2 v 17 w(append)p 406 199 V 17 w(contiguous\()f(MPI)p 781 199 V 17 w(Bd)h(bd,)h(void*)f (start,)g(int)g(count,)393 255 y(MPI)p 468 255 V 17 w(Datatype)g (datatype)f(\))75 345 y(int)h(MPI)p 245 345 V 17 w(append)p 406 345 V 17 w(hindexed\()f(MPI)p 733 345 V 17 w(Bd)i(bd,)f(void*)g (start,)g(int)h(count,)393 402 y(int)g(*index)p 636 402 V 16 w(array,)f(MPI)p 891 402 V 17 w(Datatype)g(datatype)f(\))75 492 y(int)h(MPI)p 245 492 V 17 w(append)p 406 492 V 17 w(hvec\()g(MPI)p 638 492 V 17 w(Bd)g(bd,)h(void*)f(start,)g(int)g (count,)g(int)h(stride,)393 548 y(int)g(lenblk,)f(MPI)p 755 548 V 16 w(Datatype)g(datatype)g(\))75 639 y(int)g(MPI)p 245 639 V 17 w(append)p 406 639 V 17 w(indexed\()f(MPI)p 709 639 V 17 w(Bd)i(bd,)f(void*)g(start,)g(int)h(count,)f(int)g(*index) p 1705 639 V 17 w(array,)393 695 y(MPI)p 468 695 V 17 w(Datatype)g(datatype)f(\))75 785 y(int)h(MPI)p 245 785 V 17 w(append)p 406 785 V 17 w(vec\()g(MPI)p 614 785 V 17 w(Bd)g(bd,)h(int)f(count,)g(int)h(stride,)f(int)g(lenblk,)393 842 y(MPI)p 468 842 V 17 w(Datatype)g(datatype)f(\))75 932 y(int)h(MPI)p 245 932 V 17 w(associated\(MPI)p 598 932 V 15 w(Handle)h(handle,)e(MPI)p 1043 932 V 17 w(Handle)h(handle)p 1371 932 V 17 w(type)g(\))75 1022 y(int)g(MPI)p 245 1022 V 17 w(cancel\()g(MPI)p 525 1022 V 17 w(Handle)g(handle,)g(int)g(flag)h (\))75 1113 y(int)f(MPI)p 245 1113 V 17 w(commit)p 406 1113 V 17 w(buffer\()f(MPI)p 685 1113 V 17 w(Bd)i(bd)f(\))75 1203 y(int)g(MPI)p 245 1203 V 17 w(create)p 406 1203 V 17 w(status\()f(MPI)p 685 1203 V 17 w(Handle)h(handle)g(\))75 1293 y(int)g(MPI)p 245 1293 V 17 w(create)p 406 1293 V 17 w(buffer\()f(MPI)p 685 1293 V 17 w(Bd)i(*bd,)f(MPI)p 965 1293 V 17 w(LIFETIME)g(persistence)f(\))75 1383 y(int)h(MPI)p 245 1383 V 17 w(free\()g(MPI)p 477 1383 V 17 w(Handle)g(handle)g(\))75 1474 y(int)g(MPI)p 245 1474 V 17 w(get)p 334 1474 V 17 w(len\()g(int)h(count,)f(MPI)p 805 1474 V 17 w(Status)g(*status,)f(MPI) p 1275 1474 V 17 w(Bd)i(bd)f(\))75 1564 y(int)g(MPI)p 245 1564 V 17 w(init)p 358 1564 V 17 w(recv)g(\()h(MPI)p 614 1564 V 17 w(Handle)f(handle,)g(MPI)p 1061 1564 V 16 w(Bd)h(bd,)f(int)h(source,)f(MPI)p 1603 1564 V 17 w(Tag)g(tag,)393 1620 y(MPI)p 468 1620 V 17 w(Communicator)f(context,)h (MPI)p 1082 1620 V 17 w(LIFETIME)f(persistence)h(\))75 1710 y(int)g(MPI)p 245 1710 V 17 w(init)p 358 1710 V 17 w(rsend)g(\()h(MPI)p 638 1710 V 17 w(Handle)f(handle,)f(MPI)p 1084 1710 V 17 w(Bd)i(bd,)f(int)h(dest,)f(MPI)p 1579 1710 V 17 w(Tag)g(tag,)393 1767 y(MPI)p 468 1767 V 17 w(Communicator)f(context,)h(MPI)p 1082 1767 V 17 w(LIFETIME)f (persistence)h(\))75 1857 y(int)g(MPI)p 245 1857 V 17 w(init)p 358 1857 V 17 w(send)g(\()h(MPI)p 614 1857 V 17 w(Handle)f(handle,)g(MPI)p 1061 1857 V 16 w(Bd)h(bd,)f(int)h(dest,)f (MPI)p 1555 1857 V 17 w(Tag)g(tag,)393 1914 y(MPI)p 468 1914 V 17 w(COMunicator)f(context,)h(MPI)p 1058 1914 V 17 w(LIFETIME)g(persistence)f(\))75 2004 y(int)h(MPI)p 245 2004 V 17 w(init)p 358 2004 V 17 w(ssend)g(\()h(MPI)p 638 2004 V 17 w(Handle)f(handle,)f(MPI)p 1084 2004 V 17 w(Bd)i(bd,)f(int)h(dest,)f(MPI)p 1579 2004 V 17 w(Tag)g(tag,)393 2060 y(MPI)p 468 2060 V 17 w(Communicator)f(context,)h(MPI)p 1082 2060 V 17 w(LIFETIME)f(persistence)h(\))75 2151 y(int)g(MPI)p 245 2151 V 17 w(iprobe\()g(int)h(source,)e(MPI)p 811 2151 V 17 w(Tag)i(tag,)f(MPI)p 1115 2151 V 17 w(Communicator)f (context,)h(int)g(flag,)393 2207 y(type,)g(MPI)p 611 2207 V 17 w(Status)g(*status)g(\))75 2297 y(int)g(MPI)p 245 2297 V 17 w(irecv\()g(MPI)p 501 2297 V 17 w(Handle)g(handle,)g(MPI) p 948 2297 V 17 w(Bd)g(bd,)h(int)f(source,)g(MPI)p 1490 2297 V 17 w(Tag)g(tag,)393 2354 y(MPI)p 468 2354 V 17 w(Communicator)f(context)h(\))75 2444 y(int)g(MPI)p 245 2444 V 17 w(irecvc\()g(MPI)p 525 2444 V 17 w(Handle)g(handle,)g(void*)g (start,)g(int)h(count,)393 2500 y(MPI)p 468 2500 V 17 w(Datatype)f(datatype,)f(int)i(source,)f(MPI)p 1297 2500 V 16 w(Tag)h(tag,)393 2557 y(MPI)p 468 2557 V 17 w(Communicator)e (context,)h(MPI)p 1082 2557 V 17 w(Status)g(*status)g(\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(irsend)g(\()h(MPI)p 549 2647 V 17 w(Handle)f(handle,)g(MPI)p 996 2647 V 16 w(Bd)h(bd,)g(int)f(dest,)g(MPI)p 1490 2647 V 17 w(Tag)g(tag,)393 2704 y(MPI)p 468 2704 V 17 w(Communicator)f(context)h(\))1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 119 125 119 124 bop 75 -100 a Fx(7.5.)34 b(SPECIFIC)15 b(BINDINGS)1168 b FE(119)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(irsendc)g(\()h (MPI)p 573 45 V 17 w(Handle)f(handle,)g(void*)g(start,)g(int)g(count,) 393 102 y(MPI)p 468 102 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f (MPI)p 1249 102 V 17 w(Tag)g(tag,)393 158 y(MPI)p 468 158 V 17 w(Communicator)f(context)h(\))75 246 y(int)g(MPI)p 245 246 V 17 w(isend)g(\()h(MPI)p 525 246 V 17 w(Handle)f(handle,)g (MPI)p 972 246 V 17 w(Bd)g(bd,)h(int)f(dest,)g(MPI)p 1466 246 V 17 w(Tag)h(tag,)393 302 y(MPI)p 468 302 V 17 w(Communicator)e(context)h(\))75 390 y(int)g(MPI)p 245 390 V 17 w(isendc)g(\()h(MPI)p 549 390 V 17 w(Handle)f(handle,)g (void*)g(start,)g(int)g(count,)393 446 y(MPI)p 468 446 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f(MPI)p 1249 446 V 17 w(Tag)g(tag,)393 503 y(MPI)p 468 503 V 17 w(Communicator)f (context)h(\))75 591 y(int)g(MPI)p 245 591 V 17 w(issend)g(\()h(MPI)p 549 591 V 17 w(Handle)f(handle,)g(MPI)p 996 591 V 16 w(Bd)h(bd,)g(int)f(dest,)g(MPI)p 1490 591 V 17 w(Tag)g(tag,)393 647 y(MPI)p 468 647 V 17 w(Communicator)f(context)h(\))75 735 y(int)g(MPI)p 245 735 V 17 w(issendc)g(\()h(MPI)p 573 735 V 17 w(Handle)f(handle,)g(void*)g(start,)g(int)g(count,)393 791 y(MPI)p 468 791 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f(MPI)p 1249 791 V 17 w(Tag)g(tag,)393 848 y(MPI)p 468 848 V 17 w(Communicator)f(context)h(\))75 936 y(int)g(MPI)p 245 936 V 17 w(probe\()g(int)h(source,)f(MPI)p 788 936 V 16 w(Tag)h(tag,)f(MPI)p 1091 936 V 17 w(Communicator)f(context,)393 992 y(MPI)p 468 992 V 17 w(Datatype)h(datatype,)f(MPI)p 1010 992 V 17 w(Status)h(*status)g(\))75 1080 y(int)g(MPI)p 245 1080 V 17 w(query\()g(MPI)p 501 1080 V 17 w(Handle)g(handle,)g(int) g(count,)g(int)h(source,)f(MPI)p 1497 1080 V 17 w(Tag)g(tag\))75 1168 y(int)g(MPI)p 245 1168 V 17 w(recv\()g(MPI)p 477 1168 V 17 w(Bd)h(bd,)f(int)h(source,)f(MPI)p 1020 1168 V 16 w(Tag)h(tag,)f(MPI)p 1323 1168 V 17 w(Communicator)f(context,)393 1224 y(MPI)p 468 1224 V 17 w(Status)h(*status)g(\))75 1312 y(int)g(MPI)p 245 1312 V 17 w(recvc)g(\()h(void*)f(start,)g(int)h (count,)f(MPI)p 1098 1312 V 17 w(Datatype)f(datatype,)h(int)g(source,) 393 1368 y(MPI)p 468 1368 V 17 w(Tag)g(tag,)h(MPI)p 772 1368 V 16 w(Communicator)f(context,)f(MPI)p 1385 1368 V 17 w(Status)h(*status)g(\))75 1456 y(int)g(MPI)p 245 1456 V 17 w(rsend)g(\()h(MPI)p 525 1456 V 17 w(Bd)f(bd,)h(int)f(dest,)h (MPI)p 1020 1456 V 16 w(Tag)g(tag,)f(MPI)p 1323 1456 V 17 w(Communicator)f(context)h(\))75 1544 y(int)g(MPI)p 245 1544 V 17 w(rsendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1122 1544 V 16 w(Datatype)g(datatype,)g(int)g(dest,)393 1600 y(MPI)p 468 1600 V 17 w(Tag)g(tag,)h(MPI)p 772 1600 V 16 w(Communicator)f(context)g(\))75 1688 y(int)g(MPI)p 245 1688 V 17 w(send)h(\()f(MPI)p 501 1688 V 17 w(Bd)h(bd,)f(int)h (dest,)f(MPI)p 996 1688 V 16 w(Tag)h(tag,)f(MPI)p 1299 1688 V 17 w(Communicator)f(context)h(\))75 1776 y(int)g(MPI)p 245 1776 V 17 w(sendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1098 1776 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 1832 y(MPI)p 468 1832 V 17 w(Tag)g(tag,)h(MPI)p 772 1832 V 16 w(Communicator)f(context)g(\))75 1920 y(int)g(MPI)p 245 1920 V 17 w(ssend)g(\()h(MPI)p 525 1920 V 17 w(Bd)f(bd,)h(int)f (dest,)h(MPI)p 1020 1920 V 16 w(Tag)g(tag,)f(MPI)p 1323 1920 V 17 w(Communicator)f(context)h(\))75 2008 y(int)g(MPI)p 245 2008 V 17 w(ssendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1122 2008 V 16 w(Datatype)g(datatype,)g(int)g(dest,)393 2064 y(MPI)p 468 2064 V 17 w(Tag)g(tag,)h(MPI)p 772 2064 V 16 w(Communicator)f(context)g(\))75 2152 y(int)g(MPI)p 245 2152 V 17 w(start\()g(MPI)p 501 2152 V 17 w(Handle)g(handle)g(\))75 2240 y(int)g(MPI)p 245 2240 V 17 w(Status)g(\()h(MPI)p 549 2240 V 17 w(Handle)f(handle,)g(flag,)g(MPI)p 1139 2240 V 17 w(Status)g(*status)g(\))75 2327 y(int)g(MPI)p 245 2327 V 17 w(Statusany)g(\()h(MPI)p 621 2327 V 16 w(Handle)f(*handle)p 972 2327 V 17 w(list,)g(int)g(count,)g(int)h (index,)393 2384 y(MPI)p 468 2384 V 17 w(Status)f(*status)g(\))75 2472 y(int)g(MPI)p 245 2472 V 17 w(wait)h(\()f(MPI)p 501 2472 V 17 w(Handle)g(handle,)g(MPI)p 948 2472 V 17 w(Status)g(*status)g(\))75 2559 y(int)g(MPI)p 245 2559 V 17 w(waitall\()g(MPI)p 549 2559 V 17 w(Handle)g(*handle)p 901 2559 V 16 w(list,)g(int)h(count,)f(list)p 1419 2559 V 16 w(of)p 1483 2559 V 17 w(return)p 1644 2559 V 17 w(handles\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(waitany)g(\()h(MPI)p 573 2647 V 17 w(Handle)f(*handle)p 925 2647 V 16 w(list,)g(int)h (count,)f(int)g(index,)393 2704 y(MPI)p 468 2704 V 17 w(Status)g(*status)g(\))-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 120 126 120 125 bop 75 -100 a FE(120)953 b Fx(SECTION)16 b(7.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fq(7.5.2)49 b(C)16 b(Bindings)i(fo)o(r)e(Collective)h(Communication)g(Chapter)75 135 y FE(These)f(are)e(in)i(order)f(of)g(presen)o(tation)g(in)h(the)g (Collectiv)o(e)g(Comm)o(unication)g(c)o(hapter.)75 194 y Fp(int)23 b(MPI)p 245 194 15 2 v 17 w(comm)p 358 194 V 17 w(init\()g(MPI)p 590 194 V 17 w(Group)g(group)g(\))75 283 y(int)g(MPI)p 245 283 V 17 w(rank\()g(MPI)p 477 283 V 17 w(Group)g(group,)g(int)h(rank)f(\))75 372 y(int)g(MPI)p 245 372 V 17 w(gsize\()g(MPI)p 501 372 V 17 w(Group)g(group,)g(int)h (size)f(\))75 461 y(int)g(MPI)p 245 461 V 17 w(barrier\()g(MPI)p 549 461 V 17 w(Group)g(group)g(\))75 549 y(int)g(MPI)p 245 549 V 17 w(synch\()g(MPI)p 501 549 V 17 w(Group)g(group,)g(MPI)p 900 549 V 17 w(MAP)g(map)h(\))75 638 y(int)f(MPI)p 245 638 V 17 w(bcast\()g(MPI)p 501 638 V 17 w(Bd)h(bd,)f(MPI)p 757 638 V 17 w(Group)g(group,)g(int)h(root)f(\))75 727 y(int)g(MPI)p 245 727 V 17 w(bcastc\()g(buf,)g(int)h(len,)f(MPI)p 859 727 V 17 w(Datatype)g(datatype,)f(MPI)p 1401 727 V 17 w(Group)h(group,)393 783 y(int)h(root)f(\))75 872 y(int)g(MPI)p 245 872 V 17 w(gather\()g(MPI)p 525 872 V 17 w(Bd)g(inbd,)h(list)p 853 872 V 16 w(of)p 917 872 V 17 w(MPI)p 1006 872 V 17 w(Bd)g(outbds,)e(MPI)p 1357 872 V 17 w(Group)h(group,)g(int)h(root)393 929 y(\))75 1017 y(int)f(MPI)p 245 1017 V 17 w(gatherc\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)f(MPI)p 1408 1017 V 17 w(Datatype)g(datatype,)393 1074 y(MPI)p 468 1074 V 17 w(Group)g(group,)g(int)h(root)f(\))75 1163 y(int)g(MPI)p 245 1163 V 17 w(scatter\()g(list)p 573 1163 V 17 w(of)p 638 1163 V 17 w(MPI)p 727 1163 V 16 w(Bd)h(inbds,)f(MPI)p 1054 1163 V 17 w(Bd)g(outbd,)g(MPI)p 1381 1163 V 17 w(Group)g(group,)393 1219 y(int)h(root)f(\))75 1308 y(int)g(MPI)p 245 1308 V 17 w(scatterc\()g(void*)g(inbuf,)g(void*) g(outbuf,)g(int)h(len,)393 1364 y(MPI)p 468 1364 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1364 V 17 w(Group)h(group,)g(int)h (root)f(\))75 1453 y(int)g(MPI)p 245 1453 V 17 w(allscatter\()g(MPI)p 621 1453 V 16 w(Bd)h(inbds[],)f(MPI)p 996 1453 V 16 w(Bd)h(outbds[],)f (MPI)p 1395 1453 V 16 w(Group)h(group)f(\))75 1542 y(int)g(MPI)p 245 1542 V 17 w(allscatterc\()f(void*)i(inbuf,)f(void*)g(outbuf,)g(int) g(len,)393 1598 y(MPI)p 468 1598 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1598 V 17 w(Group)h(group)g(\))75 1687 y(int)g(MPI)p 245 1687 V 17 w(allcast\()g(MPI)p 549 1687 V 17 w(Bd)g(inbds[],)g(MPI)p 924 1687 V 17 w(Bd)g(outbds[],)g(MPI)p 1323 1687 V 17 w(Group)g(group)g(\))75 1776 y(int)g(MPI)p 245 1776 V 17 w(allcastc\()g(void*)g(inbuf,)g(void*)g(outbuf,)g(int)h(len,)393 1832 y(MPI)p 468 1832 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1832 V 17 w(Group)h(group)g(\))75 1921 y(int)g(MPI)p 245 1921 V 17 w(reduce\()g(MPI)p 525 1921 V 17 w(Bd)g(inbd,)h(MPI)p 829 1921 V 16 w(Bd)g(outbd,)f(MPI)p 1156 1921 V 17 w(Group)g(group,)g (int)h(root,)393 1977 y(MPI)p 468 1977 V 17 w(Op)g(op)f(\))75 2066 y(int)g(MPI)p 245 2066 V 17 w(reducec\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)f(MPI)p 1408 2066 V 17 w(Datatype)g(datatype,)393 2123 y(MPI)p 468 2123 V 17 w(Group)g(group,)g(int)h(root,)f(MPI)p 1106 2123 V 17 w(Op)g(op)h(\))75 2211 y(int)f(MPI)p 245 2211 V 17 w(user)p 358 2211 V 17 w(reduce\()g(MPI)p 638 2211 V 17 w(Bd)g(inbd,)g(MPI)p 941 2211 V 17 w(Bd)h(outbd,)f(MPI)p 1269 2211 V 16 w(Group)h(group,)f(int)g(root,)393 2268 y(void)g(*userfunc\(\),)g(int)g(unitsize)g(\))75 2357 y(int)g(MPI)p 245 2357 V 17 w(user)p 358 2357 V 17 w(reducec\()g(void*) g(inbuf,)g(void*)g(outbuf,)g(int)g(len,)393 2413 y(MPI)p 468 2413 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2413 V 17 w(Group)h(group,)g(int)h(root,)393 2470 y(void)f(*userfunc\(\),)g (unitsize)f(\))75 2558 y(int)h(MPI)p 245 2558 V 17 w(allreduce\()g(MPI) p 597 2558 V 16 w(Bd)h(inbd,)f(MPI)p 900 2558 V 17 w(Bd)h(outbd,)f(MPI) p 1228 2558 V 16 w(Group)h(group,)f(MPI)p 1627 2558 V 16 w(Op)h(op)g(\))75 2647 y(int)f(MPI)p 245 2647 V 17 w(allreducec\()g(void*)g(inbuf,)g(void*)g(outbuf,)g(int)g(len,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2704 V 17 w(Group)h(group,)g(MPI)p 1409 2704 V 17 w(Op)h(op)f(\))1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 121 127 121 126 bop 75 -100 a Fx(7.5.)34 b(SPECIFIC)15 b(BINDINGS)1168 b FE(121)75 45 y Fp(int)23 b(MPI)p 245 45 15 2 v 17 w(user)p 358 45 V 17 w(allreduce\()f(MPI)p 709 45 V 17 w(Bd)i(inbd,)f(MPI)p 1013 45 V 17 w(Bd)g(outbd,)g(MPI)p 1340 45 V 17 w(Group)g(group,)393 102 y(void)g(*userfunc\(\)\))75 189 y(int)g(MPI)p 245 189 V 17 w(user)p 358 189 V 17 w(allreducec\()f(void*)h(inbuf,)g(void*) h(outbuf,)f(int)g(len,)393 246 y(MPI)p 468 246 V 17 w(Datatype)g (datatype,)f(MPI)p 1010 246 V 17 w(Group)h(group,)g(void)h (*userfunc\(\)\))75 333 y(int)f(MPI)p 245 333 V 17 w(scan\()g(MPI)p 477 333 V 17 w(Bd)h(inbd,)f(MPI)p 781 333 V 17 w(Bd)g(outbd,)g(MPI)p 1108 333 V 17 w(Group)g(group,)g(MPI)p 1507 333 V 17 w(Op)h(op)f(\))75 421 y(int)g(MPI)p 245 421 V 17 w(scanc\()g(void*)g (inbuf,)g(void*)h(outbuf,)f(int)g(len,)g(MPI)p 1360 421 V 17 w(Datatype)g(datatype,)393 477 y(MPI)p 468 477 V 17 w(Group)g(group,)g(MPI)p 867 477 V 17 w(Op)h(op)f(\))75 565 y(int)g(MPI)p 245 565 V 17 w(user)p 358 565 V 17 w(scan\()g(MPI)p 590 565 V 17 w(Bd)g(inbd,)h(MPI)p 894 565 V 16 w(Bd)g(outbd,)f(MPI)p 1221 565 V 17 w(Group)g(group,)393 621 y(void)g(*userfunc\(\))g(\))75 709 y(int)g(MPI)p 245 709 V 17 w(user)p 358 709 V 17 w(scanc\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)393 765 y(MPI)p 468 765 V 17 w(Datatype)f (datatype,)f(MPI)p 1010 765 V 17 w(Group)h(group,)g(void)h (*userfunc\(\)\))75 924 y Fq(7.5.3)49 b(F)o(o)o(rtran)15 b(Bindings)j(fo)o(r)d(P)o(oint-to-P)o(oint)i(Routines)75 1012 y Fp(mpi)p 150 1012 V 17 w(append)p 311 1012 V 16 w(contiguous\()22 b(bd,)i(start,)f(count,)g(datatype,)g(rc)g(\))170 1068 y(real)h(start\(*\))170 1125 y(integer)f(bd,)h(count,)f(datatype,) f(rc)75 1212 y(mpi)p 150 1212 V 17 w(append)p 311 1212 V 16 w(hindexed\()h(bd,)g(start,)g(count,)g(index)p 1115 1212 V 17 w(array,)g(datatype,)f(rc)i(\))170 1269 y(real)g(start\(*\)) 170 1325 y(integer)f(index)p 484 1325 V 17 w(array\(*\))170 1382 y(integer)g(bd,)h(count,)f(datatype,)f(rc)75 1469 y(mpi)p 150 1469 V 17 w(append)p 311 1469 V 16 w(hvec\()h(bd,)h(start,) f(count,)g(stride,)g(lenblk,)g(datatype,)f(rc)i(\))170 1526 y(real)g(start\(*\))170 1582 y(integer)f(bd,)h(count,)f(stride,)g (lenblk,)g(datatype,)f(rc)75 1670 y(mpi)p 150 1670 V 17 w(append)p 311 1670 V 16 w(indexed\()h(bd,)g(start,)g(count,)g (index)p 1091 1670 V 17 w(array,)g(datatype,)g(rc)g(\))170 1726 y(real)h(start\(*\))170 1783 y(integer)f(index)p 484 1783 V 17 w(array\(*\))170 1839 y(integer)g(bd,)h(count,)f (datatype,)f(rc)75 1927 y(mpi)p 150 1927 V 17 w(append)p 311 1927 V 16 w(vec\()h(bd,)h(count,)f(stride,)g(lenblk,)g(datatype,)f (rc)i(\))170 1983 y(integer)f(bd,)h(count,)f(stride,)g(lenblk,)g (datatype,)f(rc)75 2071 y(mpi)p 150 2071 V 17 w(associated\()g(handle,) h(handle)p 788 2071 V 16 w(type,)h(rc)f(\))170 2127 y(integer)g (handle,)g(handle)p 699 2127 V 17 w(type,)g(rc)75 2215 y(mpi)p 150 2215 V 17 w(cancel\()g(handle,)g(flag,)g(rc)g(\))170 2271 y(integer)g(handle,)g(flag,)g(rc)75 2359 y(mpi)p 150 2359 V 17 w(commit)p 311 2359 V 16 w(buffer\()g(bd,)g(rc)h(\))170 2415 y(integer)f(bd,)h(rc)75 2503 y(mpi)p 150 2503 V 17 w(create)p 311 2503 V 16 w(buffer\()f(bd,)g(persistence,)g(rc)g(\)) 170 2560 y(integer)g(bd,)h(persistence,)e(rc)75 2647 y(mpi)p 150 2647 V 17 w(create)p 311 2647 V 16 w(status\()h(handle,)g (rc)h(\))170 2704 y(integer)f(handl,)g(rce)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 122 128 122 127 bop 75 -100 a FE(122)953 b Fx(SECTION)16 b(7.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(mpi)p 150 45 15 2 v 17 w(free\()23 b(handle,)g(rc)g(\))170 102 y(integer)g(handl,)g (rce)75 190 y(mpi)p 150 190 V 17 w(get)p 239 190 V 17 w(len\()g(count,)g(status,)g(bd,)g(rc)h(\))170 247 y(integer)f(count,)g (status,)g(bd,)h(rc)75 335 y(mpi)p 150 335 V 17 w(init)p 263 335 V 16 w(recv)g(\()f(handle,)g(bd,)h(source,)f(tag,)g(context,)g (persistence,)f(rc)i(\))170 392 y(integer)f(handle,)g(bd,)h(source,)f (tag,)g(context,)g(persistence,)f(rc)75 480 y(mpi)p 150 480 V 17 w(init)p 263 480 V 16 w(rsend)i(\()f(handle,)g(bd,)h(dest,)f (tag,)g(context,)g(persistence,)f(rc)i(\))170 537 y(integer)f(handle,)g (bd,)h(dest,)f(tag,)g(context,)g(persistence,)f(rc)75 625 y(mpi)p 150 625 V 17 w(init)p 263 625 V 16 w(send)i(\()f(handle,)g (bd,)h(dest,)f(tag,)g(context,)g(persistence,)f(rc)i(\))170 681 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g(persistence,) f(rc)75 770 y(mpi)p 150 770 V 17 w(init)p 263 770 V 16 w(ssend)i(\()f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g(persistence,)f (rc)i(\))170 826 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g (persistence,)f(rc)75 915 y(mpi)p 150 915 V 17 w(iprobe\()h(source,)g (tag,)g(context,)g(flag,)g(type,)g(status,)g(rc)h(\))170 971 y(integer)f(source,)g(tag,)h(context,)e(flag,)i(type,)f(status,)g (rc)75 1060 y(mpi)p 150 1060 V 17 w(irecv\()g(handle,)g(bd,)g(source,)g (tag,)g(context,)g(rc)h(\))170 1116 y(integer)f(handle,)g(bd,)h (source,)f(tag,)g(context,)g(rc)75 1205 y(mpi)p 150 1205 V 17 w(irecvc\()g(handle,)g(start,)g(count,)g(datatype,)f(source,)h (tag,)h(context,)e(status,)393 1261 y(rc)i(\))170 1318 y(real)g(start\(*\))170 1374 y(integer)f(handle,)g(count,)g(datatype,)g (source,)g(tag,)g(context,)g(status,)g(rc)75 1463 y(mpi)p 150 1463 V 17 w(irsend)g(\()h(handle,)e(bd,)i(dest,)f(tag,)g(context,)g (rc)h(\))170 1519 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g (rc)75 1608 y(mpi)p 150 1608 V 17 w(irsendc)g(\()g(handle,)g(start,)g (count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)i(\))170 1664 y(real)g(start\(*\))170 1721 y(integer)f(handle,)g(count,)g (datatype,)g(dest,)g(tag,)g(context,)g(rc)75 1809 y(mpi)p 150 1809 V 17 w(isend)g(\()h(handle,)f(bd,)g(dest,)g(tag,)h(context,)e (rc)i(\))170 1866 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g (rc)75 1954 y(mpi)p 150 1954 V 17 w(isendc)g(\()h(handle,)e(start,)h (count,)h(datatype,)e(dest,)h(tag,)h(context,)e(rc)i(\))170 2011 y(real)g(start\(*\))170 2067 y(integer)f(handle,)g(count,)g (datatype,)g(dest,)g(tag,)g(context,)g(rc)75 2156 y(mpi)p 150 2156 V 17 w(issend)g(\()h(handle,)e(bd,)i(dest,)f(tag,)g(context,)g (rc)h(\))170 2212 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(contex,)g (rct)75 2301 y(mpi)p 150 2301 V 17 w(issendc)g(\()g(handle,)g(start,)g (count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)i(\))170 2357 y(real)g(start\(*\))170 2414 y(integer)f(handle,)g(count,)g (datatype,)g(dest,)g(tag,)g(context,)g(rc)75 2502 y(mpi)p 150 2502 V 17 w(probe\()g(source,)g(tag,)g(context,)g(datatype,)f (status,)h(rc)h(\))170 2559 y(integer)f(source,)g(tag,)h(context,)e (datatype,)h(status,)g(rc)75 2647 y(mpi)p 150 2647 V 17 w(query\()g(handle,)g(count,)g(source,)g(tag,)g(rc)h(\))170 2704 y(integer)f(handle,)g(count,)g(source,)g(tag,)h(rc)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 123 129 123 128 bop 75 -100 a Fx(7.5.)34 b(SPECIFIC)15 b(BINDINGS)1168 b FE(123)75 45 y Fp(mpi)p 150 45 15 2 v 17 w(recv\()23 b(bd,)g(source,)g(tag,)h(context,)e(status,)h(rc)h(\))170 102 y(integer)f(bd,)h(source,)f(tag,)g(context,)g(status,)g(rc)75 188 y(mpi)p 150 188 V 17 w(recvc)g(\(start,)g(count,)g(datatype,)g (source,)f(tag,)i(context,)f(statu,)g(rcs\))170 244 y(real)h (start\(*\))170 301 y(integer)f(count,)g(datatype,)g(source,)g(tag,)g (context,)g(status,)g(rc)75 387 y(mpi)p 150 387 V 17 w(rsend)g(\()h(bd,)f(dest,)g(tag,)h(context,)e(rc)i(\))170 444 y(integer)f(bd,)h(dest,)f(tag,)g(context,)g(rc)75 530 y(mpi)p 150 530 V 17 w(rsendc)g(\(start,)g(count,)g(datatype,)f (dest,)i(tag,)f(context,)g(rc)g(\))170 587 y(real)h(start\(*\))170 643 y(integer)f(count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)75 730 y(mpi)p 150 730 V 17 w(send)h(\()h(bd,)f(dest,)g(tag,)h(context,)f (rc)g(\))170 786 y(integer)g(bd,)h(dest,)f(tag,)g(context,)g(rc)75 873 y(mpi)p 150 873 V 17 w(sendc)g(\(start,)g(count,)g(datatype,)g (dest,)g(tag,)g(context,)g(rc)g(\))170 929 y(real)h(start\(*\))170 985 y(integer)f(count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)75 1072 y(mpi)p 150 1072 V 17 w(ssend)h(\()h(bd,)f(dest,)g(tag,)h (context,)e(rc)i(\))170 1128 y(integer)f(bd,)h(dest,)f(tag,)g(context,) g(rc)75 1215 y(mpi)p 150 1215 V 17 w(ssendc)g(\(start,)g(count,)g (datatype,)f(dest,)i(tag,)f(context,)g(rc)g(\))170 1271 y(real)h(start\(*\))170 1328 y(integer)f(count,)g(datatype,)g(dest,)g (tag,)h(context,)e(rc)75 1414 y(mpi)p 150 1414 V 17 w(start\()h (handle,)g(rc)g(\))170 1471 y(integer)g(handle,)g(rc)75 1557 y(mpi)p 150 1557 V 17 w(status)g(\()h(handle,)e(flag,)i(status,)f (rc)g(\))170 1614 y(integer)g(handle,)g(flag,)g(status,)g(rc)75 1700 y(mpi)p 150 1700 V 17 w(statusany)f(\()i(handle)p 597 1700 V 16 w(list,)g(count,)f(index,)g(status,)g(rc)g(\))170 1756 y(integer)g(handle)p 508 1756 V 17 w(list\(*\))170 1813 y(integer)g(count,)g(index,)g(status,)g(rc)75 1899 y(mpi)p 150 1899 V 17 w(wait)g(\()h(handle,)f(status,)g(rc)g(\))170 1956 y(integer)g(handle,)g(status,)g(rc)75 2042 y(mpi)p 150 2042 V 17 w(waitall\()g(handle)p 526 2042 V 16 w(list,)g(count,)g (list)p 948 2042 V 17 w(of)p 1013 2042 V 17 w(return)p 1174 2042 V 16 w(handles,)g(rc)g(\))170 2099 y(integer)g(handle)p 508 2099 V 17 w(list\(*\))170 2155 y(integer)g(count)170 2212 y(integer)g(list)p 460 2212 V 17 w(of)p 525 2212 V 17 w(return)p 686 2212 V 16 w(handles\(*,)g(rc\))75 2298 y(mpi)p 150 2298 V 17 w(waitany)g(\()g(handle)p 549 2298 V 17 w(list,)g(count,)g(index,)g(status,)g(rc)h(\))170 2355 y(integer)f(handle)p 508 2355 V 17 w(list\(*\))170 2411 y(integer)g(count,)g(index,)g(status,)g(rc)75 2561 y Fq(7.5.4)49 b(F)o(o)o(rtran)15 b(Bindings)j(fo)o(r)d(Collective)i (Communciation)g(Chapter)75 2647 y FE(These)f(are)e(not)h(done)h(y)o (et.)75 2704 y Fp(mpi)p 150 2704 V 17 w(comm)p 263 2704 V 16 w(init\()24 b(group,)f(rc)g(\))-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 124 130 124 129 bop 75 -100 a FE(124)953 b Fx(SECTION)16 b(7.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fp(mpi)p 150 45 15 2 v 17 w(rank\()23 b(group,)g(rank,)g(rc)h(\))75 132 y(mpi)p 150 132 V 17 w(gsize\()f(group,)g(size,)g(rc)h(\))75 218 y(mpi)p 150 218 V 17 w(barrier\()f(group,)g(rc)g(\))75 304 y(mpi)p 150 304 V 17 w(synch\()g(group,)g(map,)g(rc)h(\))75 391 y(mpi)p 150 391 V 17 w(bcast\()f(buffer)p 478 391 V 16 w(handle,)g(group,)g(root,)g(rc)h(\))75 477 y(mpi)p 150 477 V 17 w(bcastc\()f(buf,)g(len,)g(type,)g(group,)h(root,)f(rc)g (\))75 564 y(mpi)p 150 564 V 17 w(gather\()g(inhandle,)f(list)p 692 564 V 17 w(of)p 757 564 V 17 w(outhandles,)g(group,)h(root,)g(rc)h (\))75 650 y(mpi)p 150 650 V 17 w(gatherc\()f(inbuf,)g(outbuf,)f (inlen,)i(type,)f(group,)g(root,)g(rc)h(\))75 737 y(mpi)p 150 737 V 17 w(scatter\()f(list)p 478 737 V 16 w(of)p 542 737 V 17 w(inhandles,)g(outhandle,)f(group,)h(root,)g(rc)h(\))75 823 y(mpi)p 150 823 V 17 w(scatterc\()e(inbuf,)h(outbuf,)g(len,)h (type,)f(group,)g(root,)g(rc)h(\))75 910 y(mpi)p 150 910 V 17 w(allscatter\()e(list)p 549 910 V 17 w(of)p 614 910 V 17 w(inhandles,)g(list)p 989 910 V 17 w(of)p 1054 910 V 17 w(outhandles,)g(group,)h(rc)h(\))75 996 y(mpi)p 150 996 V 17 w(allscatterc\()e(inbuf,)h(outbuf,)g(len,)g(type,) g(grou,)h(rcp\))75 1083 y(mpi)p 150 1083 V 17 w(allcast\()f(inhandle,)f (list)p 716 1083 V 17 w(of)p 781 1083 V 17 w(outhandles,)g(group,)h(rc) h(\))75 1169 y(mpi)p 150 1169 V 17 w(allcastc\()e(inbuf,)h(outbuf,)g (len,)h(type,)f(grou,)g(rcp\))75 1255 y(mpi)p 150 1255 V 17 w(reduce\()g(inhandle,)f(outhandle,)h(group,)g(root,)g(op,)g(rc)h (\))75 1342 y(mpi)p 150 1342 V 17 w(reducec\()f(inbuf,)g(outbuf,)f (len,)i(type,)f(group,)g(root,)g(op,)h(rc)f(\))75 1428 y(mpi)p 150 1428 V 17 w(user)p 263 1428 V 16 w(reduce\()g(inhandle,)g (outhandle,)f(group,)h(root,)h(function,)e(unitsize,)h(rc)h(\))75 1515 y(mpi)p 150 1515 V 17 w(user)p 263 1515 V 16 w(reducec\()f(inbuf,) g(outbuf,)g(len,)g(type,)h(group,)f(root,)g(function,)393 1571 y(unitsize,)g(rc)g(\))75 1658 y(mpi)p 150 1658 V 17 w(allreduce\()f(inhandle,)h(outhandle,)f(group,)h(op,)h(rc)f(\))75 1744 y(mpi)p 150 1744 V 17 w(allreducec\()f(inbuf,)h(outbuf,)g(len,)g (type,)h(group,)f(op,)g(rc)h(\))75 1831 y(mpi)p 150 1831 V 17 w(user)p 263 1831 V 16 w(allreduce\()f(inhandle,)g(outhandle,)f (group,)h(function,)g(rc)g(\))75 1917 y(mpi)p 150 1917 V 17 w(user)p 263 1917 V 16 w(allreducec\()g(inbuf,)g(outbuf,)g(len,)g (type,)g(group,)g(function,)g(rc)g(\))75 2004 y(mpi)p 150 2004 V 17 w(scan\()g(inhandle,)g(outhandle,)f(group,)h(op,)h(rc)f (\))75 2090 y(mpi)p 150 2090 V 17 w(scanc\()g(inbuf,)g(outbuf,)g(len,)g (type,)g(group,)g(op,)h(rc)f(\))75 2177 y(mpi)p 150 2177 V 17 w(user)p 263 2177 V 16 w(scan\()h(inhandle,)e(outhandle,)h(group,) g(function,)f(rc)i(\))75 2263 y(mpi)p 150 2263 V 17 w(user)p 263 2263 V 16 w(scanc\()f(inbuf,)g(outbuf,)g(len,)h(type,)f(group,)g (function,)g(rc)g(\))1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 125 131 125 130 bop 75 357 a Fz(Section)35 b(8)75 565 y FG(En)m(vironmen)m(tal) 41 b(Managemen)m(t)f(and)75 690 y(Inquiry)75 932 y Fy(8.1)59 b(Intro)r(duction)75 1034 y FE(This)17 b(c)o(hapter)e(con)o(tains)h (MPI-sp)q(eci\014c)i(routines)f(for)e(in)o(teracting)i(with)f(the)g(en) o(vironmen)o(t)g(in)h(whic)o(h)75 1091 y(MPI)12 b(programs)e(run.)19 b(The)12 b(c)o(hapter)g(op)q(ens)g(with)g(a)g(discussion)h(of)f(MPI)f (routines)i(for)e(initialization)j(of)75 1147 y(the)g(MPI)h(en)o (vironmen)o(t.)20 b(The)14 b(c)o(hapter)h(includes)h(a)e(large)h(n)o (um)o(b)q(er)f(of)g(enquiry)i(functions,)f(b)o(y)f(whic)o(h)75 1204 y(an)h(MPI)f(program)g(can)h(determine)g(features)g(of)f(the)h(en) o(vironmen)o(t)g(in)g(whic)o(h)h(it)f(is)g(running)h(in)f(order)75 1260 y(to)g(tune)i(itself.)23 b(There)17 b(is)f(also)h(a)e(discussion)j (of)e(functions)h(that)e(are)h(not)g(sp)q(eci\014c)i(to)d(MPI,)h(suc)o (h)g(as)75 1317 y(timing)d(functions,)g(that)e(ma)o(y)h(b)q(e)h(pro)o (vided)g(b)o(y)f(other)g(standards)f(\(e.g.)18 b(POSIX\).)13 b(F)l(or)e(completeness,)75 1373 y(w)o(e)k(discuss)h(signals,)g (although)f(p)q(erhaps)h(this)g(b)q(elongs)g(in)g(the)f(c)o(hapter)g (on)g(correctness.)75 1499 y Fq(8.1.1)49 b(Program)16 b(Sta)o(rtup)f(Issues)75 1586 y FE(The)21 b(MPI)f(standard)g(should)i (pro)o(vide)f(enough)g(information)g(to)e(sp)q(ecify)j(an)f(en)o(tire,) h(source)e(co)q(de)75 1643 y(p)q(ortable)12 b(program.)18 b(T)l(o)12 b(do)f(this,)i(it)f(is)g(necessary)g(to)g(sp)q(ecify)h(ho)o (w)e(parallel)i(programs)e(start)g(and)h(end.)75 1699 y(This)f(is)f(the)h(analogue)f(of)g Fp(PROGRAM)23 b(MAIN)g(...)47 b(END)10 b FE(in)h(F)l(ortran)e(and)h Fp(main\(argc,argv\))p FA(f)p Fp(...exit\(rc\))o(;)p FA(g)p FE(.)75 1756 y(This)18 b(prop)q(osal)h(do)q(es)f(not)f(deal)i(with)f(metho)q(ds)g(for)g(spa)o (wning)g(tasks)f(during)i(the)f(execution)h(of)e(an)75 1812 y(MPI)g(program,)e(or)h(for)g(handling)i(explicitly)i(MIMD)c(mo)q (dels)h(\(b)q(ey)o(ond)g(the)g(usual)g(w)o(a)o(y)e(of)i(starting)75 1869 y(a)h(T)l(uring)i(mac)o(hine)f(on)g(all)h(no)q(des)f(and)g (sending)h(the)f(appropriate)f(tap)q(e\).)31 b(This)19 b(is)g(not)f(mean)o(t)h(to)75 1925 y(suggest)11 b(that)h(MPI)f(not)h (ha)o(v)o(e)f(suc)o(h)i(in)o(terfaces,)f(just)g(that)f(w)o(e)h(are)f (only)i(prop)q(osing)f(an)g(SPMD)f(mo)q(del.)166 1982 y(It)g(should)i(b)q(e)f(p)q(ossible)h(to)e(write)h(a)f(program)f(that)h (is)h(source-language)g(p)q(ortable)g(to)f(an)o(y)g(system,)75 2039 y(particularly)k(if)f(it)g(requires)h(only)f(a)g(simple)h(in)o (terface)f(to)f(the)h(op)q(erating)g(system.)19 b(An)14 b(example)h(is)f(an)75 2095 y(en)o(vironmen)o(t)i(that)f(starts)g(up)h (a)f(program)g(with)h(a)g(\014xed)g(n)o(um)o(b)q(er)g(of)g(pro)q (cessors.)21 b(Without)16 b(this,)g(it)75 2152 y(is)g(not)f(p)q (ossible)h(to)f(write)g Ft(any)k FE(p)q(ortable)d(programs.)166 2209 y(Ho)o(w)o(ev)o(er,)d(no)i(standard)g(should)h(restrict)e (programs)g(to)g(a)h(particular)g(in)o(terface.)20 b(F)l(or)14 b(example,)75 2265 y(some)h(applications)i(ma)o(y)e(w)o(an)o(t)f(to)h (dynamically)i(add)f(and)g(remo)o(v)o(e)e(tasks)h(during)h(the)g (execution)g(of)75 2322 y(an)d(application.)21 b(W)l(e)13 b(lea)o(v)o(e)h(it)f(to)f(others)h(on)g(the)h(committee)f(to)f(propse)i (a)e(mec)o(hanism)i(for)f(this)g(case.)166 2379 y(Belo)o(w,)i(w)o(e)g (discuss)h(three)f(di\013eren)o(t)g(approac)o(hes;)f(w)o(e)h(note)g (that)f(all)i(of)f(these)g(ha)o(v)o(e)f(b)q(een)i(tried)75 2436 y(in)g(some)f(existing)h(system)f(or)g(systems.)75 2560 y Fo(MPI)p 160 2560 14 2 v 16 w(INIT)75 2647 y FE(The)c(seemingly) i(simplest)f(approac)o(h)f(is)g(to)f(use)i(an)f(explicit)i (initialization)h(routine,)e Fp(MPI)p 1618 2647 15 2 v 17 w(Init)p FE(.)18 b(There)75 2704 y(are)h(sev)o(eral)g(problems)h (with)f(this)h(approac)o(h.)31 b(The)20 b(ma)s(jor)d(one)j(is)f(that)g (the)g(parallel)i(part)d(of)h(the)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 126 132 126 131 bop 75 -100 a FE(126)325 b Fx(SECTION)16 b(8.)34 b(ENVIR)o(ONMENT)l(AL)17 b(MANA)o(GEMENT)d(AND)h(INQUIR)l(Y)75 45 y FE(program)g(is)j(the)e(co)q(de)h(that)f(\\follo)o(ws")g(the)h Fp(MPI)p 942 45 15 2 v 17 w(Init)p FE(.)23 b(This)17 b(is)h(p)q(oten)o(tially)g(error)d(prone;)i(further,)75 102 y(it)e(can)h(b)q(e)g(di\016cult)g(to)f(implemen)o(t)h(on)f(some)g (systems.)75 227 y Fo(MPI)p 160 227 14 2 v 16 w(MAIN)75 315 y FE(Another)i(approac)o(h)g(is)h(to)f(insist)h(that)e(the)i(en)o (tire)f(program)g(b)q(e)h(run)f(in)h(parallel;)i(a)d(simple)h(w)o(a)o (y)f(to)75 371 y(do)h(this)h(is)h(to)d(replace)j(the)f(\\main")f (program)f(with)i(a)g(routine.)30 b(In)19 b(C,)f(this)h(could)h(b)q(e)f Fp(MPI)p 1742 371 15 2 v 17 w(Main\()75 428 y(argc,)k(argv)g(\))p FE(.)d(The)15 b(linking)i(step)d(then)h(pro)o(vides)g(a)g Fp(main)f FE(that)g(handles)h(all)h(appropriate)f(startup)75 484 y(\(including)h(the)e(pro)q(cessing)g(of)f(command)g(line)j (argumen)o(ts)c(where)i(appropriate)f(for)g(information)h(on)75 541 y(the)h(n)o(um)o(b)q(er)h(of)f(tasks)f(etc.\))20 b(and)15 b(then)g(calls)i(the)e(user's)g Fp(MPI)p 1168 541 V 17 w(main)f FE(routine.)166 598 y(Note)h(that)f(the)i Fp(MPI)p 528 598 V 16 w(main)f FE(in)o(terface)g(can)h(b)q(e)g (implemen)o(ted)h(with)e Fp(MPI)p 1429 598 V 17 w(call)f FE(through)147 710 y Fp(int)23 b(main\()g(argc,)g(argv)h(\))147 767 y(int)f(argc;)147 823 y(char)g(**argv;)147 880 y({)147 936 y(return)g(MPI_call\()f(argc,)h(argv,)h(MPI_main)e(\);)147 993 y(})75 1118 y Fo(MPI)p 160 1118 14 2 v 16 w(CALL)75 1206 y FE(Another)17 b(approac)o(h)f(is)h(to)f(pro)o(vide)h(a)f (routine)h(that)f(runs)h(a)f(user-de\014ned)j(routine)e(in)g(parallel.) 26 b(F)l(or)75 1262 y(example,)19 b Fp(MPI)p 344 1262 15 2 v 17 w(Call\()k(argc,)g(argv,)g(routine)g(\))18 b FE(can)g(b)q(e)h(used)f(to)f(run)i Fp(routine\()j(argc,)i(argv)75 1319 y(\))15 b FE(in)i(parallel.)23 b(A)16 b(generalization)h(of)e (this)h(w)o(ould)g(allo)o(w)g(the)g(running)h(of)e(a)g(particular)h (routine)h(on)e(a)75 1375 y(de\014ned)i(group:)k Fp(MPI)p 455 1375 V 17 w(Call\()i(groupid,)g(argc,)g(argv,)g(routine)g(\))p FE(.)f(This)16 b(is)h(more)e(\015exible)j(than)75 1432 y(the)g Fp(MPI)p 231 1432 V 17 w(Main)g FE(approac)o(h)g(but)g(retains) h(the)f(simplicit)o(y)j(of)d(running)h(a)f(w)o(ell-de\014ned)j(program) c(unit)75 1488 y(\(the)e(routine\))g(in)h(parallel.)166 1546 y(Note)g(that)f(in)i(this)f(case,)g(program)f(termination)h(is)g (handled)i(b)o(y)e(ha)o(ving)g(\(a\))f(the)h(routine)h(exit)75 1602 y(and)e(\(b\))g(the)g(calling)i(routine)f(exit.)k(F)l(or)15 b(example)170 1701 y Fp(irc)24 b(=)g(MPI_Call\()e(argc,)h(argv,)h (worker)f(\);)170 1757 y(exit\()h(irc)f(\);)75 1814 y(...)170 1870 y(int)h(worker\()f(argc,)g(argv)g(\))170 1926 y(int)h(argc;)f (char)g(**argv;)170 1983 y({)75 2039 y(...)170 2096 y(if)h(\(error)f (condition\))g(return)g(1;)170 2152 y(...)170 2209 y(return)g(0;)170 2265 y(})166 2363 y FE(All)16 b(three)g(of)e(these)i(approac)o(hes)f (are)g(curren)o(tly)g(in)h(use.)166 2420 y(These)i(are)g(not)g(the)g (most)f(general)i(metho)q(ds.)29 b(F)l(or)17 b(example,)i(these)g(pro)o (vide)f(no)g(mec)o(hanism)75 2477 y(for)c(dynamically)i(creating)f (\(remote)e(spa)o(wn\))h(tasks)f(on)i(other)f(pro)q(cessors.)19 b(Also,)c(there)f(is)h(no)f(direct)75 2533 y(supp)q(ort)e(for)f(MIMD)g (programming.)18 b(Finally)l(,)c(there)e(is)g(no)f(supp)q(ort)h(for)f (adding)i(an)e(already-running)75 2590 y(pro)q(cess)k(to)g(the)g(MPI)g (collection)i(of)e(pro)q(cesses.)166 2647 y(In)f(the)f(case)g(of)g (heterogeneous)g(en)o(vironmen)o(ts,)g(it)h(is)f(necessary)h(to)e(sp)q (ecify)j(the)e(set)g(of)f(w)o(orksta-)75 2704 y(tions)i(to)f(b)q(e)h (used.)20 b(Our)14 b(only)h(requiremen)o(t)f(is)g(that)f(this)i(b)q(e)f (done,)g(in)h(the)e(C/POSIX)i(case,)e(through)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 127 133 127 132 bop 75 -100 a Fx(8.2.)34 b(MPI)15 b(PR)o(OGRAM)h(ST)l(AR)l(TUP) g(PR)o(OPOSAL{MPI)p 1141 -100 14 2 v 17 w(INIT)553 b FE(127)75 45 y(command-line)18 b(argumen)o(ts)e(\(th)o(us)g(main)o (taining)h(source-co)q(de)g(p)q(ortabilit)o(y\).)24 b(W)l(e)17 b(could)g(sp)q(ecify)h(an)75 102 y(extensible,)j(common)d(format)f(for) h(sp)q(ecifying)i(the)f(mac)o(hines,)h(executables,)g(and)e(resource)h (limits,)75 158 y(but,)h(to)f(a)o(v)o(oid)g(additional)i(debate,)f(w)o (e)f(suggest)g(only)h(that)e(the)i(command)f(line)i(option)f Fp(-mpi)e FE(b)q(e)75 214 y(reserv)o(ed)d(for)g(future)g(extensions.)75 358 y Fy(8.2)59 b(MPI)20 b(Program)g(Sta)n(rtup)g(Prop)r(osal{MPI)p 1137 358 18 2 v 21 w(INIT)75 459 y FE(One)13 b(goal)g(of)f(MPI)h(is)g (to)f(ac)o(hiev)o(e)h Ft(sour)n(c)n(e)h(c)n(o)n(de)g(p)n(ortability)p FE(.)19 b(By)13 b(this)g(w)o(e)f(mean)h(that)f(a)g(program)g(writ-)75 516 y(ten)j(using)i(MPI)e(and)g(complying)i(with)f(the)f(relev)m(an)o (t)h(language)g(standards)f(is)g(p)q(ortable)h(as)f(written,)75 572 y(and)k(m)o(ust)e(not)h(require)i(an)o(y)e(source)g(co)q(de)h(c)o (hanges)f(when)h(mo)o(v)o(ed)f(from)g(one)g(system)g(to)g(another.)75 629 y(This)h(explicitly)i(do)q(es)e Ft(not)f FE(sa)o(y)g(an)o(ything)g (ab)q(out)g(ho)o(w)g(an)h(MPI)f(program)f(is)i(started)f(or)f(launc)o (hed)75 685 y(from)h(the)i(command)f(line,)i(nor)e(what)f(the)i(user)f (m)o(ust)f(do)i(to)e(set)h(up)h(the)f(en)o(vironmen)o(t)g(in)h(whic)o (h)75 742 y(an)h(MPI)g(program)f(will)j(run.)38 b(Ho)o(w)o(ev)o(er,)21 b(an)h(implemen)o(tation)g(ma)o(y)e(require)i(some)f(setup)h(to)e(b)q (e)75 798 y(p)q(erformed)c(b)q(efore)h(other)f(MPI)g(routines)g(ma)o(y) g(b)q(e)h(called.)24 b(T)l(o)16 b(pro)o(vide)g(for)g(this,)g(MPI)h (includes)h(an)75 854 y(initialization)g(routine)e Fp(MPI)p 572 854 15 2 v 17 w(init)p FE(.)166 911 y(All)g(MPI)f(programs)f(m)o (ust)h(con)o(tain)g(a)g(call)h(to)f Fp(MPI)p 1069 911 V 16 w(init)p FE(;)g(this)g(routine)h(m)o(ust)e(b)q(e)i(called)h(b)q (efore)75 967 y(an)o(y)e(other)g(MPI)g(routines)g(are)g(called.)22 b(The)15 b(format)f(is:)75 1061 y Fp(start_ctx)23 b(=)g(MPI_init\()g (...)g(args)h(...)f(\);)75 1155 y FE(The)13 b Fp(start)p 289 1155 V 16 w(ctx)f FE(can)h(then)g(b)q(e)g(used)g(in)h(place)f(of)f (the)h Fp(all)p 1092 1155 V 17 w(ctx)f FE(\(con)o(text)f(of)i(all)g (pro)q(cesses\).)19 b(Multiple)75 1212 y(calls)h(to)e Fp(MPI)p 314 1212 V 17 w(init)g FE(\(sa)o(y)f(\014rst)i(b)o(y)f(a)h (user-co)q(de)g(and)g(then)g(b)o(y)g(a)f(soft)o(w)o(are)f(pac)o(k)m (age)i(called)h(b)o(y)f(the)75 1268 y(user's)d(co)q(de\))h(ha)o(v)o(e)g (meaning;)h(they)e(set)h(up)g(the)g(con)o(text)f(that)g(will)j(b)q(e)e (used)g(as)g(the)g(con)o(text)f(of)g(all)75 1324 y(pro)q(cesses.)166 1457 y Fv(Discussion:)g Fu(Sev)o(eral)e(other)h(approac)o(hes)g(w)o (ere)h(discussed,)f(including)e(a)h(replacemen)o(t)g(main)f(program)75 1507 y(and)f(a)g(subroutine-call)g(in)o(terface.)18 b(I'v)o(e)12 b(included)h(the)g Fn(MPI)p 1033 1507 14 2 v 15 w(init)e Fu(v)o(ersion)i(b)q(ecause)h(it)e(seemed)h(to)f(b)q(e)h(the)g(least)75 1557 y(ob)r(jectionable)h(and)g(since)g(w)o(ork)n(able)f(v)o(ersions)i (of)e(the)h(other)h(t)o(w)o(o)e(ma)o(y)f(b)q(e)j(written)f(using)g Fn(MPI)p 1618 1557 V 15 w(init)p Fu(.)166 1606 y(I'm)8 b(not)i(sure)h(what)f(argumen)o(ts)f Fn(MPI)p 758 1606 V 15 w(init)g Fu(should)h(tak)o(e.)17 b(One)11 b(argumen)o(t)e(should)g (b)q(e)i(the)g(initial)d(con)o(text;)75 1656 y(if)j(n)o(ull,)g(then)h (the)h(\\all)d(con)o(text")j(is)e(assumed.)17 b(There)c(is)f(some)f (reason)i(to)e(pass)i(the)f(command)d(line)j(argumen)o(ts)75 1706 y Fn(argc,argv)f Fu(to)i Fn(MPI)p 403 1706 V 15 w(init)f Fu(in)h(order)g(to)g(allo)o(w)e(it)i(to)g(lo)q(ok)f(for)g (MPI-sp)q(eci\014c)j(argumen)o(ts)d(\(suc)o(h)i(as)f(the)h(n)o(um)o(b)q (er)75 1756 y(and)g(name)e(of)i(pro)q(cessors)i(to)e(use)g(in)g(a)f (heterogeneous)j(en)o(vironmen)o(t\),)d(but)h(this)g(ma)o(y)e(b)q(e)i (to)q(o)g(con)o(tro)o(v)o(ersial.)166 1812 y(W)m(e)g(should)g(also)g (discuss)h(the)g(other)g(p)q(ossibilities.)k(One,)c Fn(MPI)p 1184 1812 V 15 w(main)p Fu(,)e(is)h(a)g(main)e(program)h(replacemen)o (t)75 1869 y(\(users)21 b(use)f Fn(MPI)p 347 1869 V 15 w(main)e Fu(instead)i(of)e Fn(main)h Fu(in)f(C)i(and)f Fn(PROGRAM)h(name)e Fu(in)h(F)m(ortran\).)34 b(Another)20 b(approac)o(h)f(is)75 1925 y Fn(MPI)p 144 1925 V 15 w(call)13 b Fu(that)h(calls)f(a)h(sp)q(eci\014ed)i(routine.)75 2151 y Fy(8.3)59 b(MPI-Sp)r(eci\014c)19 b(Enquiry)g(F)n(unctions)75 2253 y FE(These)d(routines)f(are)g(sp)q(eci\014c)i(to)e(the)g(op)q (eration)g(of)g(MPI.)166 2309 y(The)g(\014rst)g(t)o(w)o(o)f(routines)i (are)f(absolutely)h(essen)o(tial;)f(they)h(are:)75 2403 y FF(MPI)p 184 2403 16 2 v 18 w(rank)22 b FE(My)15 b(pro)q(cess)h(id)g (relativ)o(e)f(to)g(a)g(group)75 2497 y FF(MPI)p 184 2497 V 18 w(size)23 b FE(The)15 b(n)o(um)o(b)q(er)h(of)f(pro)q(cess)g (id's)h(in)g(a)f(group)166 2591 y(MPI)f(also)g(con)o(tain)h(routines)g (for)e(managing)i(implemen)o(tation)g(limits)h(in)f(MPI.)f(These)h (routines)75 2647 y(will)i(tak)o(e)d(a)h(requested)h(v)m(alue)g(and)f (return)h(the)f(actual)g(v)m(alue,)h(p)q(ossibly)h(including)h(a)d(v)m (alue)h(for)f(\\not)75 2704 y(applicable.")22 b(Suc)o(h)16 b(routines)g(ma)o(y)e(include)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 128 134 128 133 bop 75 -100 a FE(128)325 b Fx(SECTION)16 b(8.)34 b(ENVIR)o(ONMENT)l(AL)17 b(MANA)o(GEMENT)d(AND)h(INQUIR)l(Y)75 45 y FF(MPI)p 184 45 16 2 v 18 w(V)l(alidT)l(ags)24 b FE(The)15 b(range)g(\(or)f(other)h(description\))h(of)f(v)m(alid)i (tags)75 139 y FF(MPI)p 184 139 V 18 w(NumGroups)22 b FE(Maxim)o(um)15 b(n)o(um)o(b)q(er)g(of)g(groups)75 232 y FF(MPI)p 184 232 V 18 w(NumCn)o(txs)21 b FE(Maxim)o(um)15 b(n)o(um)o(b)q(er)h(of)e(con)o(texts)75 326 y FF(MPI)p 184 326 V 18 w(Bu\013erSize)23 b FE(Bu\013er)15 b(space)g(for)g(unsen)o (t/unreceiv)o(ed)i(messages)75 419 y FF(MPI)p 184 419 V 18 w(Bu\013erManagemen)o(t)22 b FE(Bu\013er)15 b(managemen)o(t)f(t)o (yp)q(e)166 513 y(Note)h(that)f(all)i(of)f(these)h(could)g(return)f (\\not)f(applicable.")166 652 y Fv(Discussion:)36 b Fu(These)16 b(need)f(to)f(refer)i(to)e(a)g(diagram)e(of)i(p)q(ossible)h(bu\013er)g (usages.)20 b(W)m(e)14 b(m)o(ust)g(b)q(e)h(careful)75 708 y(to)f(handle)f(the)i(case)g(of)e(\(a\))h(no)g(bu\013ering)g(and)g (\(b\))g(bu\013ers)h(on)f(the)g(sender.)166 847 y FE(More)21 b(detailed)i(routines)f(for)g(bu\013er)f(managemen)o(t)g(ma)o(y)g(also) h(b)q(e)g(pro)o(vided.)41 b(These)22 b(could)75 904 y(include:)75 997 y FF(MPI)p 184 997 V 18 w(Bu\013erSizeP)o(airs)g FE(Bet)o(w)o(een)15 b(pairs)h(of)e(pro)q(cessors)75 1090 y FF(MPI)p 184 1090 V 18 w(Bu\013erSizeAll)23 b FE(Bet)o(w)o(een)15 b(\\me")g(and)g(all)i(others)75 1184 y FF(MPI)p 184 1184 V 18 w(Exclusiv)o(eBu\013er)22 b FE(Dedicate)g(a)f(bu\013er)g(of)g(a)g (sp)q(eci\014ed)j(size)e(b)q(et)o(w)o(een)g(a)f(selected)i(pair)e(of) 629 1241 y(pro)q(cessors)166 1410 y Fv(Discussion:)h Fu(Another)c(approac)o(h)f(is)g(to)g(use)h(a)f(single)g(routine)g(that) g(tak)o(es)h(a)f(query)g(parameter)g(and)75 1460 y(returns)e(the)g (requested)h(data)d(\(or)h(a)g(\\not)g(a)o(v)n(ailable"\).)166 1509 y(The)h(routines)h(returning)f(the)g(maxim)o(um)10 b(n)o(um)o(b)q(er)k(of)g(things)h(\(lik)o(e)f(groups\))i(could)e (instead)h(return)h(the)75 1559 y(maxim)n(um)g(n)o(um)o(b)q(er)j(of)g Fl(new)24 b Fu(things)c(that)g(could)f(b)q(e)i(created.)37 b(All)19 b(of)g(these)i(routines)g(are)f(mean)o(t)f(to)g(b)q(e)75 1609 y(advisory)m(,)12 b(particularly)i(in)f(a)h(m)o(ultithreaded)e(en) o(vironmen)o(t,)h(since)h(they)h(ma)o(y)d(b)q(e)i(allo)q(cated)g (dynamically)l(.)166 1748 y FE(There)h(is)h(also)f(some)g(desire)h(for) 75 1841 y FF(MPI)p 184 1841 V 18 w(IOmo)q(de)23 b FE(Whether)15 b(IO)h(is)g(p)q(er)g(no)q(de)f(or)g(p)q(er)h(group/con)o(text)75 1935 y FF(MPI)p 184 1935 V 18 w(ErrorMo)q(de)22 b FE(V)l(arious)16 b(c)o(hoices)g(of)e(error)h(handling)166 2028 y(Both)j(of)f(these,)h (since)h(they)f(ha)o(v)o(e)f(global)i(e\013ect,)f(are)f(somewhat)g (undesirable.)30 b(They)18 b(should)75 2085 y(return)j(the)g(previous)h (mo)q(de;)j(m)o(ultithreaded)d(applications)h(will)g(ha)o(v)o(e)d(to)h (b)q(e)h(v)o(ery)f(careful)h(with)75 2141 y(these.)166 2197 y(It)16 b(is)h(imp)q(ortan)o(t)f(to)g(b)q(e)g(able)h(to)f (determine,)h(b)q(oth)g(at)e(run)i(time)g(and)f(compile)i(time,)e(whic) o(h)h(of)75 2254 y(the)e(\\optional")h(capabilities)h(the)e(MPI)h (implemen)o(tation)g(supp)q(orts.)k(These)c(include)75 2347 y FF(MPI)p 184 2347 V 18 w(Has)p 289 2347 V 19 w(Ready)p 453 2347 V 19 w(Receiv)o(er)22 b FE(Whether)10 b(ready-receiv)o(er)h (comm)o(unication)g(is)g(sim)o(ulated)g(with)f(reg-)629 2404 y(ular)16 b(sends)g(and)f(receiv)o(es.)75 2497 y FF(MPI)p 184 2497 V 18 w(Has)p 289 2497 V 19 w(Non)o(blo)q(c)o(king)23 b FE(Whether)h(non)o(blo)q(c)o(king)h(op)q(erations)e(are)g(sim)o (ulated)i(with)f(blo)q(c)o(king)629 2554 y(sends)16 b(and)f(receiv)o (es)75 2647 y FF(MPI)p 184 2647 V 18 w(Has)p 289 2647 V 19 w(Heterogeneous)22 b FE(Whether)16 b(heterogeneous)g(pro)q (cessing)g(is)h(supp)q(orted)f(\(that)f(is,)h(this)629 2704 y(MPI)f(w)o(orks)g(with)g(heterogeneous)g(systems\))1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 129 135 129 134 bop 75 -100 a Fx(8.3.)34 b(MPI-SPECIFIC)16 b(ENQUIR)l(Y)g (FUNCTIONS)788 b FE(129)166 45 y(In)14 b(addition,)h(it)f(is)g(useful)h (to)e(b)q(e)h(able)h(to)e(unpac)o(k)h(the)g(v)m(arious)g(opaque)g(ob)s (jects)f(in)o(to)h(individual)75 102 y(v)m(ariables,)25 b(suc)o(h)d(as)f(bu\013er)h(descriptors)h(and)f(con)o(texts,)g(and)h (the)f(message)f(queue.)41 b(Op)q(erations)75 158 y(should)16 b(include)75 251 y FF(MPI)p 184 251 16 2 v 18 w(Unpac)o (kBu\013erDescriptor)22 b FE(T)l(ak)o(es)d(a)f(bu\013er)h(and)h (returns)f(a)f(con)o(text)h(and)g(\(p)q(ossibly\))h(a)629 307 y(n)o(um)o(b)q(er)c(of)f(mem)o(b)q(ers.)k(Individual)g(routines)c (can)629 413 y FF(Bu\013erDescT)o(yp)q(e)22 b FE(Giv)o(en)d(a)g(con)o (text,)f(return)h(the)g(kind)h(of)e(bu\013er)729 470 y(descriptor)e(elemen)o(t)g(\(con)o(tiguous)f(etc\).)629 542 y FF(MPI)p 738 542 V 18 w(Unpac)o(kCon)o(tig)23 b FE(Giv)o(en)18 b(a)f(con)o(text,)g(return)g(the)g(data.)26 b(Ad-)729 599 y(v)m(ance)16 b(the)g(con)o(text.)629 672 y FF(MPI)p 738 672 V 18 w(Unpac)o(kIndex)22 b FE(Ditto)15 b(of)g(indexed)629 744 y FF(MPI)p 738 744 V 18 w(Unpac)o(kBlo)q(c)o(k) 24 b FE(Ditto)14 b(for)h(blo)q(c)o(ks.)75 850 y FF(MPI)p 184 850 V 18 w(DumpMessageQueue)22 b FE(T)l(ak)o(es)14 b(a)g(length)h(and)g(a)f(user-allo)q(cated)i(arra)o(y)l(,)d(it)h (return)h(n)o(um)o(b)q(er)629 906 y(of)j(elemen)o(ts)i(a)o(v)m(ailable) g(\(ma)o(y)d(b)q(e)j(greater)d(than)i(length\))g(and)g(infor-)629 963 y(mation)g(on)g(the)g(messages)g(in)h(the)f(format)f(includes)j (\(tag,)e(con)o(text,)629 1019 y(source,)c(size\))75 1113 y FF(MPI)p 184 1113 V 18 w(Unpac)o(kCon)o(text)22 b FE(T)l(ak)o(es)15 b(a)g(con)o(text)g(and)g(returns)g(mem)o(b)q(ers)h (of)e(group/con)o(text)75 1206 y FF(MPI)p 184 1206 V 18 w(StatusOfHandles)24 b FE(I'v)o(e)15 b(forgotten)f(what)h(this)g(is) h(for.)166 1299 y(Note)f(that)f(some)h(of)g(these)g(are)g(tough)g(to)g (implemen)o(t)h(directly)h(or)d(e\016cien)o(tly)j(in)f(F)l(ortran.)166 1438 y Fv(Discussion:)i Fu(An)e(alternate)f(approac)o(h)h(for)f(the)g (queue)i(and)e(con)o(text)h(inquiry)e(functions)i(\(at)f(least)g(for)75 1495 y(C\))f(is)g(for)f(the)i(routine)f(to)g(allo)q(cate)f(the)h(space) h(rather)g(than)f(ha)o(ving)f(it)g(passed)i(in.)75 1699 y Fq(8.3.1)49 b(P)o(a)o(rallel)17 b(p)o(rogramming)75 1785 y FE(This)d(section)f(includes)i(routines)f(that)e(relate)h(to)f (parallel)j(programming)d(but)h(are)g(not)f(MPI-sp)q(eci\014c.)75 1841 y(These)k(ma)o(y)e(include)k(suc)o(h)d(things)h(as:)75 1934 y FF(MPI)p 184 1934 V 18 w(GetNbrs)22 b FE(Return)16 b(the)f(\\neigh)o(b)q(ors")h(in)g(the)f(underlying)i(ph)o(ysical)g(top) q(ology)75 2028 y FF(MPI)p 184 2028 V 18 w(GetPh)o(ysNo)q(de)22 b FE(Return)i(a)f(unique)h(sp)q(eci\014er)h(for)e(the)g(actual)g(\(as)f (opp)q(osed)i(to)f(virtual\))629 2084 y(no)q(de.)h(This)17 b(should)h(b)q(e)f(a)f(c)o(haracter)g(string)g(for)g(maxim)o(um)g (\015exibil-)629 2141 y(it)o(y)l(.)26 b(>F)l(rom)17 b(this)g(v)m(alue)h (it)g(m)o(ust)e(b)q(e)i(p)q(ossible)h(to)d(iden)o(tify)i(a)f(sp)q (eci\014c)629 2197 y(piece)h(of)e(hardw)o(are;)g(p)q(ossible)i(v)m (alues)f(include)i(\\pro)q(cessor)c(9)h(in)i(rac)o(k)629 2254 y(4)e(of)g(mpp.cs.org")f(and)h(\\231")f(\(where)h(231)f(is)i(the)f (actual)g(pro)q(cessor)629 2310 y(n)o(um)o(b)q(er)i(in)h(the)f(running) g(homogeneous)g(system\).)26 b(A)18 b(return)g(v)m(alue)629 2367 y(of)d(\\not)g(applicable")i(is)f(allo)o(w)o(ed)f(but)h(strongly)e (discouraged.)75 2460 y FF(GetT)l(op)q(ology)24 b FE(Return)18 b(ph)o(ysical)g(net)o(w)o(ork)f(top)q(ology)l(.)25 b(Prede\014ned)19 b(v)m(alues)f(include)h Fp(MPI)p 1705 2460 15 2 v 17 w(mesh2d)p FE(,)629 2517 y Fp(MPI)p 704 2517 V 17 w(hypercube)p FE(,)14 b(and)h Fp(MPI)p 1124 2517 V 17 w(omeganetwork)p FE(.)75 2610 y FF(MPI)p 184 2610 16 2 v 18 w(GetDistance)24 b FE(Return)16 b(the)f(\\distance")h(from)e(one)i(pro)q(cessor)f(to)f (another)75 2704 y FF(MPI)p 184 2704 V 18 w(GetDiameter)23 b FE(Return)16 b(the)g(maxim)o(um)f(\\distance")g(from)g(one)g(pro)q (cessor)g(to)f(another.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 130 136 130 135 bop 75 -100 a FE(130)325 b Fx(SECTION)16 b(8.)34 b(ENVIR)o(ONMENT)l(AL)17 b(MANA)o(GEMENT)d(AND)h(INQUIR)l(Y)166 45 y FE(Eac)o(h)e(of)g(these)h(routines)g(ma)o(y)e(return)i(\\not)e (applicable;")k(it)e(is)g(hop)q(ed)g(that)f(qualit)o(y)h(implemen-)75 102 y(tations)g(will)j(try)d(and)g(pro)o(vide)i(as)e(m)o(uc)o(h)h (information)f(as)h(p)q(ossible)h(to)e(the)h(user.)20 b(In)15 b(addition,)g(some)75 158 y(of)g(the)g(\\get")g(routines)g(m)o (ust)g(b)q(e)h(in)o(terpreted)g(as)f(pro)o(viding)h(appro)o(ximate)f (answ)o(ers.)20 b(F)l(or)14 b(example,)75 214 y(if)k(groups)f(and)h (con)o(texts)f(are)h(allo)q(cated)g(from)f(the)h(user's)f(address)h (space,)g(then)g(an)o(y)f(return)h(v)m(alue)75 271 y(will)f(b)q(e)f (appro)o(ximate.)166 327 y(The)g(exact)f(meaning)h(of)f(\\distance")h (is)g(left)g(to)f(the)g(implemen)o(tation.)22 b(In)16 b(a)g(system)f(with)g(only)75 384 y(lo)q(cal)23 b(connections,)g(suc)o (h)f(as)g(a)f(mesh)h(or)f(h)o(yp)q(ercub)q(e,)j(the)e(distance)g (should)h(b)q(e)f(the)f(n)o(um)o(b)q(er)h(of)75 440 y(edges)17 b(in)h(the)e(connection)i(graph)f(that)f(a)g(message)h(w)o(ould)g(tra)o (v)o(el)f(along.)25 b(In)17 b(a)f(system)h(with)g(a)f(\015at)75 497 y(in)o(terconnect,)d(suc)o(h)g(as)g(an)f(omega)g(or)g(butter\015y)g (net)o(w)o(ork,)g(the)h(distance)g(could)h(b)q(e)f(zero)f(from)g(a)g (no)q(de)75 553 y(to)j(itself)h(and)f(either)h(one)f(or)g(the)g(n)o(um) o(b)q(er)h(of)f(stages)f(in)i(the)f(net)o(w)o(ork)f(for)h(all)h(other)f (no)q(des.)75 675 y Fq(8.3.2)49 b(non-MPI)75 761 y FE(This)21 b(section)h(con)o(tains)e(routines)h(that)f(are)h Ft(not)k FE(part)20 b(of)g(MPI)h(but)f(are)h(included)i(for)d(complete-)75 817 y(ness.)k(These)17 b(will)h(include)h(v)m(arious)e(routines)g(for)f (resource)g(usage,)h(time-of-da)o(y)l(,)f(etc.)24 b(The)17 b(POSIX)75 874 y(in)o(terfaces)e(will)i(b)q(e)f(summarized)g(here.)75 994 y Fo(Interaction)g(with)g(signals)75 1080 y FE(The)f(in)o(ten)o(t)g (of)f(MPI)h(is)g(to)f(b)q(e)h(thread)g(and)f(signal)i(safe.)j(This)d (has)e(a)g(n)o(um)o(b)q(er)h(of)g(subtle)g(e\013ects.)20 b(F)l(or)75 1136 y(example,)g(on)e(Unix)h(systems,)g(a)f(catc)o(hable)h (signal)g(suc)o(h)g(as)f(SIGALRM)h(\(an)f(alarm)h(signal\))g(m)o(ust)75 1192 y(not)f(cause)g(an)h(MPI)f(routine)g(to)g(b)q(eha)o(v)o(e)h (di\013eren)o(tly)g(than)f(it)g(w)o(ould)h(ha)o(v)o(e)f(in)h(the)f (absence)h(of)f(the)75 1249 y(signal.)37 b(Of)21 b(course,)h(if)f(the)g (signal)h(handler)f(issues)h(MPI)f(calls)g(or)g(c)o(hanges)f(the)h(en)o (vironmen)o(t)g(in)75 1305 y(whic)o(h)c(the)g(MPI)g(routine)g(is)g(op)q (erating)f(\(for)g(example,)h(consuming)h(all)f(a)o(v)m(ailable)h (memory)e(space\),)75 1362 y(the)e(MPI)f(routine)h(should)h(b)q(eha)o (v)o(e)f(as)f(appropriate)h(for)f(that)g(situation)h(\(in)g (particular,)g(in)g(this)g(case,)75 1418 y(the)h(b)q(eha)o(vior)h (should)g(b)q(e)g(the)f(same)g(as)g(for)g(a)g(m)o(ultithreaded)h(MPI)f (implemen)o(tation\).)166 1475 y(A)j(second)i(e\013ect)e(is)h(that)f(a) g(signal)h(handler)h(that)e(do)q(es)h(MPI)f(calls)i(m)o(ust)e(not)g(in) o(terfere)h(with)75 1531 y(the)c(op)q(eration)f(of)g(MPI.)g(F)l(or)g (example,)h(an)g(MPI)f(receiv)o(e)i(of)e(an)o(y)g(t)o(yp)q(e)h(that)e (o)q(ccurs)i(within)h(a)e(signal)75 1588 y(handler)22 b(m)o(ust)f(not)g(cause)h(erroneous)f(b)q(eha)o(vior)h(b)o(y)f(the)h (MPI)f(implemen)o(tation.)39 b(Note)21 b(that)g(an)75 1644 y(implemen)o(tation)16 b(is)g(p)q(ermitted)g(to)e(prohibit)i(the)f (use)h(of)e(MPI)h(calls)h(from)e(within)j(a)d(signal)i(handler.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 131 137 131 136 bop 75 359 a Fz(Section)35 b(9)75 570 y FG(Pro\014ling)75 814 y Fy(9.1)59 b(Requirements)75 921 y FE(T)l(o)15 b(meet)g(the)g(MPI) g(pro\014ling)i(in)o(terface,)e(an)g(implemen)o(tation)i(of)d(the)i (MPI)f(functions)h Ft(must)131 1027 y FE(1.)22 b(pro)o(vide)14 b(a)f(mec)o(hanism)h(through)f(whic)o(h)h(all)g(of)f(the)g(MPI)h (de\014ned)h(functions)f(ma)o(y)e(b)q(e)i(accessed)189 1083 y(with)e(a)g(name)f(shift.)20 b(Th)o(us)11 b(all)i(of)f(the)g(MPI) g(functions)h(\(whic)o(h)f(normally)g(start)f(with)h(the)g(pre\014x)189 1140 y(\\)p Fp(MPI)p 287 1140 15 2 v 16 w FE("\))j(should)h(also)f(b)q (e)h(accessable)g(with)g(the)f(pre\014x)h(\\)p Fp(PMPI)p 1300 1140 V 16 w FE(".)131 1246 y(2.)22 b(ensure)c(that)f(those)h(MPI)g (functions)h(whic)o(h)f(are)g(not)f(replaced)i(ma)o(y)f(still)h(b)q(e)g (link)o(ed)g(in)o(to)f(an)189 1302 y(executable)e(image)f(without)h (causing)f(name)h(clashes.)131 1408 y(3.)22 b(do)q(cumen)o(t)c(the)f (implemen)o(tation)i(of)e(di\013eren)o(t)h(language)g(bindings)h(of)e (the)h(MPI)g(in)o(terface)f(if)189 1464 y(they)g(are)h(la)o(y)o(ered)g (on)f(top)g(of)h(eac)o(h)f(other,)h(so)f(that)g(the)h(pro\014ler)g(dev) o(elop)q(er)h(kno)o(ws)e(whether)189 1521 y(she)d(m)o(ust)f(implemen)o (t)i(the)e(pro\014le)i(in)o(terface)f(for)f(eac)o(h)h(binding,)h(or)e (can)h(economise)h(b)o(y)e(imple-)189 1577 y(men)o(ting)i(it)h(only)f (for)g(the)g(lo)o(w)o(est)g(lev)o(el)h(routines.)75 1737 y Fy(9.2)59 b(Discussion)75 1845 y FE(The)13 b(ob)s(jectiv)o(e)g(of)g (the)g(MPI)g(pro\014ling)h(in)o(terface)g(is)f(to)f(ensure)i(that)e(it) i(is)f(relativ)o(ely)h(easy)f(for)g(authors)75 1901 y(of)18 b(pro\014ling)i(\(and)f(other)f(similar\))h(to)q(ols)g(to)f(in)o (terface)h(their)g(co)q(des)g(to)f(MPI)h(implemen)o(tations)h(on)75 1958 y(di\013eren)o(t)15 b(mac)o(hines.)166 2017 y(Since)20 b(MPI)e(is)h(a)f(mac)o(hine)h(indep)q(enden)o(t)h(standard)e(with)h (man)o(y)f(di\013eren)o(t)g(implemen)o(tations,)75 2074 y(it)e(is)h(unreasonable)g(to)e(exp)q(ect)i(that)f(the)g(authors)f(of)h (pro\014ling)h(to)q(ols)f(for)g(MPI)g(will)i(ha)o(v)o(e)d(access)i(to) 75 2130 y(the)c(source)h(co)q(de)f(whic)o(h)h(implemen)o(ts)h(MPI)e(on) g(an)o(y)g(particular)h(mac)o(hine.)20 b(It)13 b(is)h(therefore)f (necessary)75 2187 y(to)21 b(pro)o(vide)i(a)f(mec)o(hanism)g(b)o(y)g (whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o(h)g(to)q(ols)g(can)g (collect)i(whatev)o(er)75 2243 y(p)q(erformance)15 b(information)h (they)f(wish)h Ft(without)g FE(access)f(to)g(the)g(underlying)i (implemen)o(tation.)166 2302 y(W)l(e)d(b)q(eliev)o(e)i(that)e(ha)o (ving)g(suc)o(h)g(an)h(in)o(terface)f(is)h(imp)q(ortan)o(t)e(if)i(MPI)f (is)g(to)g(b)q(e)h(attractiv)o(e)e(to)g(end)75 2359 y(users,)i(since)i (the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g(di\013eren)o(t)h(to)q (ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h(factor)f(in)h(attracting) 75 2415 y(users)f(to)g(the)g(MPI)g(standard.)166 2475 y(The)e(pro\014ling)h(in)o(terface)e(is)h(just)g(that,)f(an)g(in)o (terface.)20 b(It)12 b(sa)o(ys)g Ft(nothing)g FE(ab)q(out)g(the)h(w)o (a)o(y)f(in)h(whic)o(h)75 2531 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f(to)g(la)o(y)h(do)o(wn)g(what)g (information)g(is)h(collected)g(through)75 2588 y(the)g(in)o(terface,)g (or)g(ho)o(w)g(the)g(collected)i(information)e(is)h(sa)o(v)o(ed,)e (\014ltered,)i(or)f(displa)o(y)o(ed.)166 2647 y(While)g(the)g(initial)h (imp)q(etus)f(for)f(the)g(dev)o(elopmen)o(t)h(of)e(this)i(in)o(terface) f(arose)g(from)f(the)h(desire)i(to)75 2704 y(p)q(ermit)g(the)f (implemen)o(tation)i(of)d(pro\014ling)j(to)q(ols,)d(it)i(is)f(clear)h (that)f(an)g(in)o(terface)g(lik)o(e)h(that)f(sp)q(eci\014ed)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 132 138 132 137 bop 75 -100 a FE(132)1178 b Fx(SECTION)16 b(9.)35 b(PR)o(OFILING)75 45 y FE(ma)o(y)17 b(also)h(pro)o(v)o(e)f(useful)i (for)e(other)h(purp)q(oses,)g(suc)o(h)g(as)g(\\in)o(ternet)o(w)o (orking")f(m)o(ultiple)i(MPI)f(imple-)75 102 y(men)o(tations.)j(Since)c (all)f(that)f(is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there)h(is)g (no)f(ob)s(jection)h(to)f(its)h(b)q(eing)g(used)75 158 y(wherev)o(er)f(it)h(is)f(useful.)166 214 y(As)j(the)g(issues)h(b)q (eing)g(addressed)f(here)h(are)f(in)o(timately)h(tied)f(up)h(with)f (the)g(w)o(a)o(y)f(in)i(whic)o(h)g(ex-)75 271 y(ecutable)j(images)e (are)g(built,)j(whic)o(h)e(ma)o(y)f(di\013er)h(greatly)f(on)g (di\013eren)o(t)h(mac)o(hines,)h(the)e(examples)75 327 y(giv)o(en)c(b)q(elo)o(w)g(should)h(b)q(e)f(treated)g(solely)g(as)f (one)h(w)o(a)o(y)f(of)g(implemen)o(ting)j(the)d(ob)s(jectiv)o(e)h(of)f (the)h(MPI)75 384 y(pro\014ling)h(in)o(terface.)k(The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o(tation)h(are)e (those)g(detailed)75 440 y(in)i(the)f(Requiremen)o(ts)i(section)e(ab)q (o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this)i(c)o(hapter)f (is)g(only)h(presen)o(t)f(as)75 497 y(justi\014cation)g(and)f (discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.)166 553 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f(in)h (whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed)75 610 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 666 y(equally)d(v)m(alid\).)75 806 y Fy(9.3)59 b(Logic)20 b(of)g(the)f(design)75 907 y FE(Pro)o(vided)d(that)f(an)g(MPI)g (implemen)o(tation)i(meets)e(the)h(requiremen)o(ts)g(ab)q(o)o(v)o(e,)e (it)i(is)g(p)q(ossible)h(for)e(the)75 964 y(implemen)o(tor)20 b(of)e(the)h(pro\014ling)h(system)f(to)f(in)o(tercept)i(all)f(of)g(the) g(MPI)g(calls)h(whic)o(h)f(are)g(made)g(b)o(y)75 1020 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h(collect)g(whatev)o (er)f(information)g(she)h(requires)g(b)q(efore)f(calling)75 1076 y(the)c(underlying)i(MPI)d(implemen)o(tation)i(\(through)f(its)g (name)f(shifted)i(en)o(try)e(p)q(oin)o(ts\))h(to)f(ac)o(hiev)o(e)i(the) 75 1133 y(desired)g(e\013ects.)75 1272 y Fy(9.4)59 b(Examples)75 1376 y Fq(9.4.1)49 b(Pro\014ler)16 b(implementation)75 1461 y FE(Supp)q(ose)25 b(that)e(the)h(pro\014ler)h(wishes)g(to)e (accum)o(ulate)i(the)f(total)f(amoun)o(t)g(of)h(data)f(sen)o(t)h(b)o(y) g(the)75 1518 y(MPI)p 167 1518 14 2 v 16 w(SEND)18 b(function,)g(along) f(with)h(the)f(total)g(elapsed)h(time)f(sp)q(en)o(t)h(in)g(the)f (function.)27 b(This)18 b(could)75 1574 y(trivially)f(b)q(e)f(ac)o (hiev)o(ed)g(th)o(us)f(:{)166 1631 y Fa(W)-6 b(ARNING)18 b(:)d FE(syn)o(tax)f(correction)h(needed)i(in)f(examples)g(when)g(MPI)f (syn)o(tax)f(is)i(de\014ned.)75 1709 y Fp(static)23 b(int)g (totalBytes;)75 1766 y(static)g(int)g(totalTime;)75 1822 y(extern)g(int)g(now\(void\);)71 b(/*)23 b(Some)h(suitable)e(timer)i (function)e(!)i(*/)75 1935 y(int)f(MPI_SEND\(const)f(int)i(tag,)f (const)g(int)h(bytes,)f(void)g(*)h(buffer,)f(const)g(int)g(who\))75 1992 y({)147 2048 y(int)g(tstart)g(=)h(now\(\);)147 2105 y(int)f(result)g(=)h(PMPI_SEND\(tag,bytes,buffe)o(r,who\);)68 b(/*)24 b(Pass)f(on)h(all)f(the)g(arguments)g(*/)147 2217 y(totalBytes)f(+=)i(bytes;)405 b(/*)23 b(Accumulate)g(byte)g (count)g(*/)147 2274 y(totalTime)46 b(+=)24 b(now\(\))f(-)h(tstart;)190 b(/*)23 b(and)h(time)333 b(*/)147 2387 y(return)23 b(result;)75 2443 y(})75 2561 y Fq(9.4.2)49 b(MPI)17 b(lib)o(ra)o(ry)f (implementation)75 2647 y FE(On)i(a)f(Unix)h(system,)f(in)h(whic)o(h)g (the)g(MPI)f(library)h(is)g(implemen)o(ted)h(in)f(C,)f(then)g(there)h (are)e(v)m(arious)75 2704 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i (t)o(w)o(o)e(of)g(the)h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.) 35 b(Whic)o(h)20 b(is)h(b)q(etter)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 133 139 133 138 bop 75 -100 a Fx(9.4.)34 b(EXAMPLES)1370 b FE(133)75 45 y(dep)q(ends)17 b(on)e(whether)g(the)h(link)o(er)g(and)f(compiler)i (supp)q(ort)e(w)o(eak)g(sym)o(b)q(ols.)75 165 y Fo(Systems)h(with)g(w)o (eak)f(symb)q(ols)75 250 y FE(If)k(the)f(compiler)i(and)f(link)o(er)h (supp)q(ort)f(w)o(eak)f(external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g(system)75 307 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h(library)g(is)f(required)h (through)f(the)g(use)g(of)g Fp(#pragma)23 b(weak)13 b FE(th)o(us)75 363 y(:{)75 465 y Fp(#pragma)23 b(weak)g(MPI_EXAMPLE)g(=) g(PMPI_EXAMPLE)75 578 y(int)g(PMPI_EXAMPLE\(/*)f(appropriate)h(args)g (*/\))75 634 y({)170 691 y(/*)h(Useful)f(content)g(*/)75 747 y(})166 849 y FE(The)d(e\013ect)g(of)g(this)g Fp(#pragma)f FE(is)i(to)e(de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Fp(MPI)p 1472 849 15 2 v 17 w(EXAMPLE)f FE(as)h(a)g(w)o(eak)75 906 y(de\014nition,)d(this)f(means)g(that)f(the)g(link)o(er)i(will)g (not)e(complain)i(if)f(there)g(is)g(another)f(de\014nition)j(of)d(the) 75 962 y(sym)o(b)q(ol)g(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1019 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1138 y Fo(Systems)h(without)h(w)o(eak)e(symb)q(ols) 75 1224 y FE(In)k(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h(then)g (one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h(the)f(C)h (macro)75 1280 y(pre-pro)q(cessor)c(th)o(us)g(:{)75 1382 y Fp(#ifdef)23 b(PROFILELIB)75 1438 y(#)95 b(ifdef)23 b(__STDC__)75 1495 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1551 y(#)95 b(else)75 1608 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 1664 y(#)95 b(endif)75 1721 y(#else)75 1777 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 1834 y(#endif)166 1935 y FE(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2037 y Fp(int)23 b(FUNCTION\(MPI_EXAMPLE\)\(/*)e (appropriate)h(args)i(*/\))75 2094 y({)170 2150 y(/*)g(Useful)f (content)g(*/)75 2207 y(})166 2308 y FE(The)d(same)g(source)g(\014le)h (can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q(duce)i(b)q(oth)f(v)o (ersions)g(of)g(the)g(library)l(,)75 2365 y(dep)q(ending)d(on)f(the)f (state)f(of)h(the)g Fp(PROFILELIB)f FE(macro)h(sym)o(b)q(ol.)166 2421 y(It)e(is)g(required)h(that)e(the)h(standard)f(MPI)h(library)g(b)q (e)h(built)g(in)f(suc)o(h)g(a)g(w)o(a)o(y)e(that)h(the)h(inclusion)i (of)75 2478 y(MPI)g(functions)g(can)g(b)q(e)h(ac)o(hiev)o(ed)f(one)g (at)g(a)f(time.)20 b(This)c(is)f(a)f(somewhat)g(unpleasan)o(t)i (requiremen)o(t,)75 2534 y(since)22 b(it)f(ma)o(y)f(mean)h(that)f(eac)o (h)g(external)i(function)f(has)g(to)f(b)q(e)h(compiled)i(from)d(a)g (separate)g(\014le.)75 2591 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g (necessary)g(so)f(that)g(the)g(author)g(of)g(the)h(pro\014ling)g (library)g(need)h(only)f(de\014ne)g(those)75 2647 y(MPI)f(functions)g (whic)o(h)g(she)g(wishes)h(to)e(in)o(tercept,)h(references)g(to)f(an)o (y)g(others)g(b)q(eing)i(ful\014lled)h(b)o(y)e(the)75 2704 y(normal)g(MPI)g(library)l(.)21 b(Therefore)15 b(the)h(link)g (step)f(can)h(lo)q(ok)f(something)h(lik)o(e)g(this)g(:{)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 134 140 134 139 bop 75 -100 a FE(134)1178 b Fx(SECTION)16 b(9.)35 b(PR)o(OFILING)75 45 y Fp(\045)24 b(cc)f(...)h(-lmyprof)f(-lpmpi)g (-lmpi)166 151 y FE(Here)e Fp(libmyprof.a)e FE(con)o(tains)i(the)g (pro\014ler)g(functions)h(whic)o(h)f(in)o(tercept)g(some)g(of)f(the)h (MPI)75 208 y(functions.)j Fp(libpmpi.a)15 b FE(con)o(tains)i(the)f (\\name)h(shifted")g(MPI)f(functions,)h(and)g Fp(libmpi.a)e FE(con)o(tains)75 264 y(the)g(normal)g(de\014nitions)i(of)e(the)g(MPI)h (functions.)75 407 y Fy(9.5)59 b(Multiple)20 b(levels)f(of)h (interception)75 509 y FE(The)15 b(sc)o(heme)h(giv)o(en)g(here)f(do)q (es)h(not)f(directly)h(supp)q(ort)f(the)h(nesting)g(of)e(pro\014ling)j (functions,)e(since)i(it)75 565 y(pro)o(vides)f(only)f(a)g(single)h (alternativ)o(e)g(name)f(for)f(eac)o(h)h(MPI)g(function.)21 b(Consideration)16 b(w)o(as)e(giv)o(en)i(to)75 622 y(an)f(implemen)o (tation)i(whic)o(h)f(w)o(ould)g(allo)o(w)g(m)o(ultiple)h(lev)o(els)g (of)e(call)h(in)o(terception,)h(ho)o(w)o(ev)o(er)d(w)o(e)h(w)o(ere)75 678 y(unable)j(to)e(construct)g(an)g(implemen)o(tation)i(of)e(this)h (whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o(wing)i(disadv)m(an-) 75 735 y(tages)143 829 y FA(\017)23 b FE(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 922 y FA(\017)23 b FE(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o(en) g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 1016 y(Since)i(one)e(of)g(the)h(ob)s(jectiv)o(es)f(of)g(MPI)h(is)f(to)g (p)q(ermit)h(e\016cien)o(t,)g(lo)o(w)f(latency)h(implemen)o(tations,)h (and)75 1073 y(it)j(is)f(not)g(the)h(business)g(of)f(a)g(standard)g(to) g(require)h(a)f(particular)h(implemen)o(tation)g(language,)g(w)o(e)75 1129 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h(outlined)h(ab)q(o)o(v) o(e.)166 1186 y(Note,)f(ho)o(w)o(ev)o(er,)f(that)h(it)h(is)f(p)q (ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o(v)o(e)g(to)g(implemen) o(t)i(a)e(m)o(ulti-lev)o(el)75 1242 y(system,)k(since)g(the)g(function) g(called)i(b)o(y)d(the)h(user)f(ma)o(y)g(call)i(man)o(y)e(di\013eren)o (t)g(pro\014ling)i(functions)75 1299 y(b)q(efore)15 b(calling)i(the)f (underlying)h(MPI)e(function.)166 1355 y(Unfortunately)22 b(suc)o(h)g(an)g(implemen)o(tation)h(ma)o(y)e(require)h(more)g(co-op)q (eration)g(b)q(et)o(w)o(een)g(the)75 1411 y(di\013eren)o(t)g (pro\014ling)i(libraries)g(than)e(is)h(required)g(for)e(the)i(single)g (lev)o(el)h(implemen)o(tation)f(detailed)75 1468 y(ab)q(o)o(v)o(e.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 135 141 135 140 bop 75 369 a Fz(Section)35 b(10)75 590 y FG(Initial)43 b(Implemen)m(tation)f(Subset)75 844 y Fy(10.1)59 b(Intro)r(duction)75 971 y FE(This)17 b(c)o(hapter)f(de\014nes)h(the)g(minimal)h(subset)e (of)g(MPI)g(for)g(initial)i(implemen)o(tation.)25 b(This)17 b(subset)f(is)75 1027 y(b)q(eing)c(de\014ned)h(so)d(that)g(consisten)o (t)i(implemen)o(tations)g(can)f(app)q(ear)g(more)g(rapidly)l(.)20 b(It)11 b(w)o(as)f(recognized)75 1084 y(early)17 b(in)h(the)f(pro)q (cess)g(that)f(MPI)h(needed)h(to)e(app)q(ear)h(as)f(quic)o(kly)i(as)f (p)q(ossible)h(and)f(practical.)26 b(The)75 1140 y(creation)16 b(of)f(a)g(subset)h(will)h(hop)q(efully)g(allo)o(w)f(users)g(earlier)g (access)g(to)e(the)i(standard)f(and)h(still)h(allo)o(w)75 1197 y(for)e(the)g(writing)h(of)e(p)q(ortable)i(message)f(passing)g(co) q(des.)166 1266 y(This)20 b(subset)g(should)h(not)e(in)i(an)o(y)e(w)o (a)o(y)g(b)q(e)h(construed)g(as)f(a)h(limitation)h(on)e(MPI)h (implemen-)75 1323 y(tations.)35 b(It)20 b(is)h(strictly)g(the)g(minim) o(um)g(necessary)g(to)e(ha)o(v)o(e)h(an)h(initial)h(MPI)e(subset)h (conforming)75 1379 y(implemen)o(tation.)28 b(It)18 b(is)g(hop)q(ed)g (that)f(an)g(o\016cially)i(sanctioned)g(subset)e(will)i(encourage)f (and)g(allo)o(w)75 1435 y(implemen)o(tors)c(to)e(in)o(tro)q(duce)j(MPI) e(in)h(a)e(more)h(timely)h(fashion.)20 b(It)13 b(should)h(b)q(e)g (noted,)f(ho)o(w)o(ev)o(er,)g(that)75 1492 y(implemen)o(tation)20 b(of)e(the)h(subset)g(do)q(es)g(not)f(mak)o(e)h(an)f(implemen)o(tation) i(conform)e(with)h(MPI.)g(The)75 1548 y(subset)g(is)g(only)h(a)e(p)q (oten)o(tial)i(\014rst)e(step)h(in)h(the)f(pro)q(cess.)31 b(All)20 b(implemen)o(tations)g(m)o(ust)f(ultimately)75 1605 y(conform)14 b(to)h(the)g(en)o(tire)g(standard;)g(implemen)o (tations)h(are)f(encouraged)g(to)f(do)h(the)g(full)i(standard)d(as)75 1661 y(rapidly)i(as)f(p)q(ossible.)166 1731 y(The)c(subset)g(presen)o (ted)g(is)g(consisten)o(t)f(with)h(the)g(complete)g(MPI)g(standard.)18 b(This)11 b(w)o(as)f(an)g(imp)q(or-)75 1787 y(tan)o(t)i(goal)g(so)h (the)g(additional)h(MPI)f(features)f(could)i(b)q(e)g(added)f(without)g (c)o(hanging)g(an)o(y)g(functionalit)o(y)75 1844 y(from)19 b(the)i(user's)f(p)q(ersp)q(ectiv)o(e.)36 b(Th)o(us,)21 b(users)f(can)g(b)q(e)h(assured)f(that)g(programs)f(written)h(no)o(w)g (for)75 1900 y(the)c(subset)f(will)i(run)f(without)g(mo)q(di\014cation) g(under)g(the)g(full)h(MPI)e(standard.)21 b(F)l(urthermore,)14 b(using)75 1957 y(additional)j(features)f(of)g(the)g(full)h(MPI)f (standard)f(in)i(the)f(future)g(will)i(not)d(require)i(c)o(hanges)f(to) f(co)q(de)75 2013 y(already)20 b(written.)35 b(This)20 b(requires)h(that)f(the)g(MPI)g(functions)g(ha)o(v)o(e)g(their)h(full)g (calling)h(argumen)o(ts)75 2070 y(in)f(the)g(subset.)36 b(T)l(o)20 b(ac)o(hiev)o(e)h(this)g(goal,)g(the)f(subset)h(will)h (limit)g(access)f(to)e(full)j(MPI)f(functional-)75 2126 y(it)o(y)e(through)h(complete)g(elimination)i(of)d(functions)h(or)f (limitations)i(on)e(the)h(creation)f(mec)o(hanisms)75 2182 y(for)d(certain)i(MPI)f(input)h(argumen)o(ts.)24 b(An)o(y)17 b(argumen)o(t)g(capable)h(of)e(b)q(eing)i(constructed)g (within)g(the)75 2239 y(subset)f(can)g(b)q(e)h(used)f(as)g(an)g (argumen)o(t)f(to)g(a)h(subset)g(function)h(where)f(it)g(w)o(ould)g(b)q (e)h(allo)o(w)o(ed)f(in)h(the)75 2295 y(full)h(standard.)28 b(F)l(or)17 b(example,)i(an)o(y)f(group)f(that)g(can)h(b)q(e)h(created) f(with)g(the)g(subset)g(is)h(an)f(allo)o(w)o(ed)75 2352 y(argumen)o(t)12 b(in)i(an)o(y)e(subset)i(function)f(where)g(a)g(group) g(is)g(an)g(appropriate)g(argumen)o(t.)18 b(Ho)o(w)o(ev)o(er,)12 b(some)75 2408 y(of)j(the)g(more)g(complex)h(mec)o(hanisms)g(for)e (group)h(creation)g(ha)o(v)o(e)g(b)q(een)i(remo)o(v)o(ed)d(from)h(the)g (subset.)166 2478 y(Users)g(ma)o(y)g(use)g(MPI)h(features)e(outside)i (this)g(subset)g(that)e(are)h(o\013ered)g(b)o(y)g(v)m(arious)h (implemen-)75 2534 y(tors.)27 b(Ho)o(w)o(ev)o(er,)18 b(p)q(eople)h(who)f(require)h(p)q(ortabilit)o(y)f(during)h(the)f(early) h(dev)o(elopmen)o(t)f(of)g(MPI)g(ma)o(y)75 2591 y(exp)q(erience)j(some) d(di\016culties)j(un)o(til)f(later)f(in)h(the)f(dev)o(elopmen)o(t)g (pro)q(cess.)31 b(This)19 b(subset)g(is)h(mean)o(t)75 2647 y(to)g(o\013er)g(the)g(opp)q(ortunit)o(y)h(for)f(a)g(greater)f (lev)o(el)j(of)e(p)q(ortabilit)o(y)i(un)o(til)f(the)g(full)h(MPI)e (standard)g(is)75 2704 y(implemen)o(ted)d(on)e(m)o(ultiple)i (platforms.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 136 142 136 141 bop 75 -100 a FE(136)592 b Fx(SECTION)16 b(10.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)166 45 y Fv(Discussion:)166 96 y Fu(A)o(t)h(the)h(June)g(meeting)e(w)o(e)i (decided)g(to)f(try)g(and)h(mak)o(e)d(a)i(single)g(subset.)29 b(It)18 b(w)o(as)f(felt)g(that)g(ha)o(ving)f(a)75 146 y(m)o(ultilev)o(el)d(subset)18 b(w)o(ould)d(b)q(e)h(more)f(complex)g (and)h(p)q(oten)o(tially)e(confusing.)24 b(Th)o(us,)17 b(an)e(item)g(is)h(either)h(in)e(or)75 196 y(out)f(of)f(the)i(subset.)k (This)14 b(will)e(only)h(b)q(e)i(reconsidered)h(if)d(this)h(binary)f(c) o(hoice)i(b)q(ecomes)f(a)f(serious)i(problem.)75 427 y Fy(10.2)59 b(Criteria)20 b(and)g(Rational)75 531 y FE(Ha)o(ving)f(the)g(subset)g(b)q(e)h(consisten)o(t)f(with)g(the)g(en)o (tire)g(MPI)g(standard)g(w)o(as)f(considered)i(critical)h(to)75 587 y(the)16 b(e\013ort.)22 b(In)16 b(addition)i(to)d(this,)h(there)h (w)o(ere)e(man)o(y)h(p)q(ossible)i(criteria)f(to)e(use)h(in)h (determining)h(the)75 644 y(elemen)o(ts)d(of)e(MPI)h(to)f(include)j(in) f(the)f(subset.)20 b(The)14 b(main)g(criteria)h(used)f(w)o(ere)g(that)f (the)h(MPI)g(subset)75 700 y(should)131 798 y(1.)22 b(con)o(tain)13 b(routines)h(that)e(are)h(as)g(close)h(as)f(p)q(ossible)i(to)e(curren)o (t)g(standard)g(practice)h(to)f(minimize)189 854 y(the)i(e\013ort)f(to) h(p)q(ort)f(co)q(des.)131 952 y(2.)22 b(con)o(tain)15 b(as)g(man)o(y)g(new)g(and)g(imp)q(ortan)o(t)g(MPI)g(features)g(as)g(p) q(ossible.)131 1050 y(3.)22 b(allo)o(w)17 b(dev)o(elop)q(ers)i(to)e(b)q (e)i(able)f(to)f(ha)o(v)o(e)g(the)h(subset)g(sho)o(w)f(up)h(as)f (rapidly)i(as)e(p)q(ossible)i(while)189 1106 y(still)d(meeting)g(the)f (other)g(criteria.)166 1204 y(There)i(w)o(ere)g(man)o(y)f(rationales)h (for)g(these)g(criteria.)26 b(It)17 b(w)o(as)f(recognized)i(that)e(it)h (is)h(imp)q(ossible)75 1261 y(to)h(come)h(up)g(with)g(an)g(ideal)h (subset)f(and)g(compromises)g(w)o(ere)g(necessary)l(.)34 b(It)19 b(w)o(as)g(felt)i(that)e(cur-)75 1317 y(ren)o(t)h(users)g (should)i(b)q(e)f(comfortable)f(in)h(migrating)f(to)g(MPI.)g(Th)o(us,)h (the)g(subset)f(should)i(con)o(tain)75 1374 y(the)17 b(message)f(passing)h(elemen)o(ts)g(that)f(represen)o(t)g(common)g (practice)i(to)q(da)o(y)l(.)23 b(F)l(urthermore,)16 b(it)h(w)o(as)75 1430 y(felt)f(that)g(the)g(initial)i(\(and)e(later\))g(v)o(ersion)g(of) g(MPI)g(should)h(main)o(tain)g(e\016ciency)l(.)24 b(Otherwise)17 b(users)75 1486 y(could)k(su\013er)e(from)g(disenc)o(han)o(tmen)o(t)h (with)g(MPI)f(on)h(their)g(\014rst)f(usage)h(and)f(ma)o(y)g(nev)o(er)h (giv)o(e)g(the)75 1543 y(standard)e(a)h(second)g(try)l(.)29 b(Ha)o(ving)19 b(the)g(standard)f(routines)h(in)h(the)e(subset)h (should)h(encourage)e(de-)75 1599 y(v)o(elop)q(ers)g(to)f(optimized)i (these)f(routines)g(in)h(the)f(initial)h(v)o(ersion)f(of)f(MPI.)h (Also,)g(though)f(some)h(felt)75 1656 y(that)c(the)g(p)q(ortabilit)o(y) h(o\013ered)f(b)o(y)g(the)h(MPI)f(subset)h(w)o(as)e(su\016cien)o(t)i (to)f(en)o(tice)h(users,)f(man)o(y)g(felt)h(that)75 1712 y(the)20 b(subset)g(should)g(con)o(tain)g(some)f(new)h(feature\(s\))f (o)o(v)o(er)g(and)g(ab)q(o)o(v)o(e)h(curren)o(t)f(common)g(practice)75 1769 y(as)d(an)g(added)h(incen)o(tiv)o(e.)24 b(The)17 b(additional)h(items)e(selected)i(w)o(ere)e(mean)o(t)f(to)h(represen)o (t)g(some)g(of)g(the)75 1825 y(signi\014can)o(t)i(new)f(features)f(of)h (MPI.)f(Balanced)i(against)f(these)g(\014rst)f(t)o(w)o(o)g(goals)h(is)g (the)g(need)h(for)e(the)75 1882 y(subset)e(to)f(sho)o(w)g(up)i(in)f(a)g (timely)h(fashion.)k(Th)o(us,)14 b(eac)o(h)g(feature)g(c)o(hosen)g(for) f(inclusion)j(in)f(the)f(subset)75 1938 y(w)o(as)g(deemed)h(of)f (su\016cien)o(t)h(imp)q(ortance)g(to)f(out)o(w)o(eigh)g(its)h(added)g (complexit)o(y)g(in)h(implemen)o(ting)g(the)75 1995 y(subset.)k(Though) 14 b(some)h(functions)g(seem)g(easy)f(to)g(implemen)o(t)i(there)e(are)h (often)f(o)o(v)o(erlo)q(ok)o(ed)g(costs)g(in)75 2051 y(testing,)f(do)q(cumen)o(tation)h(and)g(dev)o(elopmen)o(t)g(that)e(w)o (as)h(considered)h(b)q(efore)g(a)f(feature)g(w)o(as)g(added)h(to)75 2107 y(the)j(MPI)g(subset.)25 b(Inclusion)20 b(of)c(to)q(o)g(man)o(y)h (routines)g(migh)o(t)g(lead)h(to)e(mo)q(derately)i(e\016cien)o(t)f (imple-)75 2164 y(men)o(tation)g(of)f(them)h(all)h(instead)f(of)g(a)f (v)o(ery)h(e\016cien)o(t)h(and)f(p)q(ossibly)h(more)f(useful)h (implemen)o(tation)75 2220 y(of)d(a)g(smaller)h(subset.)166 2354 y Fv(Discussion:)166 2405 y Fu(Should)g(it)g(b)q(e)i(a)e(criteria) h(that)g(the)g(rest)h(of)e(MPI)h(b)q(e)g(able)g(to)f(b)q(e)h(la)o(y)o (ered)g(on)g(top)f(so)h(users)h(migh)o(t)d(get)75 2455 y(early)f(\(though)h(lo)o(w)o(er)f(p)q(erformance\))h(access)h(to)e (the)i(complete)e(standard?)20 b(This)15 b(is)f(aimed)f(at)h(allo)o (wing)f(easy)75 2504 y(and)g(e\016cien)o(t)g(la)o(y)o(ering)f(of)h(the) h(rest)g(so)f(that)g(a)g(\\mac)o(hine)e(indep)q(enden)o(t")j(pac)o(k)n (age)f(could)g(b)q(e)h(created)g(without)75 2554 y(ha)o(ving)e(the)i (dev)o(elop)q(er)g(jump)e(through)h(lots)g(of)g(hops.)18 b(This)13 b(migh)o(t)f(happ)q(en)h(if)g(a)g(few)g(functions)h(w)o(ere)g (left)f(out)75 2604 y(that)i(didn't)g(meet)f(the)i(other)g(criteria.)21 b(Ho)o(w)o(ev)o(er,)16 b(some)e(felt)h(it)f(w)o(as)h(not)g(a)g(strong)g (enough)h(reason)f(to)g(lea)o(v)o(e)75 2654 y(it)f(in)g(and)g(ma)o(y)f (lead)h(to)g(p)q(o)q(or)g(MPI)h(implemen)o(tatio)o(ns)d(of)i(these)i (la)o(y)o(ered)e(routines.)20 b(A)o(t)15 b(the)g(June)g(meeting)e(it)75 2704 y(w)o(as)h(decided)h(that)f(unless)g(this)g(turns)h(out)f(to)g(b)q (e)g(imp)q(ortan)o(t,)e(it)h(will)g(not)g(b)q(e)i(added)f(to)g(the)g (text.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 137 143 137 142 bop 75 -100 a Fx(10.3.)34 b(SUBSET)15 b(FUNCTIONALITY)1014 b FE(137)75 45 y Fy(10.3)59 b(Subset)19 b(F)n(unctionalit)n(y)75 226 y Fv(Discussion:)166 278 y Fu(Most)12 b(of)f(the)h(material)d(in)j (this)f(section)h(is)g(either)g(substan)o(tially)f(rew)o(ork)o(ed)h(or) g(new.)17 b(It)12 b(hop)q(efully)f(re\015ects)75 328 y(the)h(c)o(hanges)g(made)d(in)i(man)o(y)e(of)i(the)g(c)o(hapters)i (since)f(the)g(June)g(meeting.)k(As)11 b(suc)o(h,)h(it)f(has)g(not)g(b) q(een)h(discussed)75 378 y(b)o(y)i(the)h(MPI)f(group)g(and)g(still)g (needs)h(p)q(olishing.)j(I)c(w)o(ould)g(lik)o(e)f(to)h(agree)h(to)f (the)h(concepts)h(at)e(this)g(p)q(oin)o(t)g(and)75 428 y(then)g(p)q(olish)f(up)h(the)g(presen)o(tation.)19 b(Appropriate)14 b(text)g(will)e(b)q(e)i(added)g(after)g(w)o(e)g(agree)g(to)g(what)f(is) h(in/out)f(of)75 477 y(the)h(subset.)75 693 y Fq(10.3.1)49 b(P)o(oint)17 b(to)f(P)o(oint)h(F)o(unctionalit)o(y)75 782 y FE(The)j(table)h(b)q(elo)o(w)g(con)o(tains)f(a)g(summary)f(b)o(y) h(section)h(of)f(whic)o(h)h(features)e(of)h(the)g(p)q(oin)o(t)h(to)e(p) q(oin)o(t)75 839 y(comm)o(unications)d(c)o(hapter)f(are)g(included)j (and)d(excluded)i(from)e(the)g(subset.)120 955 y(Section)62 b(Title)400 b(Included)414 b(Excluded)p 75 978 1802 2 v 120 1030 a(4.2)146 b(Basic)16 b(Send)g(Op)q(eration)63 b Fo(MPI)p 905 1030 14 2 v 16 w(SEND)367 b FE(||)120 1099 y(4.2.1)110 b(Message)15 b(Data)217 b(All)17 b(t)o(yp)q(es)402 b(||)120 1168 y(4.3)146 b(Basic)16 b(Recv)g(Op)q(eration)61 b Fo(MPI)p 905 1168 V 16 w(RECV)369 b FE(||)120 1237 y(4.3.1)110 b(Return)16 b(Status)213 b Fo(MPI)p 905 1237 V 16 w(GET)p 1009 1237 V 17 w(SOURCE)205 b FE(||)820 1293 y Fo(MPI)p 905 1293 V 16 w(GET)p 1009 1293 V 17 w(T)l(A)o(G)820 1350 y(MPI)p 905 1350 V 16 w(GET)p 1009 1350 V 17 w(COUNT)120 1418 y FE(4.6)146 b(Data)14 b(Con)o(v)o(ersion) 163 b(||)492 b(\\All")120 1487 y(4.7)146 b(Comm.)19 b(Mo)q(des)202 b(Standard)400 b Fo(MPI)p 1487 1487 V 16 w(RSEND)1402 1544 y(MPI)p 1487 1544 V 16 w(SSEND)120 1613 y FE(4.8.2)110 b(Comm.)19 b(Initiation)147 b Fo(MPI)p 905 1613 V 16 w(ISEND)354 b(MPI)p 1487 1613 V 16 w(IRSEND)820 1669 y(MPI)p 905 1669 V 16 w(IRECV)i(MPI)p 1487 1669 V 16 w(ISSEND)120 1738 y FE(4.8.3)110 b(Comm.)19 b(Completion)102 b Fo(MPI)p 905 1738 V 16 w(W)l(AIT)371 b FE(||)820 1794 y Fo(MPI)p 905 1794 V 16 w(TEST)120 1863 y FE(4.8.3)110 b(Multiple)17 b(Completion)83 b(||)492 b Fo(MPI)p 1487 1863 V 16 w(W)l(AIT)l(ANY)1402 1920 y(MPI)p 1487 1920 V 16 w(TEST)l(ANY)1402 1976 y(MPI)p 1487 1976 V 16 w(W)l(AIT)l(ALL)1402 2033 y(MPI)p 1487 2033 V 16 w(TEST)l(ALL)120 2102 y FE(4.9)146 b(Prob)q(e)15 b(&)h(Cancel)178 b Fo(MPI)p 905 2102 V 16 w(PROBE)336 b(MPI)p 1487 2102 V 16 w(CANCEL)820 2158 y(MPI)p 905 2158 V 16 w(IPROBE)323 b(MPI)p 1487 2158 V 16 w(TEST)p 1617 2158 V 17 w(CANCEL)820 2215 y(MPI)p 905 2215 V 16 w(PROBE)p 1069 2215 V 18 w(COUNT)120 2283 y FE(4.10)123 b(P)o(ersisten)o(t)15 b(Comm.)139 b(||)492 b Fo(MPI)p 1487 2283 V 16 w(CREA)l(TE)p 1672 2283 V 18 w(SEND)355 2340 y FE(Ob)s(jects)894 b Fo(MPI)p 1487 2340 V 16 w(CREA)l(TE)p 1672 2340 V 18 w(RSEND)1402 2396 y(MPI)p 1487 2396 V 16 w(CREA)l(TE)p 1672 2396 V 18 w(SSEND)1402 2453 y(MPI)p 1487 2453 V 16 w(CREA)l(TE)p 1672 2453 V 18 w(RECV)1402 2509 y(MPI)p 1487 2509 V 16 w(ST)l(ART)1402 2566 y(MPI)p 1487 2566 V 16 w(FREE)120 2635 y FE(4.11)123 b(Send-receiv)o(e)252 b Fo(MPI)p 905 2635 V 16 w(SENDRECV)g(MPI)p 1487 2635 V 16 w(EX)o(CHANGE)120 2704 y FE(4.12)123 b(Null)17 b(pro)q(cess)252 b Fr(MPI)p 897 2704 13 2 v 15 w(PROCNULL)278 b FE(||)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 138 144 138 143 bop 75 -100 a FE(138)592 b Fx(SECTION)16 b(10.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)120 45 y FE(4.13.1)87 b(Deriv)o(ed)16 b(Datat)o(yp)q(es)122 b Fo(MPI)p 905 45 14 2 v 16 w(TYPE)p 1038 45 V 17 w(CONTIGUOUS)820 102 y(MPI)p 905 102 V 16 w(TYPE)p 1038 102 V 17 w(VECTOR)171 b(MPI)p 1487 102 V 16 w(TYPE)p 1620 102 V 17 w(HVECTOR)820 158 y(MPI)p 905 158 V 16 w(TYPE)p 1038 158 V 17 w(INDEXED)155 b(MPI)p 1487 158 V 16 w(TYPE)p 1620 158 V 17 w(HINDEXED)1402 214 y(MPI)p 1487 214 V 16 w(TYPE)p 1620 214 V 17 w(STRUCT)120 283 y FE(4.13.2)87 b(Additional)286 b Fo(MPI)p 905 283 V 16 w(TYPE)p 1038 283 V 17 w(COMMIT)164 b(MPI)p 1487 283 V 16 w(ADDRESS)355 340 y FE(F)l(unctions)273 b Fo(MPI)p 905 340 V 16 w(FREE)375 b(MPI)p 1487 340 V 16 w(TYPE)p 1620 340 V 17 w(EXTEND)166 499 y FE(In)19 b(addition)h(to)e(these)g(sp) q(eci\014c)j(functions,)e(all)h(the)e(features)h(and)f(criteria)h(of)g (p)q(oin)o(t)g(to)e(p)q(oin)o(t)75 556 y(comm)o(unication)j(hold)f(for) g(the)g(subset.)31 b(Th)o(us,)20 b(the)f(message)f(en)o(v)o(elop)q(e)i (in)g(subsection)g(4.2.2,)e(the)75 612 y(seman)o(tics)f(in)h(section)f (4.4,)f(the)h(t)o(yp)q(e)g(matc)o(hing)g(in)h(section)f(4.5)f(and)h (the)g(comm)o(unication)h(ob)s(jects)75 669 y(in)e(subsection)g(4.8.1)e (are)h(all)h(part)e(of)h(the)g(subset.)166 802 y Fv(Discussion:)166 854 y Fu(A)o(t)f(the)g(June)h(meeting)e(w)o(e)h(v)o(oted)g(to)g(remo)o (v)o(e)f(the)h(m)o(ultiple)e(completion)g(tests)j(from)d(the)j(subset) 166 1070 y Fv(Discussion:)166 1121 y Fu(A)o(t)d(the)g(June)h(meeting)e (w)o(e)h(v)o(oted)g(to)g(remo)o(v)o(e)f(INIT,)g(ST)m(AR)m(T,)f(etc.)18 b(This)12 b(is)g(carried)g(o)o(v)o(er)g(in)g(the)g(remo)o(v)n(al)75 1171 y(of)h(the)i(p)q(ersisten)o(t)g(comm)o(unication)c(ob)r(jects.)166 1388 y Fv(Discussion:)166 1439 y Fu(send-recv)17 b(is)e(new.)21 b(It)15 b(is)g(used)h(in)f(sev)o(eral)g(systems)g(no)o(w)g(and)g(is)g (somewhat)f(common)e(practice.)22 b(Users)75 1489 y(co)q(ding)13 b(this)h(often)g(mak)o(e)e(a)i(mistak)o(e.)i(The)f(pro)q(cn)o(ull)e (seems)h(easy)g(and)g(of)f(great)h(use)h(if)e(w)o(e)h(ha)o(v)o(e)f (send-recv.)20 b(I)75 1539 y(left)14 b(these)h(in)e(the)i(subset.)k (commen)o(ts?)166 1755 y Fv(Discussion:)166 1806 y Fu(Deriv)o(ed)d (datat)o(yp)q(es)h(are)f(new.)25 b(W)m(e)15 b(previously)h(v)o(oted)g (to)g(allo)o(w)e(the)j(con)o(tiguous)f(bu\013er)h(v)o(ersions)f(and)75 1856 y(limit)11 b(bu\013er)k(op)q(erations)f(to)g(only)f(one)h(app)q (end.)k(The)d(curren)o(t)g(draft)f(do)q(es)g(not)g(ha)o(v)o(e)g(these)h (t)o(w)o(o)e(v)o(ersions)i(but)75 1906 y(uses)h(the)f(generalized)g (datat)o(yp)q(e)f(whic)o(h)h(includes)f(the)i(basic)e(datat)o(yp)q(es)h (as)g(sp)q(ecial)f(cases.)21 b(The)15 b(ab)q(o)o(v)o(e)f(table)75 1956 y(allo)o(ws)d(for)g(the)h(non-\\h")g(v)o(ersions)g(so)g(that)g (arra)o(ys)g(and)f(sparse)i(matrix)d(p)q(eople)j(can)f(get)g(what)g (they)g(need)h(\(this)75 2006 y(also)g(is)g(closest)i(to)e(what)h(w)o (e)f(had)h(b)q(efore\).)19 b(The)14 b(heterogeneous)h(cases)g(\(h)o(v)o (ector,)f(hindexed\))g(seem)g(v)o(ery)g(easy)75 2055 y(to)e(co)q(de)i(but)f(are)g(another)g(set)g(of)f(functions.)18 b(They)12 b(allo)o(w)f(elemen)o(ts)i(of)f(a)g(C)g(t)o(yp)q(e)i (structure)g(to)f(b)q(e)g(pulled)f(out)75 2105 y(in)i(the)h(subset.)22 b(What)14 b(w)o(e)h(decide)g(will)e(also)h(a\013ect)i(the)f(additional) d(functions)j(that)g(are)g(included/excluded.)75 2155 y(W)m(e)e(need)i(to)f(discuss)h(the)g(righ)o(t)e(w)o(a)o(y)g(to)h (limit)d(this)j(functionalit)o(y)e(in)i(the)g(subset.)75 2367 y Fq(10.3.2)49 b(Collective)18 b(Communication)f(F)o(unctionalit)o (y)75 2456 y FE(The)e(table)h(b)q(elo)o(w)f(con)o(tains)g(a)g(summary)f (b)o(y)h(section)h(of)e(whic)o(h)i(features)f(of)f(the)h(collectiv)o(e) i(comm)o(u-)75 2512 y(nication)f(c)o(hapter)f(are)g(included)j(and)d (excluded)i(from)e(the)g(subset.)120 2629 y(Section)62 b(Title)419 b(Included)247 b(Excluded)p 75 2651 1709 2 v 120 2704 a(5.3)146 b(Barrier)15 b(Sync)o(h.)227 b Fo(MPI)p 924 2704 14 2 v 16 w(BARRIER)130 b FE(||)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 139 145 139 144 bop 75 -100 a Fx(10.3.)34 b(SUBSET)15 b(FUNCTIONALITY)1014 b FE(139)120 45 y(5.4.1)110 b(Broadcast)314 b Fo(MPI)p 924 45 14 2 v 16 w(BCAST)172 b FE(||)120 114 y(5.4.2)110 b(Gather)374 b Fo(MPI)p 924 114 V 16 w(GA)l(THER)142 b FE(||)120 183 y(5.4.3)110 b(Scatter)372 b Fo(MPI)p 924 183 V 16 w(SCA)l(TTER)119 b FE(||)120 252 y(5.4.4)110 b(All-to-all)17 b(broadcast)128 b Fo(MPI)p 924 252 V 16 w(ALLCAST)122 b FE(||)120 321 y(5.4.5)110 b(All-to-all)17 b(scatter-gather)44 b Fo(MPI)p 924 321 V 16 w(ALL)l(TO)o(ALL)98 b FE(||)120 390 y(5.5.1)110 b(Reduce)371 b Fo(MPI)p 924 390 V 16 w(REDUCE)141 b(MPI)p 1339 390 V 16 w(USER)p 1467 390 V 17 w(REDUCE)870 459 y FE(\(All)16 b(ops\))208 b Fo(MPI)p 1339 459 V 16 w(USER)p 1467 459 V 17 w(REDUCEA)839 527 y(MPI)p 924 527 V 16 w(ALLREDUCE)61 b(MPI)p 1339 527 V 16 w(USER)p 1467 527 V 17 w(ALLREDUCE)1254 596 y(MPI)p 1339 596 V 16 w(USER)p 1467 596 V 17 w(ALLREDUCEA)120 665 y FE(5.5.2)110 b(Scan)421 b(||)325 b Fo(MPI)p 1339 665 V 16 w(SCAN)120 734 y FE(5.5.2)1039 b Fo(MPI)p 1339 734 V 16 w(USER)p 1467 734 V 17 w(SCAN)120 803 y FE(5.5.2)g Fo(MPI)p 1339 803 V 16 w(USER)p 1467 803 V 17 w(SCANA)166 922 y FE(A)13 b(collectiv)o(e)j(comm)o(unication)e(op)q(erates)f(on)h (a)f(comm)o(unicator.)19 b(This)14 b(de\014nes)g(the)g(scop)q(e)g(of)f (the)75 978 y(op)q(eration.)20 b(A)14 b(collectiv)o(e)j(op)q(eration)e (can)f(accept)h(an)o(y)f(comm)o(unicator)g(that)g(can)h(b)q(e)g (created)g(within)75 1035 y(the)20 b(subset.)36 b(As)20 b(stated)g(previously)l(,)j(the)d(limitations)i(will)g(b)q(e)f(made)f (through)g(the)g(mec)o(hanisms)75 1091 y(a)o(v)m(ailable)d(to)d(create) h(and)h(manipulate)g(comm)o(unicators.)166 1226 y Fv(Discussion:)166 1278 y Fu(There)h(are)f(a)f(lot)g(of)g(data)g(mo)o(v)o(e)f(functions.) 23 b(I)16 b(could)f(not)h(think)f(of)g(a)g(w)o(a)o(y)g(to)h(dra)o(w)f (a)h(line)f(to)g(exclude)75 1328 y(some.)i(Do)c(w)o(e)i(w)o(an)o(t)e (all)g(of)g(these)i(in)f(the)g(subset.)20 b(Ho)o(w)13 b(often)h(are)g(ALLCAST,)g(ALLSCA)m(TTER)f(used?)166 1545 y Fv(Discussion:)166 1597 y Fu(F)m(or)g(the)i(subset)g(do)f(w)o(e) g(w)o(an)o(t)f(b)q(oth)h(the)h(REDUCE)f(and)f(ALLREDUCE)h(v)o(ersions?) 166 1815 y Fv(Discussion:)166 1867 y Fu(W)m(e)f(v)o(oted)h(at)g(the)h (June)f(meeting)f(to)h(remo)o(v)o(e)f(scan)h(from)e(the)j(subset)75 2084 y Fq(10.3.3)49 b(T)l(op)q(ology)20 b(F)o(unctionalit)o(y)75 2251 y Fv(Discussion:)166 2303 y Fu(One)c(of)e(the)i(new)f(features)h (of)f(MPI)g(is)g(pro)q(cess)i(top)q(ologies.)k(As)15 b(suc)o(h)h(it)f(w)o(ould)f(b)q(e)i(nice)f(to)g(ha)o(v)o(e)g(them)75 2353 y(in)h(the)h(subset.)27 b(Since)17 b(a)f(legitimate)e(remapping)h (is)h(to)g(do)g(nothing)g(it)g(can)g(b)q(e)h(argued)g(that)f(it)g(is)h (not)f(hard)75 2402 y(to)f(solv)o(e)h(the)g(mapping)e(problem)g(on)h(a) h(\014rst)g(pass.)24 b(Ho)o(w)o(ev)o(er,)16 b(to)f(implemen)o(t)e (something)i(that)g(gains)g(sp)q(eed)75 2452 y(for)h(the)g(user)h(will) d(tak)o(e)i(e\013ort.)25 b(W)m(e)16 b(ha)o(v)o(e,)f(so)h(far,)g(a)o(v)o (oid)f(putting)g(things)h(in)f(the)i(subset)g(if)e(it)h(w)o(as)g(b)q (eliev)o(ed)75 2502 y(that)i(the)g(initial)e(implemen)o(tati)o(on)f(w)o (ould)i(b)q(e)h(p)q(o)q(or)g(and)f(th)o(us)i(p)q(oten)o(tially)d (re\015ect)k(badly)c(on)i(that)g(part)g(of)75 2552 y(MPI.)d(Giv)o(en)f (all)f(of)i(this,)f(it)h(is)f(m)o(y)g(inclination)f(to)h(exclude)i(the) g(en)o(tire)f(top)q(ology)f(c)o(hapter)h(from)f(the)h(subset.)75 2602 y(Commen)o(ts?)166 2654 y(If)f(the)h(top)q(ology)f(section)h(w)o (ere)h(to)e(b)q(e)h(included)g(in)f(the)i(subset,)f(it)g(w)o(ould)e(b)q (e)j(p)q(ossible)f(to)f(only)g(include)75 2704 y(the)f(grid/tori)e (functions)h(and)f(remo)o(v)o(e)g(the)i(more)e(general)h(graph)g (functions.)17 b(This)12 b(w)o(ould)f(simplify)e(the)k(initial)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 140 146 140 145 bop 75 -100 a FE(140)592 b Fx(SECTION)16 b(10.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)75 45 y Fu(implemen)o(tatio)o(n)11 b(and)j(probably)f(co)o(v)o(er)i(a)e(large)h (p)q(ercen)o(tage)h(of)f(applications.)75 253 y Fq(10.3.4)49 b(Language)18 b(Binding)75 417 y Fv(Discussion:)166 467 y Fu(I)d(assume)f(w)o(e)i(w)o(an)o(t)e(to)h(supp)q(ort)h(b)q(oth)f(the) g(F77)g(and)g(C)g(binding)f(in)g(the)i(subset.)22 b(The)16 b(F)m(ortran)f(90)f(and)75 517 y(C++)g(bindings)g(are)g(done)g(through) g(F77)g(and)f(C)h(and)g(th)o(us)g(are)g(not)g(an)g(issue)g(for)g(the)g (subset.)75 725 y Fq(10.3.5)49 b(Environmental)17 b(Management)f(and)h (Inquiry)75 889 y Fv(Discussion:)166 939 y Fu(Clearly)e(the)i(startup)g (mec)o(hanism)d(m)o(ust)h(b)q(e)i(in)f(the)h(subset.)26 b(W)m(e)16 b(also)f(need)i Fr(MPI)p 1524 939 13 2 v 15 w(rank)g Fu(and)f Fr(MPI)p 1789 939 V 14 w(size)p Fu(.)75 989 y(After)21 b(this)f(it)g(is)g(up)h(for)f(grabs.)37 b(I)20 b(don't)g(think)g(an)o(y)g(of)f(the)i(others)g(are)g(essen)o (tial)g(to)f(the)h(subset.)38 b(F)m(or)75 1039 y(simpli\014cation)11 b(reasons)k(ma)o(yb)q(e)e(w)o(e)h(should)f(remo)o(v)o(e)g(all)g(the)i (other)f(calls.)k(Commen)o(ts?)75 1247 y Fq(10.3.6)49 b(Pro\014ling)18 b(F)o(unctionalit)o(y)75 1410 y Fv(Discussion:)166 1461 y Fu(The)e(pro\014ling)f(c)o(hapter)j(is)d(easy)i(to)f(implemen)o (t)d(but)j(doubles)h(the)f(n)o(um)o(b)q(er)g(of)f(functions.)25 b(As)16 b(suc)o(h,)h(it)75 1511 y(increase)f(the)f(w)o(ork)f(that)g(m)o (ust)g(b)q(e)h(done)f(b)o(y)g(an)h(implem)o(en)o(ter)e(though)h(this)h (could)f(easily)g(b)q(e)h(automated.)j(It)75 1560 y(do)q(es)e(not)f (seem)g(crucial)h(to)f(other)h(parts)f(of)g(the)h(subset)h(nor)e(a)g (signi\014can)o(t)g(new)g(feature.)23 b(Not)16 b(sure)g(what)f(to)75 1610 y(do)f(here)h(but)f(from)e(a)i(simpli\014cation)d(stand)j(p)q(oin) o(t)f(it)h(w)o(ould)f(b)q(e)h(b)q(est)h(to)f(also)f(drop)h(this)g(c)o (hapter.)75 1819 y Fq(10.3.7)49 b(Maps,)17 b(Groups)e(and)i(Contexts)75 1982 y Fv(Discussion:)166 2032 y Fu(W)m(e)e(ha)o(v)o(e)h(not)f (discussed)j(a)d(single)h(uni\014ed)g(prop)q(osal)f(y)o(et)h(for)g (this)f(c)o(hapter.)25 b(Un)o(til)15 b(w)o(e)h(pin)f(do)o(wn)h(to)f(a) 75 2082 y(greater)e(exten)o(t)g(what)f(will)f(b)q(e)i(in)f(the)g(full)f (standard)i(I)f(am)f(dela)o(ying)g(doing)g(an)o(ything)g(for)h(the)h (subset.)19 b(Sev)o(eral)75 2132 y(p)q(eople)13 b(ha)o(v)o(e)g (expressed)j(that)d(con)o(texts)h(are)g(a)e(signi\014can)o(t)h(new)h (feature)f(in)g(MPI)g(and)g(w)o(e)h(should)e(try)i(and)f(get)75 2182 y(them)g(in)o(to)g(the)i(subset.)75 2390 y Fq(10.3.8)49 b(Subset)16 b(T)l(esting)75 2553 y Fv(Discussion:)166 2604 y Fu(There)e(has)f(b)q(een)h(discussion)g(of)e(creating)i(a)f (test)h(suite)f(for)g(MPI.)g(W)m(ould)e(it)i(b)q(e)h(p)q(ossible/mak)o (e)d(sense)k(to)75 2654 y(create)j(a)e(subset)h(test)g(suite)g(so)f (that)g(these)i(early)e(v)o(ersions)h(could)f(b)q(e)h(tested)g(and)f (not)g(simply)f(ha)o(v)o(e)h(p)q(eople)75 2704 y(sa)o(y)e(that)g(the)g (full)f(test)i(suite)f(will)f(ob)o(viously)f(fail)g(in)i(some)f(w)o(a)o (ys?)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 141 147 141 146 bop 75 377 a FG(Biblio)q(graph)m(y)98 600 y FE([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 656 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 713 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 806 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 863 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 919 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1013 y([3])j(R.)14 b(Butler)h(and)f(E.)f(Lusk.)18 b(User's)13 b(guide)i(to)e(the)h(P4)g(programming)f(system.)k(T)l(ec)o(hnical)f (Rep)q(ort)168 1069 y(TM-ANL{92/17,)e(Argonne)h(National)h(Lab)q (oratory)l(,)e(1992.)98 1163 y([4])21 b(S.)c(Chittor)g(and)h(R.)f(J.)g (En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d (mesh{connected)i(w)o(ormhole{)168 1220 y(routed)f(net)o(w)o(orks)e (for)i(in)o(terpro)q(cessor)f(comm)o(unication)i(in)f(m)o (ulticomputers.)25 b(In)17 b Ft(Pr)n(o)n(c)n(e)n(e)n(dings)168 1276 y(of)g(the)f(1990)i(Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p FE(,)c(pages)j(647{656,)e(1990.)98 1370 y([5])21 b(S.)f(Chittor)g(and)g (R.)g(J.)f(En)o(b)q(o)q(dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g (of)f(mapping)i(on)e(the)h(comm)o(unica-)168 1426 y(tion)g(p)q (erformance)g(of)f(large)g(m)o(ulticomputers.)34 b(In)20 b Ft(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f(1991)h(International)168 1483 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n (essing,)f(vol.)i(II)f(\(Softwar)n(e\))p FE(,)f(pages)h(I)q(I{1)h({)f (I)q(I{4,)g(1991.)98 1577 y([6])21 b(Edin)o(burgh)g(P)o(arallel)f (Computing)g(Cen)o(tre,)g(Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Ft(CHIMP)19 b(Conc)n(epts)p FE(,)168 1633 y(June)e(1991.)98 1727 y([7])k(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f (Univ)o(ersit)o(y)h(of)g(Edin)o(burgh.)22 b Ft(CHIMP)15 b(V)m(ersion)h(1.0)168 1783 y(Interfac)n(e)p FE(,)e(Ma)o(y)g(1992.)98 1877 y([8])21 b(G.)13 b(Geist)f(and)h(V.)g(Sunderam.)k(Net)o(w)o(ork)11 b(based)i(concurren)o(t)g(computing)h(on)e(the)h(PVM)g(system.)168 1933 y(T)l(ec)o(hnical)k(Rep)q(ort)f(TM-11760,)d(Oak)i(Ridge)i (National)e(Lab)q(oratory)l(,)f(June)j(1991.)98 2027 y([9])k(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o(eyton,)f (and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f (PICL:)g(a)168 2084 y(p)q(ortable)f(instrumen)o(ted)f(comm)o(unication) h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e(TM-11616,)f(Oak)h(Ridge) 168 2140 y(National)k(Lab)q(oratory)l(,)e(Octob)q(er)i(1990.)75 2234 y([10])21 b(R.)15 b(Hemp)q(el.)20 b(The)14 b(ANL/GMD)g(macros)f (\(P)l(ARMA)o(CS\))h(in)i(fortran)d(for)g(p)q(ortable)i(parallel)h (pro-)168 2290 y(gramming)f(using)g(the)g(message)f(passing)h (programming)f(mo)q(del)i({)e(users')h(guide)g(and)g(reference)168 2347 y(man)o(ual.)23 b(T)l(ec)o(hnical)18 b(rep)q(ort,)d(GMD,)g(P)o (ostfac)o(h)f(1316,)h(D-5205)g(Sankt)h(Augustin)g(1,)g(German)o(y)l(,) 168 2403 y(No)o(v)o(em)o(b)q(er)f(1991.)75 2497 y([11])21 b(R.)15 b(Hemp)q(el,)h(H.-C.)e(Hopp)q(e,)h(and)f(A.)h(Supalo)o(v.)k(P)o (armacs{6.0)12 b(library)k(in)o(terface)e(sp)q(eci\014cation.)168 2553 y(T)l(ec)o(hnical)h(rep)q(ort,)d(GMD,)g(P)o(ostfac)o(h)f(1316,)h (D-5205)f(Sankt)h(Augustin)i(1,)e(German)o(y)l(,)h(Decem)o(b)q(er)168 2610 y(1992.)75 2704 y([12])21 b(nCUBE)16 b(Corp)q(oration.)j Ft(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i(r2.0)p FE(,)f(Decem)o(b)q(er)f(1990.)-32 46 y FC(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 142 148 142 147 bop 75 -100 a FE(142)1368 b Fx(BIBLIOGRAPHY)75 45 y FE([13])21 b(H.)c(M)q(\177)-24 b(uhlen)o(b)q(ein)19 b(O.)d(Kr\177)-23 b(amer.)24 b(Mapping)16 b(strategies)g(in)i (message{based)e(m)o(ultipro)q(cessor)h(sys-)168 102 y(tems.)j Ft(Par)n(al)r(lel)15 b(Computing)p FE(,)g(9:213{225,)d(1989.) 75 195 y([14])21 b(P)o(arasoft)14 b(Corp)q(oration.)22 b Ft(Expr)n(ess)16 b(V)m(ersion)g(1.0:)22 b(A)17 b(Communic)n(ation)g (Envir)n(onment)e(for)j(Par-)168 252 y(al)r(lel)e(Computers)p FE(,)f(1988.)75 346 y([15])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g(system.)22 b(In)17 b Ft(Pr)n(o)n(c)n(e)n(e)n(dings)e(of)i(the)g(Thir)n(d)g(Confer)n(enc)n(e) e(on)168 402 y(Hyp)n(er)n(cub)n(e)g(Concurr)n(ent)f(Computers)i(and)f (Applic)n(ations)p FE(,)e(pages)h(384{390.)d(A)o(CM)j(Press,)f(1988.)75 496 y([16])21 b(A.)14 b(Skjellum)i(and)e(A.)g(Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f(comm)o (unication)h(library)168 552 y(atop)g(the)f(reactiv)o(e)h(k)o(ernel.)20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g(editors,)g Ft(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 609 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g(Confer)n(enc)n (e)p FE(,)e(pages)i(767{776.)d(IEEE)168 665 y(Press,)d(1990.)75 759 y([17])21 b(A.)12 b(Skjellum,)i(S.)e(Smith,)h(C.)e(Still,)j(A.)d (Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q(co)q(de)g(message)f(passing) 168 816 y(system.)20 b(T)l(ec)o(hnical)d(rep)q(ort,)d(La)o(wrence)i (Liv)o(ermore)f(National)h(Lab)q(oratory)l(,)e(Septem)o(b)q(er)i(1992.) 75 909 y([18])21 b(V.)e(Sunderam.)30 b(PVM:)18 b(a)g(framew)o(ork)f (for)h(parallel)i(distributed)h(computing.)30 b Ft(Concurr)n(ency:)168 966 y(Pr)n(actic)n(e)16 b(and)g(Exp)n(erienc)n(e)p FE(,)e (2\(4\):315{339)o(,)e(1990.)75 1060 y([19])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g(passing)g(in)i(a)e (distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(h-)168 1116 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National)g(Lab)q (oratory)l(,)e(August)h(1992.)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 143 149 143 148 bop 75 283 a FG(MPI)40 b(F)-10 b(unction)40 b(Index)75 490 y FE(MPI)p 167 490 14 2 v 16 w(ADDRESS,)16 b(77)75 547 y(MPI)p 167 547 V 16 w(ALLCAST,)g(94)75 604 y(MPI)p 167 604 V 16 w(ALLREDUCE,)g(99)75 660 y(MPI)p 167 660 V 16 w(ALL)l(TO)o(ALL,)h(95)75 717 y(MPI)p 167 717 V 16 w(A)l(TTRIBUTE)p 458 717 V 18 w(ALLOC,)e(28)75 773 y(MPI)p 167 773 V 16 w(A)l(TTRIBUTE)p 458 773 V 18 w(FREE,)f(28)75 874 y(MPI)p 167 874 V 16 w(BARRIER,)j(91)75 931 y(MPI)p 167 931 V 16 w(BCAST,)e(93)75 1031 y(MPI)p 167 1031 V 16 w(CANCEL,)g(64)75 1088 y(MPI)p 167 1088 V 16 w(CAR)l(T,)h(110)75 1144 y(MPI)p 167 1144 V 16 w(COLL)p 307 1144 V 18 w(SUBGR)o(OUP)l(,)g (18)75 1201 y(MPI)p 167 1201 V 16 w(COMM)p 335 1201 V 16 w(CONTEXT,)f(20)75 1257 y(MPI)p 167 1257 V 16 w(COMM)p 335 1257 V 16 w(DUP)l(,)g(21)75 1314 y(MPI)p 167 1314 V 16 w(COMM)p 335 1314 V 16 w(FREE,)g(68)75 1371 y(MPI)p 167 1371 V 16 w(COMM)p 335 1371 V 16 w(GR)o(OUP)l(,)g(20)75 1427 y(MPI)p 167 1427 V 16 w(COMM)p 335 1427 V 16 w(INIT,)h(87)75 1484 y(MPI)p 167 1484 V 16 w(COMM)p 335 1484 V 16 w(MAKE,)f(21)75 1540 y(MPI)p 167 1540 V 16 w(COMM)p 335 1540 V 16 w(NAME)p 492 1540 V 16 w(MAKE,)g(25)75 1597 y(MPI)p 167 1597 V 16 w(COMM)p 335 1597 V 16 w(NAME)p 492 1597 V 16 w(MAKE)p 650 1597 V 17 w(FINISH,)h(26)75 1654 y(MPI)p 167 1654 V 16 w(COMM)p 335 1654 V 16 w(NAME)p 492 1654 V 16 w(MAKE)p 650 1654 V 17 w(ST)l(AR)l(T,)f(26)75 1710 y(MPI)p 167 1710 V 16 w(COMM)p 335 1710 V 16 w(PEER)p 477 1710 V 17 w(MAKE,)g(24)75 1767 y(MPI)p 167 1767 V 16 w(COMM)p 335 1767 V 16 w(PEER)p 477 1767 V 17 w(MAKE)p 636 1767 V 16 w(FINISH,)h(25)75 1823 y(MPI)p 167 1823 V 16 w(COMM)p 335 1823 V 16 w(PEER)p 477 1823 V 17 w(MAKE)p 636 1823 V 16 w(ST)l(AR)l(T,)g(25)75 1880 y(MPI)p 167 1880 V 16 w(COMM)p 335 1880 V 16 w(RANK,)g(20)75 1937 y(MPI)p 167 1937 V 16 w(COMM)p 335 1937 V 16 w(SIZE,)f(20)75 1993 y(MPI)p 167 1993 V 16 w(COMM)p 335 1993 V 16 w(SPLITL,)h(27)75 2050 y(MPI)p 167 2050 V 16 w(COMM)p 335 2050 V 16 w(ST)l(A)l(T,)f(26)75 2106 y(MPI)p 167 2106 V 16 w(COMM)p 335 2106 V 16 w(UNBIND,)h(20)75 2163 y(MPI)p 167 2163 V 16 w(CONTEXTS)p 441 2163 V 17 w(ALLOC,)g(19)75 2220 y(MPI)p 167 2220 V 16 w(CONTEXTS)p 441 2220 V 17 w(FREE,)f(18)75 2276 y(MPI)p 167 2276 V 16 w(CREA)l(TE)p 374 2276 V 17 w(GR)o(OUP)l(,)g(107)75 2333 y(MPI)p 167 2333 V 16 w(CREA)l(TE)p 374 2333 V 17 w(RECV,)g(67)75 2389 y(MPI)p 167 2389 V 16 w(CREA)l(TE)p 374 2389 V 17 w(RSEND,)g(66)75 2446 y(MPI)p 167 2446 V 16 w(CREA)l(TE)p 374 2446 V 17 w(SEND,)g(66)75 2503 y(MPI)p 167 2503 V 16 w(CREA)l(TE)p 374 2503 V 17 w(SSEND,)g(67)75 2603 y(MPI)p 167 2603 V 16 w(DELETE)p 372 2603 V 17 w(A)l(TTRIBUTE,)g (29)75 2704 y(MPI)p 167 2704 V 16 w(EX)o(CHANGE,)g(69)1048 490 y(MPI)p 1140 490 V 16 w(GA)l(THER,)g(93)1048 547 y(MPI)p 1140 547 V 16 w(GET)p 1256 547 V 16 w(A)l(TTRIBUTE)p 1547 547 V 17 w(KEY,)g(28)1048 603 y(MPI)p 1140 603 V 16 w(GET)p 1256 603 V 16 w(COUNT,)g(46)1048 660 y(MPI)p 1140 660 V 16 w(GET)p 1256 660 V 16 w(SOUR)o(CE,)g(46)1048 716 y(MPI)p 1140 716 V 16 w(GET)p 1256 716 V 16 w(T)l(A)o(G,)f(46)1048 773 y(MPI)p 1140 773 V 16 w(GRAPH,)h(110)1048 829 y(MPI)p 1140 829 V 16 w(GRAPHDIMS,)g(111)1048 886 y(MPI)p 1140 886 V 16 w(GR)o(OUP)p 1324 886 V 17 w(DUP)l(,)f(18)1048 942 y(MPI)p 1140 942 V 16 w(GR)o(OUP)p 1324 942 V 17 w(FREE,)g(17)1048 999 y(MPI)p 1140 999 V 16 w(GR)o(OUP)p 1324 999 V 17 w(RANK,)h(15)1048 1055 y(MPI)p 1140 1055 V 16 w(GR)o(OUP)p 1324 1055 V 17 w(SIZE,)g(15)1048 1148 y(MPI)p 1140 1148 V 16 w(INQCAR)l(T,)h(112)1048 1204 y(MPI)p 1140 1204 V 16 w(INQDIM,)f(112)1048 1260 y(MPI)p 1140 1260 V 16 w(INQGRAPH,)h(111)1048 1317 y(MPI)p 1140 1317 V 16 w(INQLOC,)g(112)1048 1373 y(MPI)p 1140 1373 V 16 w(INQMAP)l(,)g(111)1048 1430 y(MPI)p 1140 1430 V 16 w(INQRANK,)h(112)1048 1486 y(MPI)p 1140 1486 V 16 w(IPR)o(OBE,)f(63)1048 1543 y(MPI)p 1140 1543 V 16 w(IRECV,)g(57)1048 1599 y(MPI)p 1140 1599 V 16 w(IRSEND,)g(56)1048 1656 y(MPI)p 1140 1656 V 16 w(ISEND,)f(56)1048 1712 y(MPI)p 1140 1712 V 16 w(ISSEND,)g(57)1048 1805 y(MPI)p 1140 1805 V 16 w(LOCAL)p 1314 1805 V 17 w(EX)o(CL)p 1456 1805 V 17 w(SUBGR)o(OUP)l(,)h(15)1048 1861 y(MPI)p 1140 1861 V 16 w(LOCAL)p 1314 1861 V 17 w(GR)o(OUP)p 1499 1861 V 17 w(DIFFERENCE,)9 b(17)1048 1918 y(MPI)p 1140 1918 V 16 w(LOCAL)p 1314 1918 V 17 w(GR)o(OUP)p 1499 1918 V 17 w(INTERSECT,)15 b(17)1048 1974 y(MPI)p 1140 1974 V 16 w(LOCAL)p 1314 1974 V 17 w(GR)o(OUP)p 1499 1974 V 17 w(UNION,)h(17)1048 2031 y(MPI)p 1140 2031 V 16 w(LOCAL)p 1314 2031 V 17 w(SUBGR)o(OUP)l(,)g(15)1048 2087 y(MPI)p 1140 2087 V 16 w(LOCAL)p 1314 2087 V 17 w(SUBGR)o(OUP)p 1590 2087 V 18 w(EX)o(CL)p 1733 2087 V 16 w(RANGES,)1214 2143 y(16)1048 2200 y(MPI)p 1140 2200 V 16 w(LOCAL)p 1314 2200 V 17 w(SUBGR)o(OUP)p 1590 2200 V 18 w(RANGES,)f(16)1048 2293 y(MPI)p 1140 2293 V 16 w(MAKDIM,)g(113)1048 2349 y(MPI)p 1140 2349 V 16 w(MAP)p 1263 2349 V 16 w(CAR)l(T,)g(108)1048 2405 y(MPI)p 1140 2405 V 16 w(MAP)p 1263 2405 V 16 w(GRAPH,)g(108)1048 2498 y(MPI)p 1140 2498 V 16 w(P)l(AR)l(TC,)g(113)1048 2555 y(MPI)p 1140 2555 V 16 w(PR)o(OBE,)g(64)1048 2611 y(MPI)p 1140 2611 V 16 w(PR)o(OBE)p 1317 2611 V 17 w(COUNT,)g(63)1048 2704 y(MPI)p 1140 2704 V 16 w(RECV,)g(45)941 2828 y(143)p eop %%Page: 144 150 144 149 bop 75 -100 a FE(144)1326 b Fx(MPI)15 b(F)l(unction)h(Index)75 45 y FE(MPI)p 167 45 14 2 v 16 w(REDUCE,)f(96)75 102 y(MPI)p 167 102 V 16 w(RSEND,)h(54)75 200 y(MPI)p 167 200 V 16 w(SCAN,)f(100)75 256 y(MPI)p 167 256 V 16 w(SCA)l(TTER,)h(94) 75 312 y(MPI)p 167 312 V 16 w(SEND,)f(43)75 369 y(MPI)p 167 369 V 16 w(SENDRECV,)g(69)75 425 y(MPI)p 167 425 V 16 w(SET)p 272 425 V 17 w(A)l(TTRIBUTE,)g(29)75 482 y(MPI)p 167 482 V 16 w(SHIFT)p 321 482 V 17 w(ID,)g(113)75 538 y(MPI)p 167 538 V 16 w(SSEND,)g(54)75 595 y(MPI)p 167 595 V 16 w(ST)l(AR)l(T,)h(67)75 693 y(MPI)p 167 693 V 16 w(TEST,)f(58)75 749 y(MPI)p 167 749 V 16 w(TEST)p 305 749 V 17 w(A)l(TTRIBUTE,)g(29)75 806 y(MPI)p 167 806 V 16 w(TEST)p 305 806 V 17 w(CANCELLED,)g(65)75 862 y(MPI)p 167 862 V 16 w(TEST)l(ALL,)h(61)75 918 y(MPI)p 167 918 V 16 w(TEST)l(ANY,)f(60)75 975 y(MPI)p 167 975 V 16 w(TRANSLA)l(TE)p 464 975 V 18 w(RANKS,)h(15)75 1031 y(MPI)p 167 1031 V 16 w(TYPE)p 312 1031 V 17 w(COMMIT,)e(78)75 1088 y(MPI)p 167 1088 V 16 w(TYPE)p 312 1088 V 17 w(CONTIGUOUS,)h(72)75 1144 y(MPI)p 167 1144 V 16 w(TYPE)p 312 1144 V 17 w(EXTENT,)f(77)75 1201 y(MPI)p 167 1201 V 16 w(TYPE)p 312 1201 V 17 w(FREE,)g(79)75 1257 y(MPI)p 167 1257 V 16 w(TYPE)p 312 1257 V 17 w(HINDEXED,)g(76)75 1314 y(MPI)p 167 1314 V 16 w(TYPE)p 312 1314 V 17 w(HVECTOR,)h(74)75 1370 y(MPI)p 167 1370 V 16 w(TYPE)p 312 1370 V 17 w(INDEXED,)f(75)75 1427 y(MPI)p 167 1427 V 16 w(TYPE)p 312 1427 V 17 w(STR)o(UCT,)g(76)75 1483 y(MPI)p 167 1483 V 16 w(TYPE)p 312 1483 V 17 w(VECTOR,)h(73)75 1581 y(MPI)p 167 1581 V 16 w(USER)p 306 1581 V 17 w(ALLREDUCE,)h(100)75 1637 y(MPI)p 167 1637 V 16 w(USER)p 306 1637 V 17 w(ALLREDUCEA,)g(100) 75 1694 y(MPI)p 167 1694 V 16 w(USER)p 306 1694 V 17 w(REDUCE,)f(98)75 1750 y(MPI)p 167 1750 V 16 w(USER)p 306 1750 V 17 w(REDUCEA,)g(98)75 1807 y(MPI)p 167 1807 V 16 w(USER)p 306 1807 V 17 w(SCAN,)h(101)75 1863 y(MPI)p 167 1863 V 16 w(USER)p 306 1863 V 17 w(SCANA,)g(101)75 1961 y(MPI)p 167 1961 V 16 w(W)-5 b(AIT,)15 b(58)75 2018 y(MPI)p 167 2018 V 16 w(W)-5 b(AIT)l(ALL,)16 b(61)75 2074 y(MPI)p 167 2074 V 16 w(W)-5 b(AIT)l(ANY,)16 b(60)1967 46 y FC(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .