%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: mpi-report.dvi %%CreationDate: Sun Feb 20 14:10:34 1994 %%Pages: 231 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips -o mpi-report.ps mpi-report %DVIPSSource: TeX output 1994.02.20:1408 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /@rigin{isls{[ 0 -1 1 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{Resolution hsize -72 div mul 0 TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{ /nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{ /sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0] N df-tail}B /E{pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get}B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{ 128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]} if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{ cc 1 add D}B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258431 52099146 1000 300 300 (/ogi/staff/otto/papers/Editor/MPI/draft7/mpi-report.dvi) @start /Fa 2 82 df<03FFC00FFFF030E1F860C07CC0C03C81C01C01C01801C01801C0 30018020018040038380033E000370000700000600000600000E00000C00000C00001800 0016157F9318>80 D<003F0000FFC00307E00401E00800F01000F0300070600070600070 E00070E00060E00060F000C0F000807C31807FE3001F06000008000070003FC0107FC030 0FF86001FFC0003F0014187D931B>I E /Fb 27 121 df<387C7E7E3E0E1E1C78F06007 0B798416>44 DI<03E0000FF8001FFC001E3C0038 0E00780F00700700700700E00380E00380E00380E00380E00380E00380E00380E00380F0 0780700700700700780F003C1E001E3C001FFC000FF80003E00011197E9816>48 D<01800380038007800F807F80FF80738003800380038003800380038003800380038003 8003800380038003807FF87FFC7FF80E197C9816>I<7F1FC0FFBFE07F1FC01C07001C07 001C07001C07001C07001C07001C07001FFF001FFF001FFF001C07001C07001C07001C07 001C07001C07001C07001C07001C07007F1FC0FFBFE07F1FC013197F9816>72 D<1FFC003FFE007FFF00780F00F00780E00380E00380E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380E00380E00380F00780F00780780F007FFF003FFE 001FFC0011197E9816>79 D<07E3001FFF003FFF00781F00F00700E00700E00700E00000 F000007800003F80001FF00007FC0000FE00000F00000700000380000380600380E00380 E00700F80F00FFFE00FFFC00C7F00011197E9816>83 D<7FFFE0FFFFE0FFFFE0E0E0E0E0 E0E0E0E0E0E0E0E000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00007FC000FFE0007FC0013197F9816>I<1FE000 3FF0007FF800783C00300E00000E00000E0003FE001FFE003E0E00700E00E00E00E00E00 E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<03F80FFC1FFE3C1E780C7000E0 00E000E000E000E000F000700778073E0E1FFC0FF803F010127D9116>99 D<003F00007F00003F0000070000070000070000070003C7000FF7001FFF003C1F00780F 00700700E00700E00700E00700E00700E00700E00700700F00700F003C1F001FFFE00FE7 F007C7E014197F9816>I<03E00FF81FFC3C1E780E7007E007FFFFFFFFFFFFE000E00070 0778073C0F1FFE0FFC03F010127D9116>I<001F00007F8000FF8001E78001C30001C000 01C0007FFF00FFFF00FFFF0001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C0003FFE007FFF003FFE0011197F9816>I<03E3C007F7E00FFF E01C1CC0380E00380E00380E00380E00380E001C1C000FF8001FF0001BE0003800001800 001FFC001FFF003FFF807803C0E000E0E000E0E000E0E000E07001C07C07C03FFF800FFE 0003F800131C7F9116>I<018003C003C0018000000000000000007FC07FC07FC001C001 C001C001C001C001C001C001C001C001C001C001C07FFFFFFF7FFF101A7D9916>105 D<003000780078003000000000000000001FF81FF81FF800380038003800380038003800 380038003800380038003800380038003800380038003800386070F0F0FFE07FC03F800D 237E9916>I108 DI<7E3C00FEFE007FFF000F87800F03800E03800E03800E03800E03800E03 800E03800E03800E03800E03800E03807FC7F0FFE7F87FC7F01512809116>I<03E0000F F8001FFC003C1E00780F00700700E00380E00380E00380E00380E00380F0078070070078 0F003C1E001FFC000FF80003E00011127E9116>I<7E3E00FEFF007FFF800F83C00F00E0 0E00E00E00700E00700E00700E00700E00700E00700E00E00F01E00F83C00FFF800EFF00 0E3C000E00000E00000E00000E00000E00000E00007FC000FFE0007FC000141B809116> I114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007F8003C600EE00EF00EF81EFFFCFFF8C7 E00F127D9116>I<0300000700000700000700000700007FFF00FFFF00FFFF0007000007 000007000007000007000007000007000007010007038007038007038007870003FE0001 FC0000F80011177F9616>I<7E1F80FE3F807E1F800E03800E03800E03800E03800E0380 0E03800E03800E03800E03800E03800E03800E0F800FFFF007FBF803E3F01512809116> I119 D<7F1FC07F3FC07F1FC00F1C00073C0003B80003F00001F00000E00001E00001F00003B8 00073C00071C000E0E007F1FC0FF3FE07F1FC013127F9116>I E /Fc 1 1 df0 D E /Fd 1 106 df<040C00000000007058 98983030606464683006127E910B>105 D E /Fe 11 90 df22 D<003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038FFF8FFF8FFF80D6380 8118>I<0000180000300000600000E00000C0000180000380000700000600000E00000C 00001C0000380000380000700000700000E00000E00001E00001C00001C0000380000380 000380000780000700000700000F00000E00000E00001E00001E00001E00001C00001C00 003C00003C00003C00003C00003800007800007800007800007800007800007800007800 00780000700000F00000F00000F00000F00000F00000F00000F00000F00000F00000F000 00F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F000 00F00000F00000F00000F000007000007800007800007800007800007800007800007800 007800003800003C00003C00003C00003C00001C00001C00001E00001E00001E00000E00 000E00000F000007000007000007800003800003800003800001C00001C00001E00000E0 0000E000007000007000003800003800001C00000C00000E000006000007000003800001 800000C00000E0000060000030000018157C768121>32 DI<00001C00003C0000F80001E00003C0000780000F00000E00001E00003C 00003C00003C000078000078000078000078000078000078000078000078000078000078 000078000078000078000078000078000078000078000078000078000078000078000078 000078000078000078000078000078000078000078000078000078000078000078000078 0000780000780000780000780000F00000F00000F00001E00001E00003C0000380000700 000E00001C0000780000E00000E000007800001C00000E000007000003800003C00001E0 0001E00000F00000F00000F0000078000078000078000078000078000078000078000078 000078000078000078000078000078000078000078000078000078000078000078000078 000078000078000078000078000078000078000078000078000078000078000078000078 00007800007800007800007800007800007800003C00003C00003C00001E00000E00000F 000007800003C00001E00000F800003C00001C167C7B8121>40 D<0018007800F001E003 C007800F001F001E003E003C007C007C007800F800F800F800F800F800F800F800F800F8 00F800F800F800F800F800F800F800F800F800F800F800F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C007C007C007C007C007C00F800F800F800F001F001E003 E003C0078007000E001C003800F000C000F00038001C000E000700078003C003E001E001 F000F000F800F800F8007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C007C007C007C007C0E4D798025>60 D62 D80 D89 D E /Ff 7 116 df<60F0F070101020204040040A7D830A>59 D<003E000C000C000C000C001800180018 0018073018F0307060706060C060C060C06080C080C480C4C1C446C838700F177E9612> 100 D<0300038003000000000000000000000000001C002400460046008C000C00180018 00180031003100320032001C0009177F960C>105 D<0018003800100000000000000000 0000000001C0022004300430086000600060006000C000C000C000C00180018001800180 6300E300C60078000D1D80960E>I<383C1E0044C6630047028100460301008E0703000C 0603000C0603000C060600180C0600180C0620180C0C20180C0C4030180440301807801B 0E7F8D1F>109 D<383C0044C6004702004602008E06000C06000C06000C0C00180C0018 0C40181840181880300880300F00120E7F8D15>I<07C00C201870187038001E000FC003 E000606060E060C0C0C1803F000C0E7E8D10>115 D E /Fg 3 55 df0 D<020002000200C218F2783AE00F800F803AE0F278C2 180200020002000D0E7E8E12>3 D<0003000300060006000C000C001800180030003000 60006000C000C00180018003000300060006000C000C00180018003000300060006000C0 004000101E7B9600>54 D E /Fh 25 122 df<60F0F07010101020204080040B7D830B> 44 D<03000700FF00070007000700070007000700070007000700070007000700070007 00070007000700070007000700FFF00C187D9713>49 D<00300030007000F000F0017003 70027004700C7008701070307020704070C070FFFF00700070007000700070007007FF10 187F9713>52 D<07801860303070306018E018E018E01CE01CE01C601C603C303C185C0F 9C001C00180018003870307060604021801F000E187E9713>57 D69 D<1FFC00E000E000E000E000E000E000E000E000E000E000E000E000E000E000 E000E000E000E000E040E0E0E0E0E041C061801E000E1A7D9914>74 D80 D<0FC21836200E6006C006C002C002C002E00070007E 003FE01FF807FC003E000E00070003800380038003C002C006E004D81887E0101A7E9915 >83 D86 D<3F8070C070E020700070007007F01C7030707070E070E071E071E0F171 FB1E3C10107E8F13>97 DI<007E00000E00000E00000E00 000E00000E00000E00000E00000E00000E0003CE000C3E00380E00300E00700E00E00E00 E00E00E00E00E00E00E00E00E00E00600E00700E00381E001C2E0007CFC0121A7F9915> 100 D<07C01C3030187018600CE00CFFFCE000E000E000E0006000300438080C1807E00E 107F8F11>I<01F0031807380E100E000E000E000E000E000E00FFC00E000E000E000E00 0E000E000E000E000E000E000E000E000E000E007FE00D1A80990C>I104 D<18003C003C001800000000000000000000000000FC001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C00FF80091A80990A>I108 DII<07E01C38300C700E6006E007E007E007E007E007E0076006700E381C1C3807E01010 7F8F13>II114 D<1F2060E04020C020C020F0007F003FC01FE000F08070 8030C030C020F0408F800C107F8F0F>I<0400040004000C000C001C003C00FFC01C001C 001C001C001C001C001C001C001C201C201C201C201C200E4003800B177F960F>I121 D E /Fi 6 94 df<07C018303018701C600C600CE00EE00EE00EE00EE00EE00EE0 0EE00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF000700070007 00070007000700070007000700070007000700070007000700070007007FF00C157E9412 >I<0F8030E040708030C038E0384038003800700070006000C00180030006000C080808 10183FF07FF0FFF00D157E9412>I61 D91 D93 D E /Fj 4 82 df<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFF7FFE7FFE3FFC1FF80F F003C010107E9115>15 D<000001800000078000001E00000078000001E0000007800000 1E00000078000001E00000078000001E00000078000000E0000000780000001E00000007 80000001E0000000780000001E0000000780000001E0000000780000001E000000078000 000180000000000000000000000000000000000000000000000000000000007FFFFF00FF FFFF8019227D9920>20 D<003FFF0001FFFFC0071E0FE0081C03F0381C01F8703C00F860 3C0078C03C0078003C00780038007000380070007800E0007800E0007801C00070018000 70020000F00C0000E0300000E7C00001EF000001E0000001C0000001C000000380000003 8000000780000007000000070000000E0000000C0000001D1E809B1D>80 D<0003FC00000FFF0000303F8000400F80018007C0030007C0060003E00E0003E01C0001 E01C0001E0380001E0380001E0700001E0700001E0F00001C0F00001C0F00003C0F00003 80F0000380F8000700F80006007C030E007F0C1C003FF818000FC0200000004000000180 00000E00001FF800203FF000E007FE01C000FFC180001FFE000003F8001B227C9C22>I E /Fk 13 118 df<3078F06005047C830D>46 D<03CC063C0C3C181C3838303870387038 E070E070E070E070E0E2C0E2C0E261E462643C380F127B9115>97 D<001F80000380000380000700000700000700000700000E00000E00000E00000E0003DC 00063C000C3C00181C00383800303800703800703800E07000E07000E07000E07000E0E2 00C0E200C0E20061E4006264003C3800111D7B9C15>100 D<01E007100C101808381070 1070607F80E000E000E000E000E000E0086010602030C01F000D127B9113>I<00F3018F 030F06070E0E0C0E1C0E1C0E381C381C381C381C383830383038187818F00F7000700070 00E000E0C0C0E1C0C3007E00101A7D9113>103 D<018003800100000000000000000000 00000000001C002600470047008E008E000E001C001C001C003800380071007100710072 0072003C00091C7C9B0D>105 D<3C1E0780266318C04683A0E04703C0E08E0380E08E03 80E00E0380E00E0380E01C0701C01C0701C01C0701C01C070380380E0388380E0388380E 0708380E0710701C0320300C01C01D127C9122>109 D<3C3C002646004687004707008E 07008E07000E07000E07001C0E001C0E001C0E001C1C00381C40381C4038384038388070 1900300E0012127C9117>I<03C4062C0C3C181C3838303870387038E070E070E070E070 E0E0C0E0C0E061E063C03DC001C001C0038003800380038007803FF00E1A7B9113>113 D<3C3C26C2468747078E068E000E000E001C001C001C001C003800380038003800700030 0010127C9112>I<01F006080C080C1C18181C001F001FC00FF007F0007800386030E030 C030806060C01F000E127D9111>I<00C001C001C001C00380038003800380FFE0070007 0007000E000E000E000E001C001C001C001C00384038403840388019000E000B1A7D990E >I<1E0300270700470700470700870E00870E000E0E000E0E001C1C001C1C001C1C001C 1C003838803838801838801839001C5900078E0011127C9116>I E /Fl 33 122 df<003FC00001F0300003C0380007C07C000F807C000F807C000F803800 0F8000000F8000000F8000000F800000FFFFFC00FFFFFC000F807C000F807C000F807C00 0F807C000F807C000F807C000F807C000F807C000F807C000F807C000F807C000F807C00 0F807C000F807C007FE1FF807FE1FF80191D809C1B>12 D<002000400180038003000600 0E001C001C003C0038003800780078007800F800F000F000F000F000F000F000F000F000 F000F800780078007800380038003C001C001C000E000600030003800180004000200B29 7C9E13>40 D<800040003000380018000C000E000700070007800380038003C003C003C0 03E001E001E001E001E001E001E001E001E001E003E003C003C003C00380038007800700 07000E000C00180038003000400080000B297D9E13>I<78FCFCFCFC7806067D850D>46 D<78FCFCFCFC7800000000000078FCFCFCFC7806127D910D>58 D61 D<001FE02000FFF8E003F80FE007C003E00F8001E01F 0000E03E0000E03E0000607E0000607C000060FC000000FC000000FC000000FC000000FC 000000FC000000FC000000FC0000007C0000607E0000603E0000603E0000C01F0000C00F 80018007C0030003F80E0000FFFC00001FE0001B1C7D9B22>67 DI73 D78 D80 D<003FE00001F07C0003C01E 000F800F801F0007C01F0007C03E0003E07E0003F07C0001F07C0001F0FC0001F8FC0001 F8FC0001F8FC0001F8FC0001F8FC0001F8FC0001F8FC0001F87C0001F07C0001F07E0003 F03E0003E03E0703E01F08C7C00F906F8003D03E0001F87C00003FF8080000180800001C 1800001FF800001FF800000FF000000FF0000007E0000003C01D247D9B24>I<0FF8001C 1E003E0F803E07803E07C01C07C00007C0007FC007E7C01F07C03C07C07C07C0F807C0F8 07C0F807C0780BC03E13F80FE1F815127F9117>97 D<03FC000E0E001C1F003C1F00781F 00780E00F80000F80000F80000F80000F80000F800007800007801803C01801C03000E0E 0003F80011127E9115>99 D<000FF0000FF00001F00001F00001F00001F00001F00001F0 0001F00001F00001F001F9F00F07F01C03F03C01F07801F07801F0F801F0F801F0F801F0 F801F0F801F0F801F07801F07801F03C01F01C03F00F0FFE03F9FE171D7E9C1B>I<01FC 000F07001C03803C01C07801C07801E0F801E0F801E0FFFFE0F80000F80000F800007800 007C00603C00601E00C00F038001FC0013127F9116>I<007F0001E38003C7C00787C00F 87C00F83800F80000F80000F80000F80000F8000FFF800FFF8000F80000F80000F80000F 80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80007FF8007F F800121D809C0F>I<03F8F00E0F381E0F381C07303C07803C07803C07803C07801C0700 1E0F000E0E001BF8001000001800001800001FFF001FFFC00FFFE01FFFF07801F8F00078 F00078F000787000707800F01E03C007FF00151B7F9118>II<1E003F003F003F003F001E00000000000000000000000000FF 00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B 1E7F9D0E>I108 DII<01FC000F07801C01C03C01E07800F07800F0F800F8 F800F8F800F8F800F8F800F8F800F87800F07800F03C01E01E03C00F078001FC0015127F 9118>II<03F0600F0CE01E07E03C03E07C03E07803E0F8 03E0F803E0F803E0F803E0F803E0F803E07803E07C03E03C03E01C07E00E0FE003F3E000 03E00003E00003E00003E00003E00003E0001FFC001FFC161A7E9119>II<1FD830786018E018E018F000FF807FE07F F01FF807FC007CC01CC01CE01CE018F830CFC00E127E9113>I<03000300030003000700 07000F000F003FFCFFFC1F001F001F001F001F001F001F001F001F001F0C1F0C1F0C1F0C 0F08079803F00E1A7F9913>III120 DI E /Fm 6 79 df<003800007C00007C0000FE0000DE0000DE00018F00018F000307800307 800603C007FFC00C01E00C01E01800F01800F0FE03FE17117F901A>65 D<01FE100703F01C00F0380070780030700030F00000F00000F00000F00000F000007000 307800303800601C006007018001FE0014117E9019>67 D69 D<01FE100703F01C00F0380070780030700030F00000F00000F0 0000F00FFCF000F07000F07800F03800F01C00F00701F001FF3016117E901B>71 DI78 D E /Fn 66 124 df<003F83F800FF8FF801FF9FF803E03E000380 38000780780007807800078078000780780007807800078078000780780007807800FFFC 7FC0FFFC7FC0FFFC7FC00780780007807800078078000780780007807800078078000780 780007807800078078000780780007807800078078000780780007807800078078000780 78000780780007807800078078001D2380A21C>11 D<000078007C7800FC7801FC7803C0 00038000078000078000078000078000078000078000078000078000FFFC78FFFC78FFFC 780780780780780780780780780780780780780780780780780780780780780780780780 78078078078078078078078078078078078078078078152480A31A>I<0000C018000000 C018000000C0180000018030000001803000000180300000018030000003006000000300 6000000300600000030060000003006000000600C000000600C000000600C000000600C0 00000C018000FFFFFFFFC0FFFFFFFFC00018030000001803000000180300000018030000 0030060000003006000000300600000030060000FFFFFFFFC0FFFFFFFFC000600C000000 C018000000C018000000C018000000C01800000180300000018030000001803000000180 30000003006000000300600000030060000003006000000600C000000600C000000600C0 0000222D7DA229>35 D<007000E001C003C0078007000F000E001E001C003C003C003800 78007800780078007000F000F000F000F000F000F000F000F000F000F000F000F000F000 F0007000780078007800780038003C003C001C001E000E000F000700078003C001C000E0 00700C327DA413>40 DI<787878781830306060E0050A7C830E>44 DII<00008000018000018000030000030000030000060000 0600000600000C00000C00000C0000180000180000180000300000300000300000600000 600000600000C00000C00000C00001800001800001800001800003000003000003000006 00000600000600000C00000C00000C000018000018000018000030000030000030000060 0000600000600000C00000C00000C0000011317DA418>I<00C001C007C0FFC0FFC0FBC0 03C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C0 03C003C003C003C003C003C003C0FFFFFFFFFFFF10227CA118>49 D<03F0000FFC001FFE003C1F003007807007C06003C0E003E0C001E04001E04001E00001 E00001E00001E00003C00003C0000780000780000F00001E00003C0000780000F00001E0 0001C0000380000700000E00001C0000380000700000FFFFE0FFFFE0FFFFE013227EA118 >I<01F00007FC001FFF003E0F003807807003C02003C02003C00003C00003C00003C000 0780000780000F00001E0003FC0003F80003FE00000F000007800003C00003C00001E000 01E00001E00001E00001E08001E0C003C0E003C07007803C0F801FFF000FFC0003F00013 237EA118>I<001F00001F00002F00002F00006F0000EF0000CF0001CF0001CF00038F00 038F00078F00070F000F0F000E0F001E0F003C0F003C0F00780F00780F00F00F00FFFFF8 FFFFF8FFFFF8000F00000F00000F00000F00000F00000F00000F00000F00000F0015217F A018>I<3FFF803FFF803FFF803C00003C00003C00003C00003C00003C00003C00003C00 003C00003CF8003FFE003FFF003F0F803E07803C03C03803C00001E00001E00001E00001 E00001E00001E00001E04003C04003C0E003C07007807C1F003FFE000FFC0003F0001322 7EA018>I<007E0001FF0003FF0007C1000F00001E00001E00003C00003C000078000078 000078F800F1FE00F7FF00FF0F80FC0780F803C0F803C0F801C0F001E0F001E0F001E0F0 01E0F001E07001E07001E07801E07803C03803C03C03801C07801F0F000FFE0007FC0001 F00013237EA118>II<01F00007FC000FFE001E0F003C07803C07807803C07803C07803C07803 C07803C03803803C07801E0F000F1E0007FC0003F8000FFE001E0F003C07807803C07803 C0F001E0F001E0F001E0F001E0F001E0F001E07803C07803C03C07803E0F801FFF0007FC 0001F00013237EA118>I<01F00007FC000FFE001E0F003C07003807807803807803C0F0 03C0F001C0F001E0F001E0F001E0F001E0F001E0F001E07003E07803E07803E03C07E03E 1FE01FFDE00FF1E003E3C00003C00003C0000380000780000700000F00101E00183C003F F8001FF00007C00013237EA118>I<001F0000001F0000003F8000003F8000003B800000 7BC0000073C0000071C00000F1E00000F1E00000E0E00001E0F00001E0F00001C0F00003 C0780003C078000380780007803C0007803C0007003C000F001E000F001E000FFFFE001F FFFF001FFFFF001C000F003C0007803C00078038000780780003C0780003C0700003C0F0 0001E0F00001E0E00001E01B237EA220>65 DI<000FF000007FFC0000FFFF0001 F01F0003C00700078000000F0000001E0000003E0000003C0000003C0000007800000078 00000078000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0 000000F00000007800000078000000780000003C0000003C0000003E0000001E0000000F 0000000780008003C0038001F00F8000FFFF00007FFC00000FF00019257DA31F>I III<000FF000003FFE0000FFFF8001F80F8003E00380078000000F0000 001E0000001E0000003C0000003C000000780000007800000078000000F0000000F00000 00F0000000F0000000F0000000F0000000F000FFC0F000FFC0F000FFC0780003C0780003 C0780003C03C0003C03C0003C01E0003C01E0003C00F0003C0078003C003E003C001F807 C000FFFFC0003FFF00000FF8001A257DA321>III76 DII<001FC000007FF00001FFFC0003F07E0007C01F000F800F801F0007C0 1E0003C03C0001E03C0001E0780000F0780000F0780000F070000070F0000078F0000078 F0000078F0000078F0000078F0000078F0000078F0000078F0000078780000F0780000F0 780000F07C0001F03C0001E03E0003E01E0003C01F0007C00F800F8007C01F0003F07E00 01FFFC00007FF000001FC0001D257DA324>II82 D<00FE0003FFC007FFE00F81E01E00603C00003C00007800007800007800007800007800 007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007E00001F00000F00000 F8000078000078000078000078000078000078C000F0E000F0F801E07E07C03FFF800FFF 0001FC0015257EA31B>IIIII<780003E07C0003C03E0007 C01E000F800F000F000F801F0007C03E0003C03C0003E07C0001F0780000F0F0000079F0 00007DE000003FC000001FC000001F8000000F0000001F8000001FC000003FC000007DE0 000079F00000F0F00001F0780001E03C0003C03E0007C01E0007800F000F000F801F0007 801E0003C03C0003E07C0001F0780000F0F80000F81D237FA220>I<07E01FF83FFC381E 201E000F000F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF00FF83F7FFF3FEF1F8F 10167E9517>97 DI<01FC0007FF000FFF801F03803C0180780000780000700000 F00000F00000F00000F00000F00000F000007800007800007800003C00401F03C00FFFC0 07FF8001FC0012167E9516>I<0003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C003E3C00FFBC01FFFC03F0FC03C07C07803C07803 C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07803C07803C03C07C03E0F C01FFFC00FFBC003E3C012237EA219>I<03F00007FC001FFE003E0F003C078078038078 0380F001C0FFFFC0FFFFC0FFFFC0F00000F00000F000007000007800007800003C00801F 07800FFF8007FF0001F80012167E9516>I<003F00FF01FF03C003800780078007800780 0780078007800780FFF8FFF8FFF807800780078007800780078007800780078007800780 07800780078007800780078007800780102380A20F>I<01F07807FFF80FFFF81F1F001E 0F003C07803C07803C07803C07803C07801E0F001F1F000FFE001FFC0019F00038000038 00003C00001FFE001FFFC01FFFE03FFFF07801F07800F8F00078F00078F00078F0007878 00F03E03E01FFFC00FFF8001FC0015217F9518>III<00F000F0 00F000F000000000000000000000000000000000000000F000F000F000F000F000F000F0 00F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F0 00F000F080F0E3E0FFE07FC01F000C2D83A20D>IIIII<01FC00 07FF000FFF801F07C03C01E07800F07800F0700070F00078F00078F00078F00078F00078 F000787800F07800F07C01F03E03E01F07C00FFF8007FF0001FC0015167F9518>II<03E3C007FBC00F FFC01F0FC03C07C07C03C07803C0F803C0F003C0F003C0F003C0F003C0F003C0F003C0F8 03C07803C07803C03C07C03F0FC01FFFC00FFBC003E3C00003C00003C00003C00003C000 03C00003C00003C00003C00003C00003C012207E9519>II<07F01FFC3FFE3C0E7806780078007C003F003FF01FF80FFC01FE001F000F000F00 0FC00FF81EFFFE3FFC0FF010167F9513>I<0F000F000F000F000F000F00FFF8FFF8FFF8 0F000F000F000F000F000F000F000F000F000F000F000F000F000F000F080F1C07FC07F8 03E00E1C7F9B12>IIII<7801F07C01E03E03C01E07C00F0780078F0007DE0003FC00 01FC0000F80000700000F80001FC0003DC00039E00078F000F07801E07801E03C03C01E0 7800F0F800F81516809516>II<7FFFC07FFFC07FFFC0000F80000F00001E00003E00007C0000780000 F00001F00003E00003C0000780000F80001F00001E00003C00007C0000FFFFC0FFFFC0FF FFC012167F9515>II E /Fo 55 124 df<00003FE00000E0100001803800038078000300780007003000070000000700 0000070000000E0000000E0000000E000000FFFFE0000E00E0001C01C0001C01C0001C01 C0001C01C0001C0380003803800038038000380380003807000038070000700700007007 1000700E2000700E2000700E2000E00E2000E0064000E0038000E0000000C0000001C000 0001C000003180000079800000F3000000620000003C0000001D29829F1A>12 D<0E1F3F3F1D0102020404081020C0080E779F0E>39 D<00010002000400080010002000 6000C0018001800300070006000E000C001C0018003800380030007000700060006000E0 00E000C000C000C000C000C000C000C000C000C000C000C000C000C00040006000600020 00100010000800102E79A113>I<00100000080000040000060000020000030000030000 030000010000018000018000018000018000018000018000018000038000038000038000 0300000300000300000700000700000600000600000E00000C00000C00001C0000180000 380000300000700000600000E00000C0000180000100000300000600000C000018000030 0000600000800000112E80A113>I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I<70F8F8F0E005057B840E>I<000F800030E000E07001C0 700380300380380700380F00780F00780E00781E00781E00703C00F03C00F03C00F03C00 F07801E07801E07801E07801C07003C0F003C0F00380F00780F00700700700700E00701C 003038001870000FC000151F7C9D17>48 D<000200020006000E003C00DC031C001C0038 003800380038007000700070007000E000E000E000E001C001C001C001C0038003800380 03800780FFF80F1E7B9D17>I<001F000061800080E00100E00200700220700420700410 700820F00820F00820F00840E00881E00703C0000380000700000C000018000060000080 000300000400000800401000401000802001807E030047FF0041FE0080FC00807800141F 7C9D17>I<001F800060E000807001003002003804203804203804103804207004607003 80600000E00001C000030000FE00001C0000060000070000078000078000078030078078 0780780780F00F00800F00401E00401C0040380020E0001F8000151F7C9D17>I<001F00 00718000C0C00180C00380E00700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C0 1C03C01C03C01C07C01C0F800C0F8006378003C700000F00000E00000E00001C00601C00 F03800F07000E0600080C0004380003E0000131F7B9D17>57 D<070F1F1F0E0000000000 000000000070F8F8F0E008147B930E>I<00000200000006000000060000000E0000001E 0000001E0000003F0000002F0000004F0000004F0000008F0000010F0000010F0000020F 0000020F0000040F00000C0F0000080F0000100F0000100F0000200F80003FFF80004007 8000C007800080078001000780010007800200078002000780060007801E000F80FF807F F81D207E9F22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C00 3C003C003C003C003C0078007800780078007800F0007801E000F0078000FFFE0000F00F 8000F003C001E001C001E001E001E001E001E001E003C001E003C001E003C001E003C001 C0078003C00780078007800F0007801E000F007800FFFFE0001E1F7D9E20>I<0000FE02 00078186001C004C0038003C0060003C00C0001C01C0001803800018070000180F000018 1E0000101E0000103C0000003C00000078000000780000007800000078000000F0000000 F0000000F0000000F0000000F00000807000008070000080700001003800010038000200 180004000C001800060020000381C00000FE00001F217A9F21>I<01FFFF80001E00E000 1E0070001E0038001E001C003C001C003C000E003C000E003C000E0078000E0078000E00 78000E0078000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01 E0007803C0007003C0007003C000E003C001C0078001C00780038007800E0007801C000F 007000FFFFC0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C 0004003C0004003C0004003C00040078080800780800007808000078180000F0300000FF F00000F0300000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C0 004003C00040078000800780018007800100078007000F001F00FFFFFE001F1F7D9E1F> I<01FFFFFC001E0038001E0018001E0008001E0008003C0008003C0008003C0008003C00 080078001000780800007808000078080000F0100000F0300000FFF00000F0300001E020 0001E0200001E0200001E0200003C0000003C0000003C0000003C0000007800000078000 0007800000078000000F800000FFF800001E1F7D9E1E>I<0000FC040007030C001C0098 0030007800E0007801C000380380003003800030070000300E0000301E0000201E000020 3C0000003C00000078000000780000007800000078000000F0000000F000FFF0F0000780 F0000780F0000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E00 0E00CC000383040000FC00001E217A9F23>I<01FFF3FFE0001F003E00001E003C00001E 003C00001E003C00003C007800003C007800003C007800003C007800007800F000007800 F000007800F000007800F00000F001E00000FFFFE00000F001E00000F001E00001E003C0 0001E003C00001E003C00001E003C00003C007800003C007800003C007800003C0078000 07800F000007800F000007800F000007800F00000F801F0000FFF1FFE000231F7D9E22> I<01FFF0001F00001E00001E00001E00003C00003C00003C00003C000078000078000078 0000780000F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C0 0003C0000780000780000780000780000F8000FFF800141F7D9E12>I<01FE00007FC000 1E0000FC00001E0000F80000170001780000170001780000270002F00000270004F00000 270004F00000270008F00000470009E00000470011E00000470021E00000470021E00000 870043C00000838043C00000838083C00000838083C00001038107800001038207800001 03820780000103840780000203840F00000203880F00000203900F00000203900F000004 01E01E00000401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0002A 1F7D9E29>77 D<01FFFF80001E00E0001E0070001E0038001E003C003C003C003C003C00 3C003C003C003C0078007800780078007800F0007800E000F003C000F00F0000FFFC0000 F0000001E0000001E0000001E0000001E0000003C0000003C0000003C0000003C0000007 8000000780000007800000078000000F800000FFF000001E1F7D9E1F>80 D<01FFFF00001E03C0001E00E0001E0070001E0078003C0078003C0078003C0078003C00 78007800F0007800F0007801E0007801C000F0070000F01E0000FFF00000F0380001E01C 0001E01E0001E00E0001E00F0003C01E0003C01E0003C01E0003C01E0007803C0007803C 0807803C0807803C100F801C10FFF00C20000007C01D207D9E21>82 D<0007E040001C18C0003005800060038000C0038001C001800180010003800100038001 00038001000380000003C0000003C0000003F8000001FF800001FFE000007FF000001FF0 000001F80000007800000078000000380000003800200038002000380020003000600070 00600060006000E0007000C000E8038000C606000081F800001A217D9F1A>I<0FFFFFF0 1E0780E0180780201007802020078020200F0020600F0020400F0020400F0020801E0040 001E0000001E0000001E0000003C0000003C0000003C0000003C00000078000000780000 007800000078000000F0000000F0000000F0000000F0000001E0000001E0000001E00000 01E0000003E00000FFFF00001C1F789E21>I<7FFC1FF807C003C0078001000780010007 8001000F0002000F0002000F0002000F0002001E0004001E0004001E0004001E0004003C 0008003C0008003C0008003C00080078001000780010007800100078001000F0002000F0 002000F0002000F0004000F0004000700080007001000030020000380400000C18000007 E000001D20779E22>I I<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E20F01C40F01C40703C40705C40308C800F070013147C9317>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207B9F15>I< 007E0001C1000300800E07801E07801C07003C0200780000780000780000F00000F00000 F00000F00000F0000070010070020030040018380007C00011147C9315>I<0000780003 F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1C00389 C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E 20F01C40F01C40703C40705C40308C800F070015207C9F17>I<007C01C207010E011C01 3C013802780C7BF07C00F000F000F000F0007000700170023804183807C010147C9315> I<00007800019C00033C00033C000718000700000700000E00000E00000E00000E00000E 0001FFE0001C00001C00001C00001C000038000038000038000038000038000070000070 0000700000700000700000700000E00000E00000E00000E00000C00001C00001C0000180 003180007B0000F300006600003C00001629829F0E>I<003C6000E27001C1E00380E007 00E00F00E00E01C01E01C01E01C01E01C03C03803C03803C03803C03803C07003C07001C 0F001C17000C2E0003CE00000E00000E00001C00001C00301C00783800F0700060E0003F 8000141D7E9315>I<01E0000FE00001C00001C00001C00001C000038000038000038000 038000070000070000071E000763000E81800F01C00E01C00E01C01C03801C03801C0380 1C0380380700380700380700380E10700E20700C20701C20700C40E00CC060070014207D 9F17>I<00C001E001E001C000000000000000000000000000000E003300230043804300 470087000E000E000E001C001C001C003840388030807080310033001C000B1F7C9E0E> I<0001800003C00003C0000380000000000000000000000000000000000000000000003C 00004600008700008700010700010700020E00000E00000E00000E00001C00001C00001C 00001C0000380000380000380000380000700000700000700000700000E00000E00030E0 0079C000F180006300003C00001228829E0E>I<01E0000FE00001C00001C00001C00001 C0000380000380000380000380000700000700000703C00704200E08E00E11E00E21E00E 40C01C80001D00001E00001FC00038E00038700038700038384070708070708070708070 3100E03100601E0013207D9F15>I<03C01FC0038003800380038007000700070007000E 000E000E000E001C001C001C001C0038003800380038007000700070007100E200E200E2 00E200640038000A207C9F0C>I<1C0F80F0002630C318004740640C004780680E004700 700E004700700E008E00E01C000E00E01C000E00E01C000E00E01C001C01C038001C01C0 38001C01C038001C01C0708038038071003803806100380380E100380380620070070066 00300300380021147C9325>I<1C0F802630C04740604780604700704700708E00E00E00 E00E00E00E00E01C01C01C01C01C01C01C03843803883803083807083803107003303001 C016147C931A>I<007C0001C3000301800E01C01E01C01C01E03C01E07801E07801E078 01E0F003C0F003C0F003C0F00780F00700700F00700E0030180018700007C00013147C93 17>I<01C1E002621804741C04781C04701E04701E08E01E00E01E00E01E00E01E01C03C 01C03C01C03C01C0380380780380700380E003C1C0072380071E000700000700000E0000 0E00000E00000E00001C00001C0000FFC000171D809317>I<00F0400388C00705800E03 801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E00F01C00F01C 00703C00705C0030B8000F380000380000380000700000700000700000700000E00000E0 000FFE00121D7C9315>I<1C1E002661004783804787804707804703008E00000E00000E 00000E00001C00001C00001C00001C000038000038000038000038000070000030000011 147C9313>I<00FC030206010C030C070C060C000F800FF007F803FC003E000E700EF00C F00CE008401020601F8010147D9313>I<018001C0038003800380038007000700FFF007 000E000E000E000E001C001C001C001C003800380038003820704070407080708031001E 000C1C7C9B0F>I<0E00C03300E02301C04381C04301C04701C08703800E03800E03800E 03801C07001C07001C07001C07101C0E20180E20180E201C1E200C264007C38014147C93 18>I<0E03803307802307C04383C04301C04700C08700800E00800E00800E00801C0100 1C01001C01001C02001C02001C04001C04001C08000E300003C00012147C9315>I<0E00 C1C03300E3C02301C3E04381C1E04301C0E04701C060870380400E0380400E0380400E03 80401C0700801C0700801C0700801C0701001C0701001C0602001C0F02000C0F04000E13 080003E1F0001B147C931E>I<0383800CC4401068E01071E02071E02070C040E00000E0 0000E00000E00001C00001C00001C00001C040638080F38080F38100E5810084C6007878 0013147D9315>I<0E00C03300E02301C04381C04301C04701C08703800E03800E03800E 03801C07001C07001C07001C07001C0E00180E00180E001C1E000C3C0007DC00001C0000 1C00003800F03800F07000E06000C0C0004380003E0000131D7C9316>I123 D E /Fp 50 121 df<00030004000800100020006000C00180010003000600 06000C000C001C0018001800380030003000700070006000600060006000E000E000E000 E000E000E00060006000600060006000300030001000180008000C00040002000100102E 7CA112>40 D<008000C000600020003000180018000C000C000C000E0006000600060006 0006000600060006000600060006000E000E000E000C000C001C00180018003800300030 0060006000C000C001800100030006000C0018003000600080000F2E7FA112>I<1C3E7E 7E3A0202040408081020C0070E7D840D>44 DI<3078F8787005 057C840D>I<007E0001C3000301800701C00E00C00E00E01C00E01C00E03C01E03801E0 7801E07801E07801E07801E07801E0F003C0F003C0F003C0F003C0F003C0F00380F00780 E00780E00700E00700E00E00600E00701C003038003870000FC000131F7C9D17>48 D<000C001C00FC0F380038003800380038003800700070007000700070007000E000E000 E000E000E000E001C001C001C001C001C001C0038003C0FFFE0F1E7C9D17>I<003F8000 C1E00100F00200780400780400780F007C0F807C0F807C0F00780600780000F80000F000 01E00001C0000380000700000E00001C0000380000600000C00001800003002006002008 00401000403FFFC07FFF80FFFF80161E7E9D17>I<007F000183C00201E00400F00700F0 0F00F00F01F00F01F00001E00001E00003C0000380000700000E0000F800000E00000700 0007800007C00003C00007C03007C07807C0F807C0F807C0F00780800F00400E00201C00 18780007E000141F7D9D17>I<0000600000600000E00001C00003C00005C0000DC00009 C00011C000238000438000C3800083800103800203800407000807001807001007002007 00400700FFFFF0000E00000E00000E00000E00000E00001C00001E0001FFE0141E7E9D17 >I<01803001FFE003FFC003FF0003FC0002000002000002000004000004000004000004 7C000587000603800C01800801C00001C00001E00001E00001E00001E07003C0F803C0F0 03C0E00380800780400700400E00201C0018700007C000141F7D9D17>I<000F80007040 00C0200180E00301E00701E00E00C01E00001C00003C000038000078F800790E007A0700 7C0300F80380F80380F003C0F003C0F003C0F003C0F00780E00780E00780E00700E00F00 600E00701C0030180018700007C000131F7C9D17>I<2000003FFFE07FFFC07FFF804001 00C002008002008004000008000010000020000040000040000080000180000300000300 000700000600000E00000E00001E00001C00001C00003C00003C00003C00007800007800 00780000300000131F799D17>I<003F0000C1C00100600200600400300C00300C00300C 00300C00600E00600F80C00FC18007F60003FC0001FC0001FF00063F800C0F801007C030 03C06001C06000C0C000C0C000C0C000C0C00080C0010060030030040018180007E00014 1F7D9D17>I<007E0001C3000301800601C00E01C01C00C03C00E03C00E03C01E07801E0 7801E07801E07801E07803E07803E03803C03807C01C0BC00C13C003E380000780000780 000700000E00600E00F01C00F01800E0300080600041C0003F0000131F7C9D17>I<07E0 1838201C401C701CF03CF03C603C0038007000E001C00180030002000600040004000800 0800080008000000000000000000000030007800F800780070000E20799F15>63 D<0000100000001800000038000000380000007800000078000000FC000001BC0000013C 0000033C0000023C0000063C0000043E0000081E0000081E0000101E0000101E0000201E 0000200F0000400F0000400F0000FFFF0000800F0001000F800100078002000780020007 8004000780040007800C0007C03E0007C0FF807FFC1E207E9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800F8007800 F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007C001E003 C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003C003C007 C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E061800380138 007000F801E0007803C0007007800030078000300F0000301F0000301E0000303E000020 3C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000F8000000 F80000007800004078000080780000803C0000803C0001001C0002000E00020006000C00 0300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F0003C007800 78003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00F0001F00 F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01E0007C03 C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FFFFE00020 1F7E9E23>I<07FFFFF8007C0078003C0038003C00180078001800780008007800080078 0008007800080078080800F0100000F0100000F0100000F0300000FFF00000F0700001E0 200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C0002003C0 002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFFF8007C00 78003C0038003C001800780018007800080078000800780008007800080078000800F010 0000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E0200001E020 0001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C00000078000 0007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C00E0003C 01C0003803800018078000180F0000181F0000181E0000183E0000103C0000007C000000 7C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E0780001E0 780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B800380118001E06080 003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C007800007800F000 007800F000007800F000007800F000007800F000007800F00000F001E00000F001E00000 F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003C00001E0 03C00001E003C00001E003C00003C007800003C007800003C007800003C007800003C007 800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C0000 3C00003C0000780000780000780000780000780000780000F00000F00000F00000F00000 F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C00003C00003 C00003C00007800007C000FFFC00131F7F9E10>I<07FFE07FE0007C001F00003C000C00 003C00180000780010000078004000007800800000780100000078020000007804000000 F008000000F010000000F060000000F0F0000000F1F0000000F278000001E478000001E8 78000001F03C000001E03C000001E01E000001E01E000003C00F000003C00F000003C00F 000003C007800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F7E9E 23>75 D<07FFF000007E0000003C0000003C000000780000007800000078000000780000 007800000078000000F0000000F0000000F0000000F0000000F0000000F0000001E00000 01E0000001E0000001E0000001E0008001E0010003C0010003C0010003C0030003C00200 03C0060003C0060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC0007C00 00F800003C00017800003C00017800004E0002F000004E0002F000004E0004F000004E00 04F000004E0008F000004E0008F00000870011E00000870011E00000870021E000008700 21E00000870041E00000838041E00001038083C00001038083C00001038103C000010382 03C0000101C203C0000101C403C0000201C40780000201C80780000201C80780000201D0 0780000200F00780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A1F7E 9E2A>I<07FC01FFC0003E003E00003E001800003E001800004F001000004F0010000047 80100000478010000043C010000043C010000083C020000081E020000081E020000080F0 20000080F020000080782000010078400001007C400001003C400001003C400001001E40 0001001E400002000F800002000F800002000F8000020007800002000780000600038000 06000300000F00010000FFE0010000221F7E9E22>I<0003F800001E0E000038070000E0 038001C001C003C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F83C00 00F87C0000F87C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0F800 03E0780003E0780003C0780007C07C0007803C000F003C001E001E001C000E0038000700 F00003C3C00000FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F0007800 F0007800F8007800F8007800F8007800F8007800F000F001F000F001E000F003C000F007 8000F00F0000FFF80001E0000001E0000001E0000001E0000001E0000001E0000003C000 0003C0000003C0000003C0000003C0000003C000000780000007C00000FFFC00001D1F7E 9E1F>I<0003F800001E0E000038070000F0038001E001C003C001E0078001E00F0000F0 0F0000F01F0000F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C0000F8 F80001F0F80001F0F80001F0F80001F0F80003E0780003E0780003C0780007C0781E0780 3C210F003C409E001E409C000E80B8000740F00003C1C04000FEC0400000C0400000C080 0000E1800000FF800000FF000000FF0000007E0000003C001D297B9F23>I<07FFFC0000 7C0700003C03C0003C01E0007801E0007801F0007801F0007801F0007801F0007801E000 F003E000F003C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E01C0001 E01C0001E01C0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C03E0207 803E0407C01F04FFFC0F18000003E01F207E9E21>I<003F040060CC01803C03801C0300 1C0700180600080E00080E00080E00080E00000F00000F80000FE00007FE0003FF8001FF C0007FE00007E00001E00000E00000F00000F04000E04000E04000E04000E06000C06001 80E00380F80300C60C0081F80016217D9F19>I<3FFFFFF03C0780F03007803060078030 400F0010400F0010C00F0010800F0010800F0010800F0010001E0000001E0000001E0000 001E0000001E0000001E0000003C0000003C0000003C0000003C0000003C0000003C0000 00780000007800000078000000780000007800000078000000F0000001F800007FFFE000 1C1F7A9E21>IIII<03FFC0FFC0007F007E00003E003800001E003000001E002000000F00 4000000F008000000F81000000078200000007C600000003C400000003E800000001F000 000001F000000000F000000000F800000000F8000000017C000000023C000000043C0000 000C1E000000081E000000101F000000200F000000400F800000C0078000008007C00001 0003C000070003E0001F8007E000FFE01FFE00221F7F9E22>II<03FFFFC003E007800380078007000F0006001E0004003C 000C003C00080078000800F0000801F0000001E0000003C0000007800000078000000F00 00001E0000003C0000003C00000078000000F0010001E0010001E0020003C00200078002 000F8006000F0004001E000C003C001C007C003C007800F800FFFFF8001A1F7D9E1C>I< 01FC07060E0F1C0F380E78007000F000F000F000F000E000E000E000E000F00270043008 18300FC010147C9314>99 D<0000700003F00000F00000700000700000E00000E00000E0 0000E00000E00000E00001C000F9C00305C00E03C01C03C03801C0780380700380F00380 F00380F00380F00380E00700E00700E00700E00700E00700700F00301E00186F000F8FE0 14207C9F19>I<00F800070E000E07001C0700380380780380700380F00380F00380FFFF 80F00000E00000E00000E00000E00000F001007002003004001C180007E00011147D9314 >I<01C003E003E003E001C000000000000000000000000003801F800780038003800700 070007000700070007000E000E000E000E000E000E001C001E00FF800B1F7F9E0C>105 D<038F801F90E007A0E003C0600380600780E00700E00700E00700E00700E00700E00E01 C00E01C00E01C00E01C00E01C00E01C01C03801E03C0FFCFF815147E9319>110 D<00FC000387000E01801C00C03800E03800E07000F0F000F0F000F0F000F0F000F0E001 E0E001E0E001C0E003C0F00380700700380E001C1C0007E00014147D9317>I<00800100 01000100030007000F001E00FFF80E000E000E000E001C001C001C001C001C001C003800 38203820382038203840384018800F000D1C7C9B12>116 D<1C0380FC1F803C07801C03 801C0380380700380700380700380700380700380700700E00700E00700E00700E00701E 00701E00703C00305E001F9FC012147B9319>I<1FF0FF03C07801C06001C04000E08000 E180007300007600003C00003C00001C00002E00004E000087000107000203800603800C 01C03E03E0FF07FC18147F9318>120 D E /Fq 62 123 df<00001F80003F9F80007F9F 8000FF9F8001FF9F8003E09F8003C0000007C0000007C0000007C0000007C0000007C000 0007C0000007C0000007C00000FFFF8F80FFFF8F80FFFF8F80FFFF8F8007C00F8007C00F 8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F 8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F 8007C00F8007C00F8007C00F80192A7FA920>12 D<001E003C007800F001F001E003C007 C007800F800F001F001F001E003E003E003E007C007C007C007C007C007800F800F800F8 00F800F800F800F800F800F800F800F800F800F800F800F80078007C007C007C007C007C 003E003E003E001E001F001F000F000F80078007C003C001E001F000F00078003C001E0F 3D7CAC17>40 DI<7E7E7E7E7E7E1E3C 3C387870F0070D7C8511>44 DII<007F000001FFC00007FFF0000FFFF8000FC1F8001F007C003F007E003E003E 003C001E007C001F007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F8078000F007C001F007C001F007C001F003E003E003E003E003F007E 001F80FC000FC1F8000FFFF80007FFF00001FFC000007F000019297EA71E>48 D<00180000380000F80007F800FFF800FFF800FFF800F8F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 007FFFF07FFFF07FFFF07FFFF014287CA71E>I<00FE0003FFC00FFFE01FFFF03F03F83C 00FC78007E78007EF0003EF0003F70003F60001F20001F00001F00003F00003F00003E00 003E00007C00007C0000F80001F00003E00007E00007C0000F80001E00003C0000780000 F00001E00003C0000780000F00001E00003C00007FFFFF7FFFFF7FFFFF7FFFFF18287EA7 1E>I<007F000003FFC00007FFF0000FFFF8001F81F8003E00FC007C007E0038007E0030 003E0010003E0000007E0000007E0000007C0000007C000000FC000001F8000007F00000 FFE00000FFC00000FFE00000FFF0000001F80000007C0000007E0000003F0000003F0000 001F8000001F8000001F8000001F8000001F8040001F8060003F00F0003F00F8007F007C 007E003F81FC001FFFF8000FFFF00003FFC000007F000019297EA71E>I<0003F0000007 F0000007F000000DF000000DF000001DF000003DF0000039F0000079F0000079F00000F1 F00000F1F00001F1F00003E1F00003E1F00007C1F00007C1F0000F81F0000F81F0001F01 F0001F01F0003E01F0007C01F0007C01F000F801F000FFFFFF80FFFFFF80FFFFFF80FFFF FF800001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001 F0000001F00019277EA61E>I<3FFFFC3FFFFC3FFFFC3FFFFC3E00003E00003E00003E00 003E00003E00003E00003E00003E00003E00003E3F003EFFC03FFFE03FFFF03FC1F83F80 FC3F007C3E007E00003E00003F00003F00003F00003F00003F00003F00003F20003F6000 7E70007EF800FC7C01FC3F03F83FFFF00FFFE007FF8000FE0018287EA61E>I<000FF000 003FFC0000FFFC0001FFFC0003F80C0007E000000FC000000F8000001F0000001E000000 3E0000003C0000007C0000007C0000007C3FE000F8FFF000F9FFF800FBFFFC00FF807E00 FF003E00FE003F00FC001F00FC001F00FC000F80F8000F80F8000F80F8000F80F8000F80 78000F807C000F807C000F807C000F003E001F003E001F001F003E001F807C000FC1FC00 07FFF80003FFF00001FFC000007F000019297EA71E>II<007F000001FFC00007FFF0000FFFF8001FC1FC003F007E003E003E007E00 3F007C001F007C001F007C001F007C001F007C001F003E003E003E003E001F007C000FC1 F80007FFF00003FFE00003FFE0000FFFF8001FC1FC003F007E003E003E007C001F007C00 1F00F8000F80F8000F80F8000F80F8000F80F8000F80F8000F807C001F007C001F007E00 3F003F007E001FC1FC000FFFF80007FFF00003FFE000007F000019297EA71E>I<007F00 0001FFC00003FFE0000FFFF0000FC1F8001F007C003E007C007C003E007C001E007C001F 00F8001F00F8001F00F8000F00F8000F80F8000F80F8000F80F8000F80F8001F807C001F 807C001F807E003F803E007F803F00FF801FFFEF800FFFCF8007FF8F8003FE1F0000001F 0000001F0000001E0000003E0000003E0000007C0000007C000000F8001801F0001E07E0 003FFFC0001FFF80000FFE000003F8000019297EA71E>I<03FC001FFF807FFFC0FFFFE0 FE03F07801F06000F80000F80000F80000F80000F80001F00003F00007E0000FC0001F80 003F00003E00007C00007C0000F80000F80000F80000F80000F80001F80001F80001F800 00000000000000000000000000000000000000000001F80001F80001F80001F80001F800 01F80015297DA81C>63 D<0001F000000003F800000003F800000007FC00000007FC0000 00077C0000000F3E0000000F3E0000000F3E0000001E3F0000001E1F0000003E1F800000 3C1F8000003C1F8000007C0FC00000780FC00000780FC00000F807E00000F007E00000F0 07E00001F003F00001E003F00003E003F80003E001F80003C001F80007C000FC0007FFFF FC0007FFFFFC000FFFFFFE000FFFFFFE000F00007E001F00003F001F00003F003E00003F 803E00001F803C00001F807C00001FC07C00000FC07800000FC0F8000007E0F8000007E0 23297EA828>65 DI<00 01FF00000FFFE0003FFFF8007FFFF801FF03F803F8007007F0003007E000100FC000001F 8000001F8000003F0000003F0000007E0000007E0000007E0000007E000000FC000000FC 000000FC000000FC000000FC000000FC000000FC000000FC000000FC0000007E0000007E 0000007E0000007E0000003F0000003F0000001F8000001F8000000FC0000007E0000407 F0000C03F8003C01FF00FC007FFFFC003FFFF8000FFFE00001FF001E2B7CA926>IIII<0001FF00000FFFE0003FFFFC007FFFFE01FF01FE03FC003E 07F0001C07E0000C0FC000041F8000001F8000003F0000003F0000007E0000007E000000 7E0000007E000000FC000000FC000000FC000000FC000000FC000000FC000000FC001FFE FC001FFEFC001FFE7E001FFE7E00003E7E00003E7E00003E3F00003E3F00003E1F80003E 1F80003E0FC0003E07E0003E07F0003E03FC003E01FF01FE007FFFFE003FFFFC000FFFE0 0001FF001F2B7CA928>III76 DII<0001FC0000000F FF8000003FFFE00000FFFFF80001FE03FC0003F800FE0007F0007F000FE0003F800FC000 1F801F80000FC01F000007C03F000007E03E000003E07E000003F07E000003F07E000003 F07C000001F0FC000001F8FC000001F8FC000001F8FC000001F8FC000001F8FC000001F8 FC000001F8FC000001F8FC000001F8FE000003F87E000003F07E000003F07E000003F03F 000007E03F000007E01F80000FC01F80000FC00FC0001F800FE0003F8007F0007F0003F8 00FE0001FF07FC0000FFFFF800003FFFE000000FFF80000001FC0000252B7DA92C>II82 D<007FC00001FFF80007FFFE000FFFFF001FC0 7F003F001F007E000E007C00060078000200F8000000F8000000F8000000F8000000FC00 0000FC0000007E0000007F8000003FF800001FFF80000FFFE00007FFF80003FFFC0000FF FE00000FFF000000FF0000003F8000001F8000000FC000000FC0000007C0000007C00000 07C0000007C0400007C060000F8070000F80F8001F00FE003F00FF80FE007FFFFC001FFF F80007FFE00000FF80001A2B7DA921>IIIII<00FE0007FF801FFFC03FFFE03E03F03801F03001F82000F80000 F80000F80000F80000F8007FF807FFF81FFFF83FE0F87F00F8FC00F8F800F8F800F8F800 F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8151B7E9A1D>97 DI<007F8001FFE007FFF80FFFFC1FC07C1F001C3E00087C00 007C00007C0000F80000F80000F80000F80000F80000F80000F800007C00007C00007E00 003E00041F001C1FC07C0FFFFC07FFFC01FFF0007F80161B7E9A1B>I<00003E00003E00 003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00 FC3E03FF3E07FFFE0FFFFE1FC1FE3F007E3E003E7C003E7C003EFC003EF8003EF8003EF8 003EF8003EF8003EF8003EF8003EFC003E7C003E7C003E3E007E3F00FE1FC1FE0FFFFE07 FFBE03FF3E00FC3E17297EA81F>I<007E0003FF8007FFC00FFFE01F83F03F00F03E0078 7C00787C003878003CFFFFFCFFFFFCFFFFFCFFFFFCF80000F80000F800007800007C0000 7C00003E00003F000C1FC07C0FFFFC07FFFC01FFF0007F80161B7E9A1B>I<001FC0007F C000FFC001FFC003E04003C00007C00007C00007C00007C00007C00007C00007C00007C0 00FFFE00FFFE00FFFE00FFFE0007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00012297FA812>I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0 003E03E0003E03E0007C01F0007C01F0007C01F0007C01F0007C01F0007C01F0003E03E0 003E03E0001F07C0001FFFC0003FFF80003BFE000038F8000078000000780000003C0000 003FFFC0003FFFF8001FFFFC001FFFFE003FFFFF007C007F00F8001F80F8000F80F8000F 80F8000F80FC001F807E003F003F80FE003FFFFE000FFFF80007FFF00000FF80001A287E 9A1E>III107 DI II<007F000001FFC00007FFF0000FFFF8001FC1FC 003F007E003E003E007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F80F8000F807C001F007C001F007E003F003E003E003F007E001FC1FC 000FFFF80007FFF00001FFC000007F0000191B7E9A1E>II<00FC3E03FF3E07FFBE0FFFFE1FC1FE3F80FE3E007E7E003E7C003E7C003EF800 3EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C003E7E003E3E007E3F007E1FC1 FE1FFFFE0FFFBE03FF3E01F83E00003E00003E00003E00003E00003E00003E00003E0000 3E00003E00003E00003E00003E17277E9A1F>II<03FC001FFF803FFFC07FFFC07C07C0F80180F80000F80000F80000 FC00007F80007FF8003FFE001FFF0007FF8000FFC0000FE00007E00003E04003E06003E0 F007E0FC0FC0FFFFC07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C00007C0 0007C00007C00007C000FFFFC0FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C04007E1C003FFE003FFE001FF8000FC0013227FA116>IIII<7C000FC03E001F803F001F001F803E000F807C0007C0FC0003 E0F80001F1F00001FBE00000FFC000007FC000003F8000001F0000001F0000003F800000 7FC00000FBC00000F3E00001F1F00003E0F80007C07C000F807C000F803E001F001F003E 000F807E000FC0FC0007E01B1B809A1C>I II E /Fr 19 121 df<00001E000000003E000000 00FE00000003FE0000003FFE0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00007FFFFF FFC07FFFFFFFC07FFFFFFFC07FFFFFFFC0223879B731>49 D<0007FE0000007FFFE00001 FFFFF80003FFFFFE0007F01FFF000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC0 007FF0FFE0007FF8FFF0003FF8FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE000 3FFC3FC0003FFC1F80003FFC0000003FFC0000003FF80000007FF80000007FF00000007F F0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00000007F800 00000FF00000001FC00000003F800000007F00000000FC00000001F800000001F0003C00 03E0003C0007C0003C000F000078001E000078003C00007800780000F800F00000F801FF FFFFF803FFFFFFF007FFFFFFF00FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFF FFF0FFFFFFFFE0FFFFFFFFE0FFFFFFFFE026387BB731>I<0003FF8000001FFFF000007F FFFE0000FE03FF0001F000FF8003C000FFC00780007FE00FF0007FF00FF8007FF01FFC00 7FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF80FFC007FF007F8007F F003F0007FF0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00 00001FF000000FFFC000000FFF8000000FFFF800000003FE00000000FF800000007FE000 00003FF00000003FF80000003FFC0000001FFC0000001FFE0000001FFE0200001FFF1FC0 001FFF3FE0001FFF7FF0001FFF7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF800 1FFEFFF0001FFE7FF0003FFC7FE0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FF C003FFFFFF0001FFFFFE00003FFFF0000007FF800028397CB731>I<00000007C0000000 000FC0000000000FC0000000001FC0000000003FC0000000007FC000000000FFC0000000 00FFC000000001FFC000000003FFC000000007FFC00000000FFFC00000000FFFC0000000 1EFFC00000003CFFC00000007CFFC0000000F8FFC0000000F0FFC0000001E0FFC0000003 C0FFC0000007C0FFC000000F80FFC000000F00FFC000001E00FFC000003C00FFC000007C 00FFC00000F800FFC00000F000FFC00001E000FFC00003C000FFC00007C000FFC0000F80 00FFC0000F0000FFC0001E0000FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFF FFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC0000007FFFFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A37 7DB631>I<04000000C00F800007C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000F FFFFFC000FFFFFF8000FFFFFF0000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F80 0000000F800000000F800000000F800000000F800000000F800000000F800000000F8000 00000F81FF00000F8FFFE0000FBFFFF8000FFE03FE000FF000FF000FC000FF800F80007F C00F00007FE00700007FF00000003FF00000003FF80000003FF80000003FF80000003FFC 0000003FFC0600003FFC1F80003FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFF F0003FFCFFF0003FF8FFE0003FF8FFE0003FF87FC0007FF07F00007FF03C00007FE03E00 00FFC01F0000FF800FC003FF0007F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC 000026397BB731>I<00000FF80000007FFF000003FFFF80000FFC07C0001FE001E0003F C001F0007F0007F000FF000FF001FE001FF803FC003FF807FC003FF80FFC003FF80FF800 3FF81FF8001FF01FF8000FE03FF80007C03FF00000003FF00000007FF00000007FF00000 007FF00000007FF07FF000FFF0FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0 FFFC001FF0FFFC000FF8FFF8000FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFF F0000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0 000FFF3FF0000FFF3FF0000FFE3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF800 1FF807FC001FF003FC001FF001FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF8 000000FFC00028397CB731>I<1E00000000001F00000000001FF0000000001FFFFFFFFF C01FFFFFFFFFC01FFFFFFFFFC03FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE 003FFFFFFFFC003FFFFFFFF8003FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC0 007800001F80007800001F00007800003E0000F000007E0000F00000FC0000F00001F800 00000003F00000000003E00000000007E0000000000FC0000000000F80000000001F8000 0000003F80000000003F00000000007F00000000007F0000000000FF0000000000FE0000 000001FE0000000001FE0000000003FE0000000003FE0000000003FE0000000007FC0000 000007FC0000000007FC000000000FFC000000000FFC000000000FFC000000000FFC0000 00000FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000 00001FFC000000001FFC000000001FFC000000001FFC000000000FF80000000007F00000 000003E00000002A3B7BB931>I<0001FF8000000FFFF800003FFFFE00007F00FF0000F8 003F8001F0000FC003E0000FE007C00007F007C00007F00FC00003F80FC00003F81FC000 03F81FC00003F81FE00003F81FF00003F81FF80003F81FFC0007F01FFF0007F01FFFC007 E00FFFF00FE00FFFFC1FC007FFFE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE00 007FFFFF00003FFFFF80003FFFFFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1F C007FFFC3F8001FFFE3F8000FFFE7F00003FFF7F00000FFFFE000003FFFE000001FFFE00 0000FFFE000000FFFE0000007FFE0000007FFE0000007EFE0000007E7F0000007E7F0000 00FC3F800000FC3FC00001F81FE00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE 00003FFFF8000003FF800028397CB731>I<0001FF8000001FFFF000007FFFFC0000FF81 FE0001FE007F0007FC003F800FF8003FC00FF8001FE01FF8001FF03FF0001FF03FF0000F F87FF0000FF87FF0000FFCFFF0000FFCFFF0000FFCFFF0000FFEFFF0000FFEFFF0000FFE FFF0000FFEFFF0000FFEFFF0000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0001FFF7F F0001FFF3FF0001FFF3FF0001FFF1FF0003FFF0FF8003FFF07F8007FFF03FC006FFF01FE 01CFFF00FFEF8FFF007FFF0FFF000FFE0FFE0000000FFE0000000FFE0000000FFE000000 0FFC0000000FFC03E0001FFC07F0001FF80FF8001FF81FFC001FF01FFC003FF01FFC003F E01FFC003FC01FF8007FC00FF000FF800FE001FF0007C003FE0007F01FFC0003FFFFF000 01FFFFC000007FFF0000000FF8000028397CB731>I<00000001F80000000000000001F8 0000000000000003FC0000000000000003FC0000000000000007FE0000000000000007FE 0000000000000007FE000000000000000FFF000000000000000FFF000000000000001FFF 800000000000001FFF800000000000001FFF800000000000003FFFC00000000000003FFF C00000000000007FFFE00000000000007DFFE00000000000007DFFE0000000000000FDFF F0000000000000F8FFF0000000000001F8FFF8000000000001F07FF8000000000001F07F F8000000000003F07FFC000000000003E03FFC000000000007E03FFE000000000007C01F FE00000000000FC01FFF00000000000F801FFF00000000000F800FFF00000000001F800F FF80000000001F0007FF80000000003F0007FFC0000000003E0003FFC0000000003E0003 FFC0000000007E0003FFE0000000007C0001FFE000000000FC0001FFF000000000F80000 FFF000000000F80000FFF000000001FFFFFFFFF800000001FFFFFFFFF800000003FFFFFF FFFC00000003FFFFFFFFFC00000003E000003FFC00000007E000003FFE00000007C00000 1FFE0000000FC000001FFF0000000F8000000FFF0000001F8000000FFF8000001F000000 0FFF8000001F00000007FF8000003F00000007FFC000003E00000003FFC000007E000000 03FFE00000FF00000001FFE000FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001 FFFFFFF0FFFFFC0001FFFFFFF0443B7DBA4B>65 D<0000001FFF000030000001FFFFE000 F000000FFFFFFC01F000007FFFFFFE03F00001FFFE007F87F00003FFE0000FCFF0000FFF 000003FFF0001FFC000001FFF0003FF80000007FF0007FF00000003FF000FFC00000003F F001FFC00000001FF003FF800000000FF007FF000000000FF00FFF0000000007F00FFE00 00000007F01FFE0000000003F01FFE0000000003F03FFC0000000003F03FFC0000000001 F03FFC0000000001F07FFC0000000001F07FF80000000001F07FF80000000000007FF800 0000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF80000000000 00FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF800 0000000000FFF8000000000000FFF80000000000007FF80000000000007FF80000000000 007FF80000000000007FFC0000000000F03FFC0000000000F03FFC0000000000F03FFC00 00000000F01FFE0000000000F01FFE0000000001E00FFE0000000001E00FFF0000000001 E007FF0000000003C003FF8000000003C001FFC0000000078000FFE00000000F00007FF0 0000001F00003FF80000003E00001FFC0000007C00000FFF000001F8000003FFE00007F0 000001FFFE003FC00000007FFFFFFF000000000FFFFFFC0000000001FFFFF00000000000 1FFF0000003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007FE00 000FF801FF00001FFC00FF80001FFC007FC0001FFC007FE0001FFC003FE0000FF8003FF0 000FF8003FF00007F0003FF00001C0003FF0000000003FF0000000003FF0000000003FF0 000000FFFFF000000FFFFFF000007FF83FF00001FF803FF00007FE003FF0000FF8003FF0 001FF0003FF0003FE0003FF0007FE0003FF0007FE0003FF000FFC0003FF000FFC0003FF0 00FFC0003FF000FFC0003FF000FFC0007FF0007FE0007FF0007FE000DFF0003FF0039FF8 001FFC0F0FFFF007FFFE0FFFF001FFFC07FFF0003FE000FFF02C267DA530>97 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007FC001FC00FF8000FE0 0FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE00003FC7FE00003FC7F E00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE0000000FFE0000000FFE0 000000FFE00000007FE00000007FE00000007FE00000003FE00000003FF000003C1FF000 003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E000FFC03FC0003FFFFF 000007FFFC000000FFE00026267DA52D>101 D<00FF00000000FFFF00000000FFFF0000 0000FFFF00000000FFFF0000000007FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF007FC00003FF01FFF80003FF07FFFC0003FF0F03FE0003FF1C01 FF0003FF3001FF8003FF6000FF8003FFE000FFC003FFC000FFC003FF8000FFC003FF8000 FFC003FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3F FFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF303C7CBB37>104 D<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FFFE1C01FF0007FE3001FF 8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003FF8000FFC003FF8000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFF FFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<00FF01FF8000FFFF0FFFF000FFFF 3FFFFC00FFFFFE03FF00FFFFF000FF8003FFC0007FC003FF80003FE003FF00003FF003FF 00001FF803FF00001FFC03FF00000FFC03FF00000FFE03FF00000FFE03FF000007FE03FF 000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF 000007FF03FF000007FF03FF000007FF03FF000007FE03FF000007FE03FF00000FFE03FF 00000FFC03FF00000FFC03FF00001FF803FF00001FF803FF00003FF003FF80003FE003FF C0007FC003FFF001FF8003FFFC07FF0003FF3FFFFC0003FF0FFFF00003FF01FF000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 00000000FFFFFC000000FFFFFC000000FFFFFC000000FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80FFFE707FC007FE60FFE003FEE0FFE0 03FEC0FFE003FFC0FFE003FF807FC003FF807FC003FF803F8003FF800E0003FF00000003 FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF000000FFFFFE0000FFFFFE00 00FFFFFE0000FFFFFE000023267DA529>114 D<00078000000780000007800000078000 00078000000F8000000F8000000F8000000F8000001F8000001F8000003F8000003F8000 007F800000FF800001FF800007FF80001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C00FF8078 00FFC078007FC070003FE0E0001FFFC00007FF800001FF001E377EB626>116 D120 D E /Fs 78 123 df<007F07F001FF1FF003FF3FF007807800070070000F00F0000F00F0000F00F0000F00 F0000F00F0000F00F0000F00F000FFF8FF80FFF8FF80FFF8FF800F00F0000F00F0000F00 F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00 F0000F00F0000F00F0000F00F0000F00F0000F00F0001C20809F1B>11 D<0000F000F8F001F8F003F8F00780000700000F00000F00000F00000F00000F00000F00 000F0000FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00 F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0142180A018>I<00 F8F001F8F003F8F00780F00700F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0FF F8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F 00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01420809F18>I<783C783C783C 783C180C3018301860306030E0700E0A7F9F17>34 D<007000E001C00380078007000E00 1E001E003C003C003C0078007800780078007000F000F000F000F000F000F000F000F000 F000F000F000F000700078007800780078003C003C003C001E001E000E00070007800380 01C000E000700C2E7EA112>40 DI<018001C001800180C183E187F99F7DBE1FF807E007E01FF87DBEF99FE187C1 830180018001C0018010147DA117>I<0006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 00060000FFFFFFF0FFFFFFF0000600000006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 1C207D9A23>I<787878781830306060E0050A7D830D>II< F0F0F0F004047C830D>I<000100030003000600060006000C000C000C00180018001800 300030003000600060006000C000C000C00180018001800300030003000600060006000C 000C000C00180018001800300030003000600060006000C000C000C000102D7DA117>I< 03F00007F8001FFE001E1E003C0F00380700780780780780700380F003C0F003C0F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0780780 7807807807803807003C0F001E1E001FFE000FFC0003F00012207E9E17>I<00C001C00F C0FFC0FFC0F3C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C0FFFEFFFEFFFE0F1F7C9E17>I<07F0000FFC001FFE 00383F00700F00600780E00780E003C04003C04003C00003C00003C00003C00007800007 80000F00000E00001C00003C0000780000E00001C0000380000700000E00001C00003800 00700000FFFFC0FFFFC0FFFFC0121F7E9E17>I<03F0000FFC001FFE003C1F00780F0030 0780200780000780000780000780000F00000F00003E0003FC0003F80003FC00001E0000 0F000007800003800003C00003C00003C00003C08003C0C003C0C00780700F807C1F003F FE000FFC0003F00012207E9E17>I<003E00003E00005E00005E0000DE0001DE00019E00 039E00039E00079E00071E000F1E000E1E001E1E003C1E003C1E00781E00781E00F01E00 FFFFF0FFFFF0FFFFF0001E00001E00001E00001E00001E00001E00001E00001E00141E7F 9D17>I<7FFF007FFF007FFF0078000078000078000078000078000078000078000079F0 007FFC007FFE007F1F007C07007C07807807800003C00003C00003C00003C00003C00003 C00003C0400780600780F00F007C1E003FFC001FF80007E000121F7E9D17>I<007C0001 FE0003FE000782000F00001E00003C00003C000078000078000078F800F3FC00F7FE00FE 1F00FC0F80F80780F80780F003C0F003C0F003C0F003C0F003C07003C07803C07803C078 07803807803C0F001E1E000FFE0007F80001F00012207E9E17>II<03F0000FFC001FFE003E1F003C0F007807807807 807807807807807807807807803C0F001E1E000FFC0007F8000FFC001F3E003C0F007807 80780780F003C0F003C0F003C0F003C0F003C0F003C07807807C0F803E1F001FFE000FFC 0003F00012207E9E17>I<03F00007F8000FFC001E1E003C0F00780700780780F00780F0 0380F003C0F003C0F003C0F003C0F003C0F003C07807C07807C07C0FC03E1FC01FFBC00F F3C007C780000780000780000700000F00001E00201E00307C007FF8003FF0000FC00012 207E9E17>II61 D<001F0000001F0000003F8000003B8000003B8000 007BC0000073C0000071C00000F1E00000E1E00000E0E00001E0F00001E0F00001C0F000 03C0780003C078000380780007803C0007803C0007003C000FFFFE000FFFFE000FFFFE00 1E000F001E000F003C000F803C0007803C000780780007C0780003C0780003C0F00003E0 1B207F9F1E>65 DI<001FC000FFF801FFFC03E03C07800C0F00001E00003E00003C00007C0000780000 780000780000F00000F00000F00000F00000F00000F00000F00000F00000780000780000 7800007C00003C00003E00001E00000F000207800E03E03E01FFFC00FFF0001FC017227D A01D>IIII<001FE000FFF801FFFE03E0 3E07800E0F00001E00003E00003C00007C0000780000780000780000F00000F00000F000 00F00000F00000F00000F003FEF003FE7803FE78001E78001E7C001E3C001E3E001E1E00 1E0F001E07801E03E03E01FFFE00FFF8001FC017227DA01E>III75 DII I<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F 00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003 C0F00003C0F00003C0F00003C0F80007C07800078078000780780007803C000F003C000F 001E001E001F003E000F807C0007E1F80003FFF00000FFC000003F00001A227DA021>I< FFF800FFFF00FFFF80F00FC0F003E0F001E0F000F0F000F0F000F0F000F0F000F0F000F0 F000F0F001E0F003E0F00FC0FFFF80FFFF00FFF800F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000F00000F00000F0000014207B9F1D>I<003F000000 FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F007800078078 00078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F0 0003C0F00003C0F00003C07800078078000780781E07803C0F0F003C0F8F001E079E001F 03FE000F83FC0007E1F80003FFF00000FFF800003F780000007C0000003E0000001E0000 001F0000000F801A277DA021>II<01FC0007FF800FFFC01F03C03C00C03C0000780000780000780000 7800007800007C00003C00003F00001FE0000FFC0007FE0001FF00003F800007C00003C0 0003E00001E00001E00001E00001E00001E00001C0C003C0F007C0FC0F807FFF001FFE00 03F80013227EA019>I IIII<780007807C000F003E001F001E001E000F003C000F807C 000780780003C0F00003E1F00001F1E00000F3C000007FC000007F8000003F0000001F00 00001E0000003F0000007F8000007FC00000F3C00001F1E00001E0F00003C0F80007C078 0007803C000F003E001F001E001E000F003C000F807C0007C0780003C0F00003E01B207F 9F1E>IIII<381C3018301860306030C060F078F078F078F0780E0A799F17>II<07E03FF87FFC701E401F000F000F000F003F07FF1FFF7E0F F80FF00FF00FF00FF83F7FFF3FEF1F8F10147E9316>97 DI<03F00FFC1FFE3E0E3C0278007800F000F0 00F000F000F000F000780078003C013E0F1FFF0FFE03F010147E9314>I<000780000780 00078000078000078000078000078000078000078000078000078000078007C7800FF780 1FFF803E1F807C0780780780F80780F00780F00780F00780F00780F00780F00780F00780 780780780F803E1F801FFF800FF78007C78011207E9F17>I<03F0000FFC001FFE003E1F 003C0700780700700380FFFF80FFFF80FFFF80F00000F00000F000007000007800003C01 003E07001FFF0007FE0001F80011147F9314>I<007E01FE03FE078007000F000F000F00 0F000F000F000F00FFF0FFF0FFF00F000F000F000F000F000F000F000F000F000F000F00 0F000F000F000F000F000F000F20809F0E>I<03E0F00FFFF01FFFF03E3E003C1E00780F 00780F00780F00780F00780F003C1E003E3E001FFC003FF80033E0003000003800003FFE 003FFF801FFFC03FFFE07803F0F000F0F000F0F000F0F801F07E07E03FFFC00FFF0003FC 00141E7F9317>III<01E001E001E001E00000000000000000000000000000000001E0 01E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E0 01E001E001E001E001E001E0C3C0FFC0FF803F000B29839F0C>IIIII<01F80007FE001FFF803F0FC03C03C07801E07801E0 F000F0F000F0F000F0F000F0F000F0F000F07801E07801E03C03C03F0FC01FFF8007FE00 01F80014147F9317>II<03C7800F F7801FFF803E1F807C0F80780780780780F00780F00780F00780F00780F00780F00780F8 07807807807C0F803E1F801FFF800FF78007C78000078000078000078000078000078000 0780000780000780000780111D7E9317>II<07F01FFC3FFC78 0C7800780078007C003FC01FF00FF803F8007C003C003CC03CF07CFFF87FF00FC00E147F 9311>I<1E001E001E001E001E001E00FFF0FFF0FFF01E001E001E001E001E001E001E00 1E001E001E001E001E001E001E201FF00FF007C00C1A7F9910>IIII<7801E07C03C03E07801E0F000F0F00079E0003FC0003F800 01F80000F00001F00001F80003FC00079E000F0F000E0F001E07803C03C07801E0F801F0 1414809315>II<7FFF7FFF7FFF00 3E003C007800F800F001E003E007C007800F001F001E003C007C00FFFFFFFFFFFF10147F 9314>I E /Ft 76 125 df<007E1F0001C1B1800303E3C00703C3C00E03C1800E01C000 0E01C0000E01C0000E01C0000E01C0000E01C000FFFFFC000E01C0000E01C0000E01C000 0E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C000 0E01C0000E01C0000E01C0000E01C0007F87FC001A1D809C18>11 D<007E0001C1800301800703C00E03C00E01800E00000E00000E00000E00000E0000FFFF C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C07F87F8151D809C17>I<007FC001C1C00303C00703C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C0FFFFC00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C07F CFF8151D809C17>I<003F07E00001C09C18000380F018000701F03C000E01E03C000E00 E018000E00E000000E00E000000E00E000000E00E000000E00E00000FFFFFFFC000E00E0 1C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C 000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C00 0E00E01C007FC7FCFF80211D809C23>I<6060F0F0F8F868680808080808081010101020 20404080800D0C7F9C15>34 D<60F0F8680808081010204080050C7C9C0C>39 D<004000800100020006000C000C0018001800300030007000600060006000E000E000E0 00E000E000E000E000E000E000E000E000E000600060006000700030003000180018000C 000C00060002000100008000400A2A7D9E10>I<800040002000100018000C000C000600 060003000300038001800180018001C001C001C001C001C001C001C001C001C001C001C0 01C0018001800180038003000300060006000C000C00180010002000400080000A2A7E9E 10>I<01800180018001804182F18F399C0FF003C003C00FF0399CF18F41820180018001 80018010127E9E15>I<0006000000060000000600000006000000060000000600000006 0000000600000006000000060000000600000006000000060000FFFFFFE0FFFFFFE00006 000000060000000600000006000000060000000600000006000000060000000600000006 00000006000000060000000600001B1C7E9720>I<60F0F0701010101020204080040C7C 830C>II<60F0F06004047C830C>I<0001000300060006000600 0C000C000C0018001800180030003000300060006000C000C000C0018001800180030003 000300060006000C000C000C00180018001800300030003000600060006000C000C00010 297E9E15>I<03C00C301818300C300C700E60066006E007E007E007E007E007E007E007 E007E007E007E007E007E00760066006700E300C300C18180C3007E0101D7E9B15>I<03 0007003F00C7000700070007000700070007000700070007000700070007000700070007 0007000700070007000700070007000F80FFF80D1C7C9B15>I<07C01830201C400C400E F00FF80FF807F8077007000F000E000E001C001C00380070006000C00180030006010C01 180110023FFE7FFEFFFE101C7E9B15>I<07E01830201C201C781E780E781E381E001C00 1C00180030006007E00030001C001C000E000F000F700FF80FF80FF80FF00E401C201C18 3007E0101D7E9B15>I<4000007FFF807FFF007FFF004002008004008004008008000010 0000100000200000600000400000C00000C00001C0000180000180000380000380000380 00038000078000078000078000078000078000078000030000111D7E9B15>55 D<03E00C301008200C20066006600660067006780C3E083FB01FE007F007F818FC307E60 1E600FC007C003C003C003C00360026004300C1C1007E0101D7E9B15>I<03C00C301818 300C700C600EE006E006E007E007E007E007E0076007700F300F18170C2707C700060006 000E300C780C78187010203030C00F80101D7E9B15>I<60F0F060000000000000000000 0060F0F06004127C910C>I<60F0F0600000000000000000000060F0F070101010102020 4080041A7C910C>I<7FFFFFC0FFFFFFE000000000000000000000000000000000000000 00000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<0FE03038401CE00EF00EF00EF00E000C001C0030006000C00080018001000100010001 00010001000000000000000000000003000780078003000F1D7E9C14>63 D<000600000006000000060000000F0000000F0000000F00000017800000178000001780 000023C0000023C0000023C0000041E0000041E0000041E0000080F0000080F0000180F8 000100780001FFF80003007C0002003C0002003C0006003E0004001E0004001E000C001F 001E001F00FF80FFF01C1D7F9C1F>65 DI< 001F808000E0618001801980070007800E0003801C0003801C0001803800018078000080 7800008070000080F0000000F0000000F0000000F0000000F0000000F0000000F0000000 F0000000700000807800008078000080380000801C0001001C0001000E00020007000400 0180080000E03000001FC000191E7E9C1E>IIII<001F8080 00E0618001801980070007800E0003801C0003801C000180380001807800008078000080 70000080F0000000F0000000F0000000F0000000F0000000F0000000F000FFF0F0000F80 700007807800078078000780380007801C0007801C0007800E00078007000B8001801180 00E06080001F80001C1E7E9C21>III76 DII<003F800000E0E0000380380007001C000E000E001C 0007003C00078038000380780003C0780003C0700001C0F00001E0F00001E0F00001E0F0 0001E0F00001E0F00001E0F00001E0F00001E0700001C0780003C0780003C0380003803C 0007801C0007000E000E0007001C000380380000E0E000003F80001B1E7E9C20>II82 D<07E0801C1980300580700380600180E00180E00080E00080E00080F00000F800007C00 007FC0003FF8001FFE0007FF0000FF80000F800007C00003C00001C08001C08001C08001 C0C00180C00180E00300D00200CC0C0083F800121E7E9C17>I<7FFFFFC0700F01C0600F 00C0400F0040400F0040C00F0020800F0020800F0020800F0020000F0000000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F 0000000F0000000F0000000F0000000F0000000F0000001F800003FFFC001B1C7F9B1E> II87 D<7FF0FFC00FC03E00078018 0003C0180003E0100001E0200001F0600000F0400000788000007D8000003D0000001E00 00001F0000000F0000000F8000000F80000013C0000023E0000021E0000041F00000C0F8 000080780001007C0003003C0002001E0006001F001F003F80FFC0FFF01C1C7F9B1F>I< FFF007FC0F8001E00780008007C0018003C0010003E0020001F0020000F0040000F80400 00780800007C1800003C1000001E2000001F2000000F4000000FC0000007800000078000 000780000007800000078000000780000007800000078000000780000007800000078000 007FF8001E1C809B1F>I<7FFFF07C01F07001E06003C06003C0400780400F80400F0040 1E00001E00003C00007C0000780000F00000F00001E00003E00003C0100780100780100F 00101F00301E00203C00203C00607800E0F803E0FFFFE0141C7E9B19>I<080810102020 40404040808080808080B0B0F8F8787830300D0C7A9C15>92 D<1FC00030700078380078 1C00301C00001C00001C0001FC000F1C00381C00701C00601C00E01C40E01C40E01C4060 3C40304E801F870012127E9115>97 DI<07E00C301878307870306000E000E000E000E000E000E00060007004300418080C30 07C00E127E9112>I<003F00000700000700000700000700000700000700000700000700 00070000070003E7000C1700180F00300700700700600700E00700E00700E00700E00700 E00700E00700600700700700300700180F000C370007C7E0131D7E9C17>I<03E00C3018 18300C700E6006E006FFFEE000E000E000E00060007002300218040C1803E00F127F9112 >I<00F8018C071E061E0E0C0E000E000E000E000E000E00FFE00E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E007FE00F1D809C0D>I<00038003C4 C00C38C01C3880181800381C00381C00381C00381C001818001C38000C300013C0001000 003000001800001FF8001FFF001FFF803003806001C0C000C0C000C0C000C06001803003 001C0E0007F800121C7F9215>II< 18003C003C0018000000000000000000000000000000FC001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C00FF80091D7F9C0C>I<00C001E001E000 C000000000000000000000000000000FE000E000E000E000E000E000E000E000E000E000 E000E000E000E000E000E000E000E000E000E000E060E0F0C0F1C061803E000B25839C0D >IIIII<03F0000E1C 00180600300300700380600180E001C0E001C0E001C0E001C0E001C0E001C06001807003 803003001806000E1C0003F00012127F9115>II<03C100 0C3300180B00300F00700700700700E00700E00700E00700E00700E00700E00700600700 700700300F00180F000C370007C700000700000700000700000700000700000700000700 003FE0131A7E9116>II<1F9030704030C010C010E010F8007F803FE00F F000F880388018C018C018E010D0608FC00D127F9110>I<04000400040004000C000C00 1C003C00FFE01C001C001C001C001C001C001C001C001C001C101C101C101C101C100C10 0E2003C00C1A7F9910>II< FF07E03C03801C01001C01000E02000E020007040007040007040003880003880003D800 01D00001D00000E00000E00000E00000400013127F9116>II<7F8FF00F 03800F030007020003840001C80001D80000F00000700000780000F800009C00010E0002 0E000607000403801E07C0FF0FF81512809116>II<7FFC 70386038407040F040E041C003C0038007000F040E041C043C0C380870087038FFF80E12 7F9112>III E /Fu 5 86 df<60F0F06004047C830C>58 D<0000038000000F0000003C000000F00000 03C000000F0000003C000000F0000003C000000F0000003C000000F0000000F00000003C 0000000F00000003C0000000F00000003C0000000F00000003C0000000F00000003C0000 000F000000038019187D9520>60 D62 D<01FFFF00003C01C0003800E0003800F0003800700038007000 700070007000F0007000F0007001E000E003C000E0078000E01F0000FFFC0001C00F0001 C0078001C003C001C003C0038003C0038003C0038003C0038003C0070007800700070007 000E0007001C000E007800FFFFC0001C1C7E9B1F>66 D<7FF03FE00F0007000E0006000E 0004000E0004000E0004001C0008001C0008001C0008001C000800380010003800100038 0010003800100070002000700020007000200070002000E0004000E0004000E0004000E0 008000E0008000E00100006002000060040000300800001830000007C000001B1D7D9B1C >85 D E /Fv 64 123 df<01F1C003F1C007F1C00701C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C0FFF1C0FFF1C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0121D809C16>13 D<00F0000001F8000003FC0000039C0000070E0000070E0000070E0000070E0000070E00 00071E0000073C0000073800000778000003F0000003E0030007C003000F8006001FC006 003DC0060078E00C0070F00C00F0701800E0783800E03C3000E01C6000E00EC000700F80 0070078040381FE1C01FF8FFC007E03F001A1F7E9D1F>38 D<00E001C0038007000E000E 001C001C003800380038007000700070007000E000E000E000E000E000E000E000E000E0 00E000E000E00070007000700070003800380038001C001C000E000E000700038001C000 E00B2A7E9E10>40 DI<018001C001800180 6186F99F7DBE1FF807E007E01FF87DBEF99F61860180018001C0018010127E9E15>I<00 060000000600000006000000060000000600000006000000060000000600000006000000 060000000600000006000000060000FFFFFFE0FFFFFFE000060000000600000006000000 060000000600000006000000060000000600000006000000060000000600000006000000 0600001B1C7E9720>III<01 0007003F00FF00C700070007000700070007000700070007000700070007000700070007 000700070007000700070007000700FFF8FFF80D1C7C9B15>49 D<07C01FF03878701C60 1EE00EC00F400F400700070007000F000E001E001C003C007800F001E001C0038007000E 001C0038007000FFFFFFFF101C7E9B15>I58 D<7FFFFFC0FFFFFFE00000000000000000000000000000000000000000 000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<001C0000003E0000003E0000002E0000006700000067000000E7800000C7800000C380 0001C3C0000183C0000181C0000381E0000381E0000700F0000700F0000600F0000E0078 000FFFF8000FFFF8001C003C001C003C0018003C0038001E0038001E0070001F0070000F 0070000F00E0000780191D7F9C1C>65 DI<003FC000FFF003C0F00780300F00001E00003C00003C0000780000780000780000 F00000F00000F00000F00000F00000F00000F00000F00000F00000780000780000780000 3C00003C00001E00000F000807801803C07800FFF0003F80151F7D9D1B>IIII<003F8001FFF003C0F80780 380F00181E00003C00003C0000780000780000780000F00000F00000F00000F00000F000 00F00000F007F8F007F8F000387800387800387800383C00383C00381E00380F00380780 3803C0F801FFF0003F80151F7D9D1C>III75 DIII<003F000001FFE00003FFF00007C0F800 0F807C001E001E003E001F003C000F00780007807800078078000780F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F80007C07800078078000780 7C000F803C000F003E001F001F003E000F807C0007C0F80003FFF00001FFE000003F0000 1A1F7E9D1F>II<003F000001FFE0 0003FFF00007C0F8000F807C001F003E003E001F003C000F007800078078000780780007 80F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003 C07800078078000780780E07803C0F0F003E079F001E03DE000F83FC0007C1F80003FFF0 0001FFF800003F780000003C0000003E0000001F0000000F801A237E9D1F>II<03F8000FFE001C0F00380700700300600000 E00000E00000E00000E00000F000007800007F00003FE0001FFC0007FE0001FF00001F80 0007800003C00003C00001C00001C00001C00001C0C00180E00380F007007C0E001FFC00 07F000121F7E9D17>IIII I<78000E007C001E003C003C001E0038000F0070000F00F0000781E00003C1C00001C3C0 0001E7800000F70000007E0000003E0000003C0000003C0000007E00000077000000E780 0001E3800003C1C0000381E0000700F0000F00F8000E0078001C003C003C003E0078001F 0070000F00F0000F80191D7F9C1C>II<7FFFF07FFF F00001E00003E00003C00007C0000780000F00001F00001E00003E00003C0000780000F8 0000F00001F00001E00003C00007C0000780000F80000F00001E00003E00003C00007C00 00780000FFFFF0FFFFF0141D7E9C19>II93 D<0FC03FF07FF87038401C001C001C00FC0FFC3FFC781CE01CE01CE01CF07C 7FFC7FDC3F1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000E000E000E000700070043C 1C1FFC0FF807E00E127E9112>I<000E000E000E000E000E000E000E000E000E000E000E 0F8E1FEE3FFE7C3E700E700EE00EE00EE00EE00EE00EE00EF00E701E7C3E3FFE1FEE0F8E 0F1D7E9C15>I<07C01FE03FF078787018601CFFFCFFFCFFFCE000E000E000700070043C 1C3FFC1FF807E00E127E9112>I<00FC01FC03FC07000E000E000E000E000E000E000E00 FFE0FFE00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E1D809C0D>I<03C3C00FFFC01FFFC01C3800381C00381C00381C00381C00381C001C38 001FF8001FF0003BC0003800003800001FFC001FFF003FFF80700780E001C0E001C0E001 C0F003C07C0F803FFF001FFE0007F800121B7F9115>III107 DIII<03F0000FFC001FFE003C0F 00780780700380E001C0E001C0E001C0E001C0E001C0F003C07003807807803C0F001FFE 000FFC0003F00012127F9115>I I<078E1FEE3FFE7C3E781E700EE00EE00EE00EE00EE00EE00EF00E701E7C3E3FFE1FEE0F 8E000E000E000E000E000E000E000E000E0F1A7E9115>II<1FC03FF07F F0F030E000E000F0007F003FC01FE000F0003800388038F078FFF07FE01FC00D127F9110 >I<1C001C001C001C001C001C00FFE0FFE01C001C001C001C001C001C001C001C001C00 1C001C001C001C201FF00FF007C00C187F970F>IIII<7003807807003C0E001C1C000E1C0007380003F00001E00001C00001E000 03F0000738000E18000E1C001C0E00380700700380F003C01212809113>II<7FFC7FFC7FFC007800F000E001E003C00380 07000F001E001C003C007800FFFCFFFCFFFC0E127F9112>I E /Fw 33 123 df<007E01C007000E001C003C003800780078007FF0F000F000F0007000700070 00300018000C1807E00F147E9312>15 D<70F8F8F87005057C840D>58 D<70F8FCFC74040404080810102040060E7C840D>I<000001C00000078000001E000000 78000001E00000078000000E00000038000000F0000003C000000F0000003C000000F000 0000F00000003C0000000F00000003C0000000F0000000380000000E0000000780000001 E0000000780000001E0000000780000001C01A1A7C9723>I<0001000300030006000600 06000C000C000C00180018001800300030003000600060006000C000C000C00180018001 800300030003000600060006000C000C000C001800180018003000300030006000600060 00C000C000C000102D7DA117>II<000002000000060000000E0000000E0000001E 0000001F0000002F0000002F0000004F0000008F0000008F0000010F0000010F0000020F 0000040F0000040F0000080F8000080780001007800020078000200780007FFF80004007 8000800780018007800100078002000780020007C0040003C00C0003C01E0007C0FF807F FC1E207E9F22>65 D<0001FC0000070700001C01C0003000E000E0006001C00070038000 7007800038070000380E0000381E0000381C0000383C0000383C00003878000078780000 787800007878000078F00000F0F00000F0F00000E0F00001E0F00001C0F00003C0700003 807000070078000F0038001E0038003C001C0070000E00E0000783800001FC00001D217E 9F23>79 D<0FFFFFFC1E03C0381803C0181003C0082003C0082007800860078008400780 0840078008800F0010000F0000000F0000000F0000001E0000001E0000001E0000001E00 00003C0000003C0000003C0000003C00000078000000780000007800000078000000F000 0000F0000000F0000000F0000001F000007FFFC0001E1F7F9E1B>84 D<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E10F01C20F01C20703C20705C40308C400F078014147E9318>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207E9F14>I< 0000780003F80000700000700000700000700000E00000E00000E00000E00001C00001C0 00F1C00389C00707800E03801C03803C0380380700780700780700780700F00E00F00E00 F00E00F00E10F01C20F01C20703C20705C40308C400F078015207E9F18>100 D<007C01C207010E011C013C013802780C7BF07C00F000F000F000F00070007001700230 04183807C010147E9315>I<00007C0000CE00019E00039E00030C000700000700000700 000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C00001C00001C00 00380000380000380000380000380000700000700000700000700000700000E00000E000 00E00000E00000C00001C000318000798000F300006200003C000017297E9F16>I<001E 3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E01C01E01C01E01 C01E01C01E03801E03800E07800E0B8006170001E700000700000700000E00000E00300E 00781C00F038006070003FC000151D809316>I<01E0000FE00001C00001C00001C00001 C000038000038000038000038000070000070000071F000761800E80C00F00C00E00E00E 00E01C01C01C01C01C01C01C01C0380380380380380380380704700708700E08700E1070 0610E006206003C016207E9F1A>I<00E001E001E000C000000000000000000000000000 000E00130023804380438043808700070007000E000E001C001C001C2038403840384038 8019000E000B1F7E9E10>I<0000C00001E00001E00001C0000000000000000000000000 000000000000000000001E00006300004380008380010380010380020700000700000700 000700000E00000E00000E00000E00001C00001C00001C00001C00003800003800003800 00380000700000700030700078E000F1C0006380003E00001328819E13>I<01E0000FE0 0001C00001C00001C00001C0000380000380000380000380000700000700000701E00706 100E08700E10F00E20F00E40601C80001D00001E00001FC000387000383800383800381C 20703840703840703840701880E01880600F0014207E9F18>I<03C01FC0038003800380 038007000700070007000E000E000E000E001C001C001C001C0038003800380038007000 700070007100E200E200E200E200640038000A207E9F0E>I<1E07C07C00231861860023 A032030043C0340300438038038043803803808700700700070070070007007007000700 7007000E00E00E000E00E00E000E00E00E000E00E01C101C01C01C201C01C038201C01C0 38401C01C0184038038018801801800F0024147E9328>I<1E07802318C023A06043C070 4380704380708700E00700E00700E00700E00E01C00E01C00E01C00E03821C03841C0704 1C07081C03083803101801E017147E931B>I<007C0001C3000301800E01C01E01C01C01 E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780F00700700F00700E003018 0018700007C00013147E9316>I<03C1E004621804741C08781C08701E08701E10E01E00 E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380700380E003C1C007238007 1E000700000700000E00000E00000E00000E00001C00001C0000FFC000171D819317>I< 1E1E0023210023C38043C7804387804383008700000700000700000700000E00000E0000 0E00000E00001C00001C00001C00001C000038000018000011147E9315>114 D<007C018203010603060706060E00078007F803FC01FE001F00077007F006F006E00440 0820301FC010147E9315>I<00C000E001C001C001C001C003800380FFF8038007000700 070007000E000E000E000E001C001C001C001C10382038203820384018800F000D1C7F9B 10>I<0F00601180702180E021C0E041C0E04380E08381C00701C00701C00701C00E0380 0E03800E03800E03840E07080C07080C07080E0F1006131003E1E016147E931A>I<0F01 801183C02183E021C1E041C0E04380608380400700400700400700400E00800E00800E00 800E01000E01000C02000E04000E040006180001E00013147E9316>I<0F006060118070 F02180E0F821C0E07841C0E0384380E0188381C0100701C0100701C0100701C0100E0380 200E0380200E0380200E0380400E0380400E0380800E078080060781000709860001F078 001D147E9321>I<03C1C00C62201034701038F02038F020386040700000700000700000 700000E00000E00000E00000E02061C040F1C040F1C080E2C080446300383C0014147E93 1A>I<0F00601180702180E021C0E041C0E04380E08381C00701C00701C00701C00E0380 0E03800E03800E03800E07000C07000C07000E0F00061E0003EE00000E00000E00001C00 78180078380070700060600021C0001F0000141D7E9316>I<01E02003F04007F8C00C1F 8008010000020000040000080000100000600000C0000100000200000400800801001003 003F060061FC0040F80080700013147E9315>I E /Fx 54 122 df<000FF07F00007FF9 FF8000F83FC7C001E07F8FC003E07F0FC007C07F0FC007C03F078007C01F000007C01F00 0007C01F000007C01F000007C01F0000FFFFFFF800FFFFFFF80007C01F000007C01F0000 07C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007 C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F00003FF8 FFF0003FF8FFF0002220809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C00 07C07C0007C07C0007C0380007C0000007C0000007C0000007C1FE00FFFFFE00FFFFFE00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E003FF9FFC03FF9FFC0 1A20809F1D>I<000FF03FFC00007FFDFFFC0000F83FE0FC0001E03F81FC0003E07F81FC 0007C07F00FC0007C03F00FC0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C00FFFFFFFFFC00FFFFFFFFFC0007C01F007C0007C01F007C0007C01F007C 0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C003FF8FFE3FF803FF8FFE3FF802920809F2C>15 D<387CFEFFFF7F3B0303 06060E0C18702008107C860F>44 DI<01FC0007FF00 1F07C01E03C03E03E07C01F07C01F07C01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8 FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F87C01F07C01F07C01F03E03E01E03C0 1F8FC007FF0001FC00151D7E9C1A>48 D<00E00001E0000FE000FFE000F3E00003E00003 E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003 E00003E00003E00003E00003E00003E00003E00003E00003E000FFFF80FFFF80111D7C9C 1A>I<07F0001FFE00383F007C1F80FE0FC0FE0FC0FE0FE0FE07E07C07E03807E0000FE0 000FC0000FC0001F80001F00003E0000780000F00000E00001C0000380600700600E0060 1C00E01FFFC03FFFC07FFFC0FFFFC0FFFFC0131D7D9C1A>I<01FC0007FF000E0F801E0F C03F07E03F07E03F07E03F07E01E0FC0000FC0000F80001F0001FC0001FC00000F800007 C00003E00003F00003F83803F87C03F8FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF 8003FC00151D7E9C1A>I<0001C00003C00007C00007C0000FC0001FC0003BC00073C000 63C000C3C00183C00383C00703C00E03C00C03C01803C03803C07003C0E003C0FFFFFEFF FFFE0007C00007C00007C00007C00007C00007C000FFFE00FFFE171D7F9C1A>I<380380 3FFF803FFF003FFE003FFC003FF0003F800030000030000030000030000033F80037FE00 3C1F00380F801007C00007C00007E00007E07807E0FC07E0FC07E0FC07E0FC07C0780FC0 600F80381F001FFC0007F000131D7D9C1A>I<003F0001FFC007E0E00F81E01F03F01E03 F03E03F07C03F07C01E07C0000FC1000FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01 F8FC01F8FC01F87C01F87C01F87C01F83C01F03E01F01E03E00F07C007FF8001FE00151D 7E9C1A>I<6000007FFFF87FFFF87FFFF07FFFE07FFFC0E00180C00300C00300C0060000 0C0000180000380000380000780000700000F00000F00001F00001F00001F00001F00003 F00003F00003F00003F00003F00003F00001E00000C000151E7D9D1A>I<01FC0007FF00 0F07801E03C01C01E03C01E03C01E03E01E03F01E03FC3C01FE3801FFF000FFE0007FF80 07FFC01FFFE03C3FF0780FF07803F8F001F8F000F8F00078F00078F000707800707C00E0 3E03C00FFF8003FC00151D7E9C1A>I<01FC000FFF001F07803E03C07C03E07C01E0FC01 F0FC01F0FC01F0FC01F8FC01F8FC01F8FC01F87C03F87C03F83E05F81FFDF807F9F80041 F80001F03C01F07E01F07E03E07E03E07E07C03C0780381F001FFC0007F000151D7E9C1A >I<387CFEFEFE7C38000000000000387CFEFEFE7C3807147C930F>I<07F8001FFE00381F 80780F80FC0FC0FC0FC0FC0FC0780FC0301F80001F00003E00007C0000700000E00000E0 0000C00000C00000C00000C00000C00000C00000000000000000000000000001C00003E0 0007F00007F00007F00003E00001C00012207D9F19>63 D<0000E000000000E000000001 F000000001F000000001F000000003F800000003F800000006FC00000006FC0000000EFE 0000000C7E0000000C7E000000183F000000183F000000303F800000301F800000701FC0 0000600FC00000600FC00000C007E00000FFFFE00001FFFFF000018003F000018003F000 030001F800030001F800060001FC00060000FC000E0000FE00FFE00FFFE0FFE00FFFE023 1F7E9E28>65 DI<0007FC02003F FF0E00FE03DE03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E00 00067E000006FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E00 00007E0000067F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000FE 01C0003FFF800007FC001F1F7D9E26>I69 DI<0007FC0200003FFF 0E0000FE03DE0003F000FE0007E0003E000FC0001E001F80001E003F00000E003F00000E 007F000006007E000006007E00000600FE00000000FE00000000FE00000000FE00000000 FE00000000FE003FFFE0FE003FFFE07E00007E007E00007E007F00007E003F00007E003F 00007E001F80007E000FC0007E0007E0007E0003F000FE0000FE01FE00003FFF8E000007 FC0600231F7D9E29>I73 D76 DI 80 D<03FC080FFF381E03F83800F8700078700038F00038F00018F00018F80000FC0000 7FC0007FFE003FFF801FFFE00FFFF007FFF000FFF80007F80000FC00007C00003CC0003C C0003CC0003CE00038E00078F80070FE01E0E7FFC081FF00161F7D9E1D>83 D<7FFFFFFC7FFFFFFC7C07E07C7007E01C6007E00C6007E00CE007E00EC007E006C007E0 06C007E006C007E0060007E0000007E0000007E0000007E0000007E0000007E0000007E0 000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0 000007E00003FFFFC003FFFFC01F1E7E9D24>I<07FC001FFF003F0F803F07C03F03E03F 03E00C03E00003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F803E0FC05E07E 0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC 0000FC0000FC0000FC0000FC00007C00007E00003E00603F00C01F81C007FF0001FC0013 147E9317>I<0007F80007F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F801F8F80FFEF81F83F83E01F87E00F87C00F87C00F8FC00F8FC00F8FC00F8 FC00F8FC00F8FC00F87C00F87C00F87E00F83E01F81F07F80FFEFF03F8FF18207E9F1D> I<01FE0007FF800F83C01E01E03E00F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC00 00FC00007C00007C00003E00181E00180F807007FFE000FF8015147F9318>I<001F8000 FFC001F3E003E7E003C7E007C7E007C3C007C00007C00007C00007C00007C000FFFC00FF FC0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007 C00007C00007C00007C00007C0003FFC003FFC0013207F9F10>I<01FC3C07FFFE0F079E 1E03DE3E03E03E03E03E03E03E03E03E03E01E03C00F07800FFF0009FC00180000180000 1C00001FFF800FFFF007FFF81FFFFC3C007C70003EF0001EF0001EF0001E78003C78003C 3F01F80FFFE001FF00171E7F931A>II<1C003E003F007F003F003E001C000000000000000000000000 00FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F 00FFE0FFE00B217EA00E>I<0038007C00FE00FE00FE007C003800000000000000000000 000001FE01FE003E003E003E003E003E003E003E003E003E003E003E003E003E003E003E 003E003E003E003E003E303E783EFC3CFC7C78783FF01FC00F2A83A010>IIIII<01FF0007FFC01F83F03E00F83E00F87C00 7C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C007C7C007C3E00F83E00F81F83 F007FFC001FF0017147F931A>II< 01F81807FE381F87783F01F83E01F87E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8 FC00F87C00F87E00F87E00F83F01F81F87F80FFEF803F8F80000F80000F80000F80000F8 0000F80000F80000F80007FF0007FF181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07FF83F FC1FFE03FE001FC007C007E007F006F81EFFFCC7F010147E9315>I<0180018001800380 0380038007800F803F80FFFCFFFC0F800F800F800F800F800F800F800F800F800F800F86 0F860F860F860F8607CC03F801F00F1D7F9C14>IIIIII E /Fy 37 122 df<00000007FF800000000001FFFFF0000000000FFFFFFC000000003FFFFFFE00 000000FFFC00FF00000001FFC0003F80000007FF00007FC000000FFE0001FFC000001FFC 0001FFE000001FF80003FFE000003FF00003FFE000003FF00003FFE000007FE00003FFE0 00007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00001FFC000007FE0 0000FF8000007FE000003E0000007FE00000000000007FE00000000000007FE000000000 00007FE00000000000007FE00000000000007FE00000000000007FE0003FFFF000FFFFFF FFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF0 00007FF00000FFF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFF E03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC007FFE00FFFE00FFFF00FFFF 00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E780000780000780000F00000F 00000F00000F00001E00001E00003C00003C0000780000F80000F00001F00003E00007C0 000F80001F00001E00000C00001124788F21>44 D<0000000003E0000000000000000000 07F000000000000000000007F00000000000000000000FF80000000000000000000FF800 00000000000000000FF80000000000000000001FFC0000000000000000001FFC00000000 00000000003FFE0000000000000000003FFE0000000000000000007FFF00000000000000 00007FFF0000000000000000007FFF000000000000000000FFFF800000000000000000FF FF800000000000000001FFFFC00000000000000001FFFFC00000000000000001FFFFC000 00000000000003FFFFE00000000000000003EFFFE00000000000000007EFFFF000000000 00000007CFFFF00000000000000007C7FFF0000000000000000FC7FFF800000000000000 0F83FFF8000000000000001F83FFFC000000000000001F03FFFC000000000000001F01FF FC000000000000003F01FFFE000000000000003E00FFFE000000000000007E00FFFF0000 00000000007C007FFF00000000000000FC007FFF80000000000000F8007FFF8000000000 0000F8003FFF80000000000001F8003FFFC0000000000001F0001FFFC0000000000003F0 001FFFE0000000000003E0001FFFE0000000000003E0000FFFE0000000000007E0000FFF F0000000000007C00007FFF000000000000FC00007FFF800000000000F800007FFF80000 0000000F800003FFF800000000001F800003FFFC00000000001F000001FFFC0000000000 3FFFFFFFFFFE00000000003FFFFFFFFFFE00000000007FFFFFFFFFFF00000000007FFFFF FFFFFF00000000007FFFFFFFFFFF0000000000FC0000007FFF8000000000F80000003FFF 8000000001F80000003FFFC000000001F00000003FFFC000000001F00000001FFFC00000 0003F00000001FFFE000000003E00000000FFFE000000007E00000000FFFF000000007C0 0000000FFFF000000007C000000007FFF00000000FC000000007FFF80000000F80000000 03FFF80000001F8000000003FFFC0000001F0000000003FFFC0000007FC000000001FFFC 0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FF FFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF8051487CC75A>65 DI<00 0000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E000003FFFFFFFE00FE00 000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001FFFC000003FFFE00 07FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003FFF000000001FFE00 7FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FFF00000000003FE03 FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFFC00000000000FE0F FFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF8000000000007E3F FF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF0000000000003E7F FF000000000000007FFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF000000000000007FFF000000000000007F FF000000000000007FFF000000000000007FFF8000000000003E3FFF8000000000003E3F FF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFFC000000000003E0F FFC000000000007C0FFFC000000000007C07FFE000000000007C07FFE00000000000F803 FFF00000000000F803FFF00000000001F801FFF80000000001F000FFFC0000000003E000 7FFE0000000007E0003FFF000000000FC0001FFF800000001F80000FFFC00000003F0000 07FFF0000000FE000001FFFC000001FC000000FFFF80000FF80000003FFFF8007FF00000 000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000000FFFFFE0000000 0000003FFE000000474979C756>I69 DI<000000003FFE 00000E000000000FFFFFC0001E000000007FFFFFF8003E00000003FFFFFFFE00FE000000 0FFFFFFFFF81FE0000003FFFF800FFC3FE000000FFFF80000FF7FE000001FFFC000003FF FE000007FFF0000001FFFE00000FFFC00000007FFE00001FFF800000003FFE00003FFF00 0000001FFE00007FFE000000000FFE0000FFFC0000000007FE0001FFF80000000007FE00 03FFF00000000003FE0003FFF00000000001FE0007FFE00000000001FE0007FFE0000000 0000FE000FFFC00000000000FE000FFFC000000000007E001FFFC000000000007E001FFF 8000000000007E003FFF8000000000007E003FFF8000000000003E003FFF800000000000 3E007FFF8000000000003E007FFF0000000000003E007FFF00000000000000007FFF0000 000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000 FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF00000000 00000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF 0000000000000000FFFF0000000000000000FFFF0000007FFFFFFFFE7FFF0000007FFFFF FFFE7FFF0000007FFFFFFFFE7FFF8000007FFFFFFFFE7FFF8000007FFFFFFFFE3FFF8000 000000FFFE003FFF8000000000FFFE003FFF8000000000FFFE001FFF8000000000FFFE00 1FFFC000000000FFFE000FFFC000000000FFFE000FFFC000000000FFFE0007FFE0000000 00FFFE0007FFE000000000FFFE0003FFF000000000FFFE0003FFF800000000FFFE0001FF F800000000FFFE0000FFFC00000000FFFE00007FFE00000000FFFE00003FFF00000000FF FE00001FFF80000000FFFE00000FFFE0000001FFFE000007FFF0000003FFFE000001FFFC 000007FFFE000000FFFF80001FFFFE0000003FFFFC00FFC7FE0000000FFFFFFFFF83FE00 000003FFFFFFFE00FE000000007FFFFFF8003E000000000FFFFFE0000E00000000003FFE 000000004F4979C75D>I73 D76 DI80 D<00007FF00007000007FFFF000F00001FFFFFC01F00007FFFFFF03F0000 FFFFFFFC7F0003FFC00FFEFF0007FE0000FFFF0007FC00003FFF000FF000001FFF001FF0 000007FF001FE0000003FF003FE0000001FF003FC0000001FF007FC0000000FF007FC000 00007F007FC00000007F00FFC00000003F00FFC00000003F00FFC00000003F00FFE00000 003F00FFE00000001F00FFF00000001F00FFF80000001F00FFFC0000001F00FFFF000000 00007FFFC0000000007FFFFE000000007FFFFFE00000003FFFFFFF0000003FFFFFFFF000 001FFFFFFFFC00001FFFFFFFFF00000FFFFFFFFFC00007FFFFFFFFE00007FFFFFFFFF000 03FFFFFFFFF80000FFFFFFFFFC00007FFFFFFFFE00003FFFFFFFFE00000FFFFFFFFF0000 01FFFFFFFF0000003FFFFFFF80000001FFFFFF800000000FFFFFC000000000FFFFC00000 00001FFFC0000000000FFFE00000000007FFE00000000003FFE07800000001FFE0F80000 0000FFE0F800000000FFE0F8000000007FE0F8000000007FE0F8000000007FE0FC000000 007FE0FC000000007FC0FC000000007FC0FE000000007FC0FE000000007F80FF00000000 FF80FF80000000FF80FFC0000000FF00FFE0000001FE00FFF8000003FE00FFFE000007FC 00FFFF80000FF800FFFFFC007FF000FE3FFFFFFFE000FC0FFFFFFF8000F803FFFFFF0000 F0007FFFF80000E00003FFC00000334979C742>83 D<3FFFFFFFFFFFFFFFFF003FFFFFFF FFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF00 3FFF0003FFF8003FFF007FF80003FFF80007FF807FE00003FFF80001FF807FC00003FFF8 0000FF807F800003FFF800007F807F000003FFF800003F807F000003FFF800003F807E00 0003FFF800001F807E000003FFF800001F807E000003FFF800000F807C000003FFF80000 0F807C000003FFF800000F807C000003FFF800000F807C000003FFF800000F80FC000003 FFF800000FC0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007C0 F8000003FFF8000007C0F8000003FFF8000007C000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF8000000000003FFFFFFFFFFF800000003 FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF8 00004A467CC553>I<0007FFFC000000007FFFFFC0000001FFFFFFF8000003FFFFFFFE00 0007FE001FFF000007FF0003FFC0000FFF8001FFE0000FFF8000FFF0000FFF80007FF000 0FFF80007FF8000FFF80007FF80007FF00003FFC0007FF00003FFC0003FE00003FFC0000 F800003FFC00000000003FFC00000000003FFC00000000003FFC00000000003FFC000000 07FFFFFC000000FFFFFFFC000007FFFFFFFC00003FFFE03FFC0000FFFE003FFC0003FFF0 003FFC0007FFC0003FFC000FFF00003FFC001FFE00003FFC003FFC00003FFC007FF80000 3FFC007FF800003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000003F FC00FFF000003FFC00FFF000007FFC007FF80000FFFC007FF80001EFFC003FFC0003EFFC 003FFF0007CFFF000FFFC03F8FFFF807FFFFFF07FFFC01FFFFFC03FFFC007FFFF001FFFC 0003FF80007FF8362E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC00000 000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC00FFC00000001FFC07FFFC0000001FFC3FFFFF0000001FFCFFFFF FC000001FFDFF00FFF000001FFFF8003FF800001FFFE0001FFC00001FFF800007FE00001 FFF000007FF00001FFE000003FF80001FFE000001FFC0001FFE000001FFC0001FFE00000 1FFE0001FFE000001FFE0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001 FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000000FFF0001FFE00000 0FFF0001FFE000001FFE0001FFE000001FFE0001FFE000001FFC0001FFE000001FFC0001 FFE000003FF80001FFF000003FF80001FFF800007FF00001FFFC0000FFE00001FFFE0001 FFC00001FFBF0007FF800001FF1FE01FFE000001FE0FFFFFFC000001FC03FFFFF0000001 F800FFFF80000001F0001FF800000039487CC742>I<00001FFFC0000000FFFFF8000007 FFFFFE00001FFFFFFF80007FFC00FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF 0003FFE00FFE0003FFE00FFE0003FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC 00003E007FF8000000007FF8000000007FF800000000FFF800000000FFF800000000FFF8 00000000FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF8 00000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC000000003FFC 000000001FFC000000F81FFE000000F80FFE000000F80FFF000001F007FF800003F003FF C00007E001FFE0000FC000FFF0001F80007FFE00FF00001FFFFFFE000007FFFFF8000000 FFFFE00000001FFE00002D2E7CAD35>I<00000000007FC00000000000FFFFC000000000 00FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC00000000FFC01FFC0000000FFFF81FFC0000007FFFFE1FFC000001FFF FFF9FFC000007FFC03FFFFC00000FFF0007FFFC00001FFC0001FFFC00003FF80000FFFC0 0007FF000007FFC0000FFE000003FFC0000FFE000003FFC0001FFC000003FFC0001FFC00 0003FFC0003FFC000003FFC0003FFC000003FFC0007FF8000003FFC0007FF8000003FFC0 007FF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF800 0003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC0 00FFF8000003FFC000FFF8000003FFC0007FF8000003FFC0007FF8000003FFC0007FF800 0003FFC0003FF8000003FFC0003FFC000003FFC0003FFC000003FFC0001FFC000003FFC0 001FFC000003FFC0000FFE000007FFC00007FF00000FFFC00003FF00001FFFC00001FFC0 003FFFC00000FFE000FFFFE000007FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FFFF 800001FFFF03FFFF8000001FF803FFFF8039487CC742>I<00001FFE00000001FFFFE000 0007FFFFF800001FFFFFFE00007FFC07FF0000FFE001FF8001FFC0007FC003FF80003FE0 07FF00003FF00FFE00001FF01FFE00000FF81FFC00000FF83FFC00000FFC3FFC000007FC 7FFC000007FC7FF8000007FC7FF8000007FE7FF8000007FEFFF8000007FEFFF8000007FE FFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFCFFF800000000FFF800000000 FFF800000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC00000000 3FFC000000003FFC0000001C1FFE0000003E0FFE0000003E07FF0000007E07FF000000FC 03FF800001F801FFC00003F0007FF0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE00 0000FFFFF80000000FFF80002F2E7DAD36>I<000000FFC000000007FFF80000003FFFFC 000000FFFFFF000001FFC1FF000007FF03FF80000FFC03FF80000FF807FFC0001FF807FF C0003FF007FFC0003FF007FFC0003FE003FF80007FE003FF80007FE001FF00007FE000FE 00007FE0003800007FE0000000007FE0000000007FE0000000007FE0000000007FE00000 00007FE0000000007FE0000000007FE0000000007FE0000000007FE0000000FFFFFFFE00 00FFFFFFFE0000FFFFFFFE0000FFFFFFFE0000FFFFFFFE0000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF00000003FFFFFF800003FFFFFF800 003FFFFFF800003FFFFFF800003FFFFFF800002A487DC724>I<00000000001F8000007F F000FFE00007FFFF03FFF0001FFFFFC7FFF0007FFFFFFFC7F800FFE03FFE0FF801FF800F FC0FF803FF0007FE0FF807FE0003FF07F007FE0003FF07F00FFC0001FF81C00FFC0001FF 80000FFC0001FF80001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0 001FFC0001FFC0001FFC0001FFC0000FFC0001FF80000FFC0001FF80000FFC0001FF8000 07FE0003FF000007FE0003FF000003FF0007FE000001FF800FFC000000FFE03FF8000001 FFFFFFF0000001DFFFFFC0000003C7FFFF00000003C07FF000000007C0000000000007C0 000000000007C0000000000007C0000000000007E0000000000007F0000000000007F800 0000000007FFFFFFF0000007FFFFFFFF000003FFFFFFFFE00003FFFFFFFFF80001FFFFFF FFFE0001FFFFFFFFFF0000FFFFFFFFFF80007FFFFFFFFF8003FFFFFFFFFFC00FFFFFFFFF FFC01FF800001FFFE03FE0000001FFE07FC00000007FF07FC00000003FF0FF800000001F F0FF800000001FF0FF800000001FF0FF800000001FF0FF800000001FF07FC00000003FE0 7FC00000003FE03FE00000007FC03FF0000000FFC01FFC000003FF800FFF00000FFF0003 FFF000FFFC0000FFFFFFFFF000003FFFFFFFC0000007FFFFFE000000003FFFC000003544 7DAE3B>I<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC000000000 00FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 01FFC0000001FFC00FFFF8000001FFC03FFFFE000001FFC0FFFFFF000001FFC1FC07FF80 0001FFC3E003FFC00001FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC 0000FFE00001FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF0 0001FFF00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 00FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFF C07FFFFFE03B487CC742>I<00FC0001FF0003FF8007FFC00FFFC01FFFE01FFFE01FFFE0 1FFFE01FFFE01FFFE00FFFC007FFC003FF8001FF0000FC00000000000000000000000000 000000000000000000000000000000000000000000007FC0FFFFC0FFFFC0FFFFC0FFFFC0 FFFFC003FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 01FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 01FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF18497CC820>I<007FC000000000FFFFC000000000 FFFFC000000000FFFFC000000000FFFFC000000000FFFFC00000000003FFC00000000001 FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FF C00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC0 0000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000 00000001FFC00000000001FFC00000000001FFC00000000001FFC0007FFFF801FFC0007F FFF801FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0000FFE0001FFC00007F0 0001FFC0000FE00001FFC0003FC00001FFC0007F800001FFC000FE000001FFC001FC0000 01FFC007F8000001FFC00FF0000001FFC01FC0000001FFC03F80000001FFC0FF00000001 FFC1FE00000001FFC3FF00000001FFCFFF80000001FFDFFFC0000001FFFFFFC0000001FF FFFFE0000001FFFFFFF0000001FFFCFFF8000001FFF87FFC000001FFE03FFC000001FFC0 1FFE000001FFC01FFF000001FFC00FFF800001FFC007FFC00001FFC003FFC00001FFC001 FFE00001FFC001FFF00001FFC000FFF80001FFC0007FFC0001FFC0003FFC0001FFC0001F FE0001FFC0000FFF0001FFC0000FFF8001FFC0000FFFC0FFFFFF807FFFFFFFFFFF807FFF FFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FFFFC000FFFFC000FFFFC00003FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC000FFFFFF80FFFFFF80FFFFFF80FFFFFF80FFFFFF 8019487CC720>I<007FC001FFC00000FFE00000FFFFC00FFFF80007FFFC0000FFFFC03F FFFE001FFFFF0000FFFFC0FFFFFF007FFFFF8000FFFFC1FC07FF80FE03FFC000FFFFC3E0 03FFC1F001FFE00003FFC7C001FFC3E000FFE00001FFCF0001FFE78000FFF00001FFDE00 00FFEF00007FF00001FFDC0000FFEE00007FF00001FFFC0000FFFE00007FF80001FFF800 00FFFC00007FF80001FFF00000FFF800007FF80001FFF00000FFF800007FF80001FFF000 00FFF800007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800FFFFFFC0 7FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC0 7FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF05C2E7CAD63>I<007FC001FFC00000FF FFC00FFFF80000FFFFC03FFFFE0000FFFFC0FFFFFF0000FFFFC1FC07FF8000FFFFC3E003 FFC00003FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001 FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF00001FFF00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFFC07F FFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B 2E7CAD42>I<00000FFF0000000000FFFFF000000007FFFFFE0000001FFFFFFF8000003F FC03FFC00000FFE0007FF00001FF80001FF80003FF00000FFC0007FE000007FE000FFE00 0007FF000FFC000003FF001FFC000003FF803FFC000003FFC03FF8000001FFC03FF80000 01FFC07FF8000001FFE07FF8000001FFE07FF8000001FFE0FFF8000001FFF0FFF8000001 FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FF F0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF07FF8000001FFE07FF8000001FFE0 7FF8000001FFE07FF8000001FFE03FFC000003FFC03FFC000003FFC01FFC000003FF801F FE000007FF800FFE000007FF0007FF00000FFE0003FF80001FFC0001FFC0003FF80000FF E0007FF000007FFC03FFE000001FFFFFFF80000007FFFFFE00000000FFFFF0000000000F FF000000342E7DAD3B>I<007FC00FFC000000FFFFC07FFFC00000FFFFC3FFFFF00000FF FFCFFFFFFC0000FFFFDFF01FFF0000FFFFFF8007FF800003FFFE0001FFC00001FFF80000 FFE00001FFF00000FFF00001FFE000007FF80001FFE000003FFC0001FFE000003FFC0001 FFE000003FFE0001FFE000001FFE0001FFE000001FFF0001FFE000001FFF0001FFE00000 1FFF0001FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000001FFF0001 FFE000001FFF0001FFE000001FFE0001FFE000001FFE0001FFE000003FFC0001FFE00000 3FFC0001FFE000007FF80001FFF000007FF80001FFF80000FFF00001FFFC0001FFE00001 FFFE0003FFC00001FFFF0007FF800001FFFFE03FFE000001FFEFFFFFFC000001FFE3FFFF F0000001FFE0FFFF80000001FFE01FF800000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE00000 00000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE00000000000FFFFFFC000 000000FFFFFFC000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC00000000039 427CAD42>I<00FF803F8000FFFF80FFF000FFFF83FFFC00FFFF87FFFE00FFFF8FC3FF00 FFFF8F07FF0003FF9E0FFF8001FFBC0FFF8001FFB80FFF8001FFF80FFF8001FFF00FFF80 01FFF007FF0001FFF007FF0001FFE003FE0001FFE000F80001FFE000000001FFE0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000292E7CAD31> 114 D<000FFF00E0007FFFF3E001FFFFFFE007FFFFFFE00FF800FFE01FC0001FE03F8000 0FE03F000007E07F000003E07F000003E0FF000003E0FF000003E0FF800003E0FFC00000 00FFF0000000FFFE000000FFFFF800007FFFFFC0007FFFFFF0003FFFFFFC001FFFFFFF00 0FFFFFFF8007FFFFFFC003FFFFFFE000FFFFFFF0003FFFFFF00003FFFFF800001FFFF800 0000FFFC0000001FFC7800000FFCF8000007FCF8000003FCFC000003FCFC000003FCFE00 0003F8FE000003F8FF000003F8FF800007F0FFC0000FF0FFF0001FE0FFFC00FFC0FFFFFF FF80FC7FFFFE00F81FFFF800E003FF8000262E7CAD2F>I<0001F000000001F000000001 F000000001F000000001F000000001F000000003F000000003F000000003F000000007F0 00000007F000000007F00000000FF00000000FF00000001FF00000003FF00000003FF000 00007FF0000001FFF0000003FFF000000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC0 FFFFFFFFC000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000 FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0 000000FFF0000000FFF0000000FFF001F000FFF001F000FFF001F000FFF001F000FFF001 F000FFF001F000FFF001F000FFF001F000FFF001F0007FF001E0007FF803E0003FF803E0 003FFC07C0001FFE0F80000FFFFF800007FFFE000001FFFC0000001FF00024427EC12E> I<007FE000003FF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0 007FFFF000FFFFE0007FFFF00003FFE00001FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00001FFF00001FFE00001FFF00001FFE00001FFF0 0001FFE00003FFF00000FFE00007FFF00000FFE0000F7FF000007FE0001F7FF000007FF0 003E7FF800003FFC00FC7FFFE0001FFFFFF87FFFE00007FFFFE07FFFE00001FFFF807FFF E000003FFE007FFFE03B2E7CAD42>III<7FFFFF801FFFFF007FFFFF801FFFFF00 7FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF00007FF80001FF0000007FFC00 01FC0000003FFE0003F80000001FFF0007F00000000FFF0007E000000007FF800FC00000 0003FFC01F8000000003FFE03F8000000001FFF07F0000000000FFF8FE00000000007FF9 FC00000000003FFFF800000000003FFFF000000000001FFFE000000000000FFFC0000000 000007FFC0000000000003FFC0000000000001FFE0000000000001FFF0000000000001FF F8000000000003FFFC000000000003FFFE000000000007FFFE00000000000FEFFF000000 00001FCFFF80000000003F87FFC0000000007F03FFE000000000FE01FFE000000001FC00 FFF000000001F8007FF800000003F0007FFC00000007F0003FFE0000000FE0001FFF0000 001FC0000FFF0000003F800007FF800000FF800007FFC000FFFFF8003FFFFFC0FFFFF800 3FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC03A2E7EAD3F>I<7F FFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000 FFFF8000FFF000000FE00000FFF800000FC00000FFF800000FC000007FFC00000F800000 7FFC00001F8000003FFC00001F0000003FFE00003F0000001FFE00003E0000001FFF0000 7E0000000FFF00007C0000000FFF8000FC00000007FF8000F800000007FFC001F8000000 03FFC001F000000003FFE003F000000003FFE003F000000001FFF003E000000001FFF007 E000000000FFF007C000000000FFF80FC0000000007FF80F80000000007FFC1F80000000 003FFC1F00000000003FFE3F00000000001FFE3E00000000001FFF7E00000000000FFF7C 00000000000FFFFC00000000000FFFFC000000000007FFF8000000000007FFF800000000 0003FFF0000000000003FFF0000000000001FFE0000000000001FFE0000000000000FFC0 000000000000FFC00000000000007F800000000000007F800000000000003F0000000000 00003F000000000000003F000000000000003E000000000000007E000000000000007C00 000000000000FC000000001F8000F8000000003FC001F8000000007FE001F000000000FF F003F000000000FFF003E000000000FFF007E000000000FFF00FC000000000FFF01F8000 000000FFF03F80000000007FE07F00000000007F43FE00000000003FFFF800000000001F FFF0000000000007FFC0000000000001FE00000000000039427EAD3F>I E /Fz 10 58 df<1F00318060C04040C060C060C060C060C060C060C060C060404060C0 31801F000B107F8F0F>48 D<0C003C00CC000C000C000C000C000C000C000C000C000C00 0C000C000C00FF8009107E8F0F>I<1F00618040C08060C0600060006000C00180030006 000C00102020207FC0FFC00B107F8F0F>I<1F00218060C060C000C0008001800F000080 00400060C060C060804060801F000B107F8F0F>I<0300030007000F000B001300330023 004300C300FFE003000300030003001FE00B107F8F0F>I<20803F002C00200020002000 2F0030802040006000600060C06080C061801F000B107F8F0F>I<0780184030C060C060 00C000CF00F080E040C060C060C060406060C030801F000B107F8F0F>I<40007FE07FC0 8080808001000200040004000C0008000800180018001800180018000B117E900F>I<1F 00318060C060C060C071803F000F00338061C0C060C060C060404060801F000B107F8F0F >I<1F00318060C0C040C060C060C06040E021E01E600060004060C0608043003E000B10 7F8F0F>I E /FA 14 104 df0 D<70F8F8F87005057C8D0D>I<400004C0000C6000183000301800600C00C00601800303 0001860000CC0000780000300000300000780000CC000186000303000601800C00C01800 60300030600018C0000C40000416187A9623>I<01800180018001800180C183F18F399C 0FF003C003C00FF0399CF18FC1830180018001800180018010147D9417>I<0001FE0000 0007FF8000001E01E000007800780000E0001C000180000600030000030006000001800C 000000C00C000000C0180000006030000000303000000030300000003060000000186000 0000186000000018C00000000CC00000000CC00000000CC00000000CC00000000CC00000 000CC00000000CC00000000CC00000000C60000000186000000018600000001830000000 303000000030300000003018000000600C000000C00C000000C006000001800300000300 018000060000E0001C000078007800001E01E0000007FF80000001FE0000262B7DA02D> 13 D<03C00FF01C38300C60066006C003C003C003C003C003C00360066006300C1C380F F003C010127D9317>I<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF7FFE 7FFE3FFC1FF80FF003C010127D9317>I<000000C0000003C000000F0000003C000000F0 000003C00000070000001C00000078000001E00000078000001E00000078000000E00000 00780000001E0000000780000001E0000000780000001C0000000700000003C0000000F0 0000003C0000000F00000003C0000000C000000000000000000000000000000000000000 0000000000000000007FFFFF80FFFFFFC01A247C9C23>20 DI<003FF800 FFF803C0000700000C0000180000300000300000600000600000C00000C00000C00000FF FFF8FFFFF8C00000C00000C000006000006000003000003000001800000C000007000003 C00000FFF8003FF8151C7C981E>50 D<00000C00000C0000180000180000300000300000 600000600000C00000C0000180000180000180000300000300000600000600000C00000C 0000180000180000300000300000600000600000C00000C0000180000180000300000300 000600000600000600000C00000C0000180000180000300000300000600000600000C000 00400000162C7AA000>54 D<400002C00006C00006C00006C00006C00006C00006C00006 C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006 C0000660000C60000C3000181C00700F01E003FF8000FE00171C7D9A1E>91 D<000F0038006000E001C001C001C001C001C001C001C001C001C001C001C001C001C001 C001C0038007001E00F8001E000700038001C001C001C001C001C001C001C001C001C001 C001C001C001C001C001C000E000600038000F102D7DA117>102 DI E /FB 89 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8F870000000000070F8F8F87005 1C779B18>33 D<4010E038F078E038E038E038E038E038E038E038E038E038E03860300D 0E7B9C18>I<030600078F00078F00078F00078F00078F00078F007FFFC0FFFFE0FFFFE0 7FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFFC0FFFFE0FFFFE07FFFC01E3C00 1E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18>I<3803007C07807C0780EE0F 80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C007C3C00387C0000780000780000F8 0000F00001F00001E00001E00003E00003C00003C00007C0000783800787C00F87C00F0E E00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07C018038013247E9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C38001C73F01C73F01CE3F00FE3 800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00E0F800E0F800E07070E078 7070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C7C7E3E0E0E0E1C1C38F8F0C0070E 789B18>I<007000F001E003C007800F001E001C00380038007000700070007000E000E0 00E000E000E000E000E000E0007000700070007000380038001C001E000F00078003C001 F000F000700C24799F18>I<6000F00078003C001E000F000780038001C001C000E000E0 00E000E00070007000700070007000700070007000E000E000E000E001C001C003800780 0F001E003C007800F00060000C247C9F18>I<01C00001C00001C00001C000C1C180F1C7 80F9CF807FFF001FFC0007F00007F0001FFC007FFF00F9CF80F1C780C1C18001C00001C0 0001C00001C00011147D9718>I<00600000F00000F00000F00000F00000F00000F00000 F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F00000F00000F00000F00000F00000 600013147E9718>I<1C3E7E7F3F1F070E1E7CF860080C788518>I<7FFF00FFFF80FFFF80 7FFF0011047D8F18>I<3078FCFC78300606778518>I<000300000780000780000F80000F 00001F00001E00001E00003E00003C00007C0000780000780000F80000F00001F00001E0 0003E00003C00003C00007C0000780000F80000F00000F00001F00001E00003E00003C00 003C00007C0000780000F80000F00000F0000060000011247D9F18>I<01F00007FC000F FE001F1F001C07003803807803C07001C07001C0E000E0E000E0E000E0E000E0E000E0E0 00E0E000E0E000E0E000E0F001E07001C07001C07803C03803801C07001F1F000FFE0007 FC0001F000131C7E9B18>I<01800380038007800F803F80FF80FB804380038003800380 03800380038003800380038003800380038003800380038003807FFCFFFE7FFC0F1C7B9B 18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0F000E06000E00000E00000E0 0001C00001C00003C0000780000F00001E00003C0000780000F00001E00007C0000F8000 1E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8001FFE003FFF007807807803 C07801C03001C00001C00003C0000380000F0003FF0003FE0003FF000007800003C00001 C00000E00000E00000E0F000E0F000E0F001C0F003C07C07803FFF001FFE0003F800131C 7E9B18>I<001F00003F0000770000770000E70001E70001C7000387000787000707000E 07001E07003C0700380700780700F00700FFFFF8FFFFF8FFFFF800070000070000070000 0700000700000700007FF000FFF8007FF0151C7F9B18>I<1FFF803FFF803FFF80380000 3800003800003800003800003800003800003800003BF8003FFE003FFF003C07801803C0 0001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F803FFF001FFC0003F000 131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03C03801803800007000007000 00E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000E0F000E07000E07000E07000 E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18>II<03F8000FFE001FFF003E0F803803807001C07001C07001C0 7001C03803803C07801FFF0007FC000FFE001F1F003C07807001C0F001E0E000E0E000E0 E000E0E000E07001C07803C03E0F801FFF000FFE0003F800131C7E9B18>I<03F0000FFC 001FFE003C0F00780780700380E001C0E001C0E001C0E001E0E001E07001E07803E03C0F E01FFFE00FFEE003F0E00000E00001C00001C00001C0300380780780780F00783E003FFC 001FF00007C000131C7E9B18>I<3078FCFC783000000000000000003078FCFC78300614 779318>I<183C7E7E3C180000000000000000183C7E7E3E1E0E1C3C78F060071A789318> I<000300000780001F80003F00007E0001FC0003F00007E0001FC0003F00007E0000FC00 00FC00007E00003F00001FC00007E00003F00001FC00007E00003F00001F800007800003 0011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE0000000000000000000000000FFFFE0FF FFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00007E00003F00001FC00007E000 03F00001FC00007E00003F00001F80001F80003F00007E0001FC0003F00007E0001FC000 3F00007E0000FC0000F0000060000011187D9918>I<007C0001FE0007FF000F87801E03 C03C1DC0387FC070FFE071E3E071C1E0E1C1E0E380E0E380E0E380E0E380E0E380E0E380 E0E1C1C071C1C071E3C070FF80387F003C1C001E00E00F83E007FFC001FF80007E00131C 7E9B18>64 D<00700000F80000F80000D80000D80001DC0001DC0001DC00018C00038E00 038E00038E00038E000306000707000707000707000707000FFF800FFF800FFF800E0380 0E03801C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>II<00F8E003FEE007FFE00F07E01E03E03C01E03800E07000E07000E070 0000E00000E00000E00000E00000E00000E00000E00000E000007000007000E07000E038 00E03C00E01E01C00F07C007FF8003FE0000F800131C7E9B18>I<7FF800FFFE007FFF00 1C0F801C03C01C03C01C01E01C00E01C00E01C00F01C00701C00701C00701C00701C0070 1C00701C00701C00701C00F01C00E01C00E01C01E01C01C01C03C01C0F807FFF00FFFE00 7FF800141C7F9B18>III<01F1C003FDC00FFFC01F0FC01C03C03803C03801C07001C0 7001C0700000E00000E00000E00000E00000E00000E00FF0E01FF0E00FF07001C07001C0 7003C03803C03803C01C07C01F0FC00FFFC003FDC001F1C0141C7E9B18>I<7F07F0FF8F F87F07F01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01FFFC01FFF C01FFFC01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C07F07 F0FF8FF87F07F0151C7F9B18>I<7FFF00FFFF807FFF0001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C0007FFF00FFFF807FFF00111C7D9B18>I<7F07F0 FF87F87F07F01C03C01C07801C07001C0E001C1E001C3C001C38001C70001CF0001DF000 1DF0001FB8001FB8001F1C001E1C001C0E001C0E001C07001C07001C03801C03801C01C0 7F03F0FF87F87F03F0151C7F9B18>75 D<7FE000FFE0007FE0000E00000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E 00000E00000E00700E00700E00700E00700E00707FFFF0FFFFF07FFFF0141C7F9B18>I< FC01F8FE03F8FE03F83B06E03B06E03B06E03B06E03B8EE03B8EE0398CE0398CE039DCE0 39DCE039DCE038D8E038D8E038F8E03870E03870E03800E03800E03800E03800E03800E0 3800E0FE03F8FE03F8FE03F8151C7F9B18>I<7E07F0FF0FF87F07F01D81C01D81C01D81 C01DC1C01CC1C01CC1C01CE1C01CE1C01CE1C01C61C01C71C01C71C01C31C01C39C01C39 C01C39C01C19C01C19C01C1DC01C0DC01C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18 >I<0FF8003FFE007FFF00780F00700700F00780E00380E00380E00380E00380E00380E0 0380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380F0078070 0700780F007FFF003FFE000FF800111C7D9B18>II<0FF8003FFE007FFF00780F00700700F00780E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380E00380E00380E00380E00380E1E380E1E380F0E7 8070F700787F007FFF003FFE000FFC00001C00001E00000E00000F000007000007001122 7D9B18>I<7FF800FFFE007FFF001C0F801C03801C03C01C01C01C01C01C01C01C03C01C 03801C0F801FFF001FFE001FFE001C0F001C07001C03801C03801C03801C03801C03801C 039C1C039C1C039C7F01F8FF81F87F00F0161C7F9B18>I<03F3801FFF803FFF807C0F80 700780E00380E00380E00380E000007000007800003F00001FF00007FE0000FF00000F80 0003C00001C00000E00000E06000E0E000E0E001E0F001C0F80780FFFF80FFFE00E7F800 131C7E9B18>I<7FFFF8FFFFF8FFFFF8E07038E07038E07038E070380070000070000070 000070000070000070000070000070000070000070000070000070000070000070000070 0000700000700000700007FF0007FF0007FF00151C7F9B18>IIII<7F8FE07F9FE07F8FE00E07000F0700070E00078E00039C0003 DC0001F80001F80000F00000F00000700000F00000F80001F80001DC00039E00038E0007 0F000707000E07800E03801E03C07F07F0FF8FF87F07F0151C7F9B18>II<3FFFE07FFFE07FFFE07001C07003C0700780700700000F 00001E00001C00003C0000780000700000F00001E00001C00003C0000780000700000F00 001E00E01C00E03C00E07800E07000E0FFFFE0FFFFE0FFFFE0131C7E9B18>II<600000F00000F00000F800007800007C00003C00003C00003E00001E00001F0000 0F00000F00000F800007800007C00003C00003C00003E00001E00001F00000F00000F800 007800007800007C00003C00003E00001E00001E00001F00000F00000F80000780000780 00030011247D9F18>II<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C38070E789E18>I<1FE0003FF8007FFC00781E00300E 0000070000070000FF0007FF001FFF007F0700780700E00700E00700E00700F00F00781F 003FFFF01FFBF007E1F014147D9318>I<7E0000FE00007E00000E00000E00000E00000E 00000E00000E3E000EFF800FFFC00FC1E00F80E00F00700E00700E00380E00380E00380E 00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF80063E00151C809B18>I< 01FE0007FF001FFF803E0780380300700000700000E00000E00000E00000E00000E00000 E000007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80003F 80001F8000038000038000038000038000038003E3800FFB801FFF803C1F80380F807007 80700380E00380E00380E00380E00380E00380E00380700780700780380F803C1F801FFF F00FFBF803E3F0151C7E9B18>I<01F00007FC001FFE003E0F00380780700380700380E0 01C0E001C0FFFFC0FFFFC0FFFFC0E000007000007001C03801C03E03C01FFF8007FF0001 FC0012147D9318>I<001F80007FC000FFE000E1E001C0C001C00001C00001C0007FFFC0 FFFFC0FFFFC001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C0007FFF007FFF007FFF00131C7F9B18>I<01E1F007FFF80FFF F81E1E301C0E003807003807003807003807003807001C0E001E1E001FFC001FF80039E0 003800001C00001FFE001FFFC03FFFE07801F0700070E00038E00038E00038E000387800 F07E03F01FFFC00FFF8001FC00151F7F9318>I<7E0000FE00007E00000E00000E00000E 00000E00000E00000E3E000EFF800FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B 18>I<03800007C00007C00007C0000380000000000000000000000000007FC000FFC000 7FC00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C000FFFF00FFFF80FFFF00111D7C9C18>I<0038007C007C007C003800 000000000000000FFC1FFC0FFC001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C6038F078FFF07FE03F800E277E9C18 >II<7FE000FFE0007FE00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E0007FFFC0FFFFE07FFFC0131C7E 9B18>I<7CE0E000FFFBF8007FFFF8001F1F1C001E1E1C001E1E1C001C1C1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 7F1F1F00FFBFBF807F1F1F001914819318>I<7E3E00FEFF807FFFC00FC1C00F80E00F00 E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3 FCFFE7FE7FC3FC1714809318>I<01F0000FFE001FFF003E0F803803807001C07001C0E0 00E0E000E0E000E0E000E0E000E0F001E07001C07803C03C07803E0F801FFF000FFE0001 F00013147E9318>I<7E3E00FEFF807FFFC00FC1E00F80E00F00700E00700E00380E0038 0E00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF800E3E000E0000 0E00000E00000E00000E00000E00000E00007FC000FFE0007FC000151E809318>I<01E3 8007FB801FFF803E1F80380F80700780700780E00380E00380E00380E00380E00380E003 80700780700780380F803C1F801FFF800FFB8003E3800003800003800003800003800003 80000380000380003FF8003FF8003FF8151E7E9318>I<7F87E0FF9FF07FBFF803F87803 F03003E00003C00003C00003800003800003800003800003800003800003800003800003 80007FFE00FFFF007FFE0015147F9318>I<07F7003FFF007FFF00780F00E00700E00700 E007007C00007FE0001FFC0003FE00001F00600780E00380E00380F00380F80F00FFFF00 FFFC00E7F00011147D9318>I<0180000380000380000380000380007FFFC0FFFFC0FFFF C00380000380000380000380000380000380000380000380000380000380400380E00380 E00380E001C1C001FFC000FF80003E0013197F9818>I<7E07E0FE0FE07E07E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E01E00F 03E007FFFC03FFFE01FCFC1714809318>I<7F8FF0FF8FF87F8FF01E03C00E03800E0380 0E0380070700070700070700038E00038E00038E00038E0001DC0001DC0001DC0000F800 00F80000700015147F9318>II<7F8FF07F9FF07F8FF0070700078E00039E0001DC0001F80000F80000 700000F00000F80001DC00039E00038E000707000F07807F8FF0FF8FF87F8FF015147F93 18>I<7F8FF0FF8FF87F8FF00E01C00E03800E0380070380070700070700038700038600 038E0001CE0001CE0000CC0000CC0000DC00007800007800007800007000007000007000 00F00000E00079E0007BC0007F80003F00001E0000151E7F9318>I<3FFFF07FFFF07FFF F07001E07003C0700780000F00001E00003C0000F80001F00003C0000780000F00701E00 703C0070780070FFFFF0FFFFF0FFFFF014147F9318>I<0007E0001FE0007FE000780000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00001E0007F C000FF8000FF80007FC00001E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E000007800007FE0001FE00007E013247E9F18>I<60F0F0F0F0F0 F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0600424769F18> I<7C0000FF0000FFC00003C00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000F000007FC0003FE0003FE0007FC000F00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00003C000FFC000FF00007C00 0013247E9F18>I E /FC 1 98 df<001800001800001800003C00003C00004E00004E00 004E000087000087000187800103800103800201C00201C003FFC00400E00400E0080070 0800701800703C0078FE01FF18177F961C>97 D E /FD 93 128 df<001F83E000F06E3001C078780380F8780300F0300700700007007000070070000700 7000070070000700700007007000FFFFFF80070070000700700007007000070070000700 700007007000070070000700700007007000070070000700700007007000070070000700 7000070070000700700007007000070070007FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700000700000700 00FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000 E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F04 0001C07C06000380F80F000300F00F000700F00F00070070000007007000000700700000 070070000007007000000700700000FFFFFFFF0007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 07007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C07C0F000380F80F000300 F00700070070070007007007000700700700070070070007007007000700700700070070 0700FFFFFFFF000700700700070070070007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 070070070007007007000700700700070070070007007007007FE3FE3FF02420809F26> I22 D<70F8F8F8F8F8F8F87070707070707070707020202020200000 00000070F8F8F87005217CA00D>33 D<7038F87CFC7EFC7E743A04020402040208040804 10081008201040200F0E7E9F17>I<000300C0000300C0000300C0000300C00006018000 060180000601800006018000060180000C0300000C0300000C0300000C0300000C030000 1806007FFFFFFCFFFFFFFE00300C0000300C0000300C0000300C00006018000060180000 60180000601800FFFFFFFE7FFFFFFC00C030000180600001806000018060000180600001 8060000300C0000300C0000300C0000300C0000300C0000601800006018000060180001F 297D9F26>I<007800000084000001840000030200000702000007020000070200000702 0000070400000704000007080000070800000310000003A00FFC03C003E0038001C001C0 008001C0010003E0010004E0020008F00200187004003078080070380800701C1000F01E 1000F00E2000F0074000F003C0087003C0087801C010380670301C18386007E00F801E22 7EA023>38 D<70F8FCFC74040404080810102040060E7C9F0D>I<002000400080010002 0006000C000C00180018003000300030007000600060006000E000E000E000E000E000E0 00E000E000E000E000E000E0006000600060007000300030003000180018000C000C0006 00020001000080004000200B2E7DA112>I<800040002000100008000C00060006000300 030001800180018001C000C000C000C000E000E000E000E000E000E000E000E000E000E0 00E000E000C000C000C001C001800180018003000300060006000C000800100020004000 80000B2E7DA112>I<01800180018001800180C183F18F399C0FF003C003C00FF0399CF1 8FC1830180018001800180018010147DA117>I<00060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 0006000000060000FFFFFFF0FFFFFFF00006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 000600001C207D9A23>I<70F8FCFC74040404080810102040060E7C840D>II<70F8F8F87005057C840D>I<000100030003000600060006000C000C000C00 180018001800300030003000600060006000C000C000C001800180018003000300030006 00060006000C000C000C00180018001800300030003000600060006000C000C000C00010 2D7DA117>I<03F0000E1C001C0E00180600380700700380700380700380700380F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0 7003807003807003807807803807001806001C0E000E1C0003F000121F7E9D17>I<0180 03800F80F380038003800380038003800380038003800380038003800380038003800380 03800380038003800380038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C0010 0E00200700400780800780F007C0F803C0F803C0F803C02007C00007C000078000078000 0F00000E00001C0000380000700000600000C0000180000300000600400C004018004010 00803FFF807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00780F80780780 780780380F80000F80000F00000F00000E00001C0000380003F000003C00000E00000F00 0007800007800007C02007C0F807C0F807C0F807C0F00780400780400F00200E001C3C00 03F000121F7E9D17>I<000600000600000E00000E00001E00002E00002E00004E00008E 00008E00010E00020E00020E00040E00080E00080E00100E00200E00200E00400E00C00E 00FFFFF0000E00000E00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I<18 03001FFE001FFC001FF8001FE00010000010000010000010000010000010000011F00016 1C00180E001007001007800003800003800003C00003C00003C07003C0F003C0F003C0E0 0380400380400700200600100E000C380003E000121F7E9D17>I<007C00018200070100 0E03800C07801C0780380300380000780000700000700000F1F000F21C00F40600F80700 F80380F80380F003C0F003C0F003C0F003C0F003C07003C07003C0700380380380380700 1807000C0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF804001008002 0080020080040000080000080000100000200000200000400000400000C00000C00001C0 000180000380000380000380000380000780000780000780000780000780000780000780 00030000121F7D9D17>I<03F0000C0C0010060030030020018060018060018060018070 01807803003E03003F06001FC8000FF00003F80007FC000C7E00103F00300F8060038040 01C0C001C0C000C0C000C0C000C0C000806001802001001002000C0C0003F000121F7E9D 17>I<03F0000E18001C0C00380600380700700700700380F00380F00380F003C0F003C0 F003C0F003C0F003C07007C07007C03807C0180BC00E13C003E3C0000380000380000380 000700300700780600780E00700C002018001070000FC000121F7E9D17>I<70F8F8F870 0000000000000000000070F8F8F87005147C930D>I<70F8F8F870000000000000000000 0070F0F8F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF0000000000000 0000000000000000000000000000000000000000000000000000FFFFFFF07FFFFFE01C0C 7D9023>61 D<07000F800F800F8007000000000000000000000002000200020002000200 02000600060004000C001C00380078007000F018F03CF03CF01CF008701838200FC00E20 7D9615>I<0FC0307040384038E03CF03CF03C603C0038007000E000C001800180010003 000200020002000200020002000000000000000000000007000F800F800F8007000E207D 9F15>I<000100000003800000038000000380000007C0000007C0000007C0000009E000 0009E0000009E0000010F0000010F0000010F00000207800002078000020780000403C00 00403C0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F0002000780 0200078002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F204F00 07A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000001FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F00 001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004078004 0F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C>II<080410082010201040204020804080408040B85CFC7E FC7E7C3E381C0F0E7B9F17>II<08102020404080 8080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00000E00 000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10386720 0F83C014147E9317>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00780E00 780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E001520 7F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0007000700038 0138011C020E0C03F010147E9314>I<000380003F800003800003800003800003800003 8000038000038000038000038000038003E380061B801C07803803803803807003807003 80F00380F00380F00380F00380F00380F003807003807003803803803807801C07800E1B 8003E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F00380F0 0380FFFF80F00000F00000F000007000007000003800801800800C010007060001F80011 147F9314>I<007C00C6018F038F07060700070007000700070007000700FFF007000700 07000700070007000700070007000700070007000700070007000700070007007FF01020 809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F00780F0038 0E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC03001E060 0070C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I<0E0000 FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E3E00 0E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C001E003E001E00 1C000000000000000000000000000E007E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001F001F000E000 0000000000000000000000007007F000F000700070007000700070007000700070007000 70007000700070007000700070007000700070007000706070F060F0C061803F000C2882 9E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70000EF8000F38 000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF815207F9F18>I<0E 00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B20809F0C>I<0E1F 01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I<0E3E00 FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E001C03 803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000E07000 E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F01C00F00E00E 00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E00F00E00E00F 01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E00000E0000FF E000151D7F9319>I<03E0800619801C05803C0780380380780380700380F00380F00380 F00380F00380F00380F003807003807803803803803807801C0B800E138003E380000380 000380000380000380000380000380000380000380003FF8151D7E9318>I<0E78FE8C0F 1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00F 147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070803880188018 C018C018E030D0608F800D147E9312>I<020002000200060006000E000E003E00FFF80E 000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610031001 E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16147F93 19>III<7FC3FC0F01E00701C007018003810001C20000E40000EC 00007800003800003C00007C00004E000087000107000303800201C00601E01E01E0FF07 FE1714809318>II<3FFF380E200E 201C40384078407000E001E001C00380078007010E011E011C0338027006700EFFFE1014 7F9314>III<30307878F87C787830 300E057C9E17>127 D E /FE 46 122 df<70F8FCFC7404040404080810102040060F7C 840E>44 DI<70F8F8F87005057C840E>I<01F000071C000C0600 1803003803803803807001C07001C07001C07001C0F001E0F001E0F001E0F001E0F001E0 F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001C07001C07001C0 7803C03803803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038003800380038003800380038007C0FFFE0F217CA018 >I<03F0000C1C001007002007804003C04003C08003E0F003E0F801E0F801E0F801E020 03E00003E00003C00003C0000780000700000E00001C0000180000300000600000C00001 80000100000200200400200800201800603000403FFFC07FFFC0FFFFC013217EA018>I< 03F8000C1E001007002007804007C07807C07803C07807C03807C0000780000780000700 000F00000E0000380003F000001C00000F000007800007800003C00003C00003E02003E0 7003E0F803E0F803E0F003C04003C0400780200780100F000C1C0003F00013227EA018> I<000200000600000E00000E00001E00001E00002E00004E00004E00008E00008E00010E 00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E00800E00FFFF F8000E00000E00000E00000E00000E00000E00000E00001F0001FFF015217FA018>I<10 00801E07001FFF001FFE001FF80013E00010000010000010000010000010000010000010 F800130E001407001803801003800001C00001C00001E00001E00001E00001E07001E0F0 01E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013227EA018>I< 007E0001C1000300800601C00E03C01C03C0180180380000380000780000700000700000 F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0F001E0F001E0 7001E07001E07001E03801C03801C01803801C03000C0600070C0001F00013227EA018> I<01F800060E000803001001802001802000C06000C06000C06000C07000C07801803E01 003F02001FC4000FF80003F80003FC00067F00083F80100F803007C06001C06000E0C000 E0C00060C00060C00060C000606000406000C03000801803000E0E0003F00013227EA018 >56 D<01F000060C000C0600180700380380700380700380F001C0F001C0F001C0F001E0 F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E003E1E00001C0 0001C00001C0000380000380300300780700780600700C002018001030000FC00013227E A018>I<0001800000018000000180000003C0000003C0000003C0000005E0000005E000 000DF0000008F0000008F0000010F800001078000010780000203C0000203C0000203C00 00401E0000401E0000401E0000800F0000800F0000FFFF000100078001000780030007C0 020003C0020003C0040003E0040001E0040001E00C0000F00C0000F03E0001F8FF800FFF 20237EA225>65 D<0007E0100038183000E0063001C00170038000F0070000F00E000070 1E0000701C0000303C0000303C0000307C0000107800001078000010F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F800000078000000780000107C000010 3C0000103C0000101C0000201E0000200E000040070000400380008001C0010000E00200 00381C000007E0001C247DA223>67 D69 DI73 D77 DI80 D82 D<03F0200C0C601802603001E07000E06000 60E00060E00060E00020E00020E00020F00000F000007800007F00003FF0001FFE000FFF 0003FF80003FC00007E00001E00000F00000F0000070800070800070800070800070C000 60C00060E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF8780780786007 8018400780084007800840078008C007800C800780048007800480078004800780040007 800000078000000780000007800000078000000780000007800000078000000780000007 800000078000000780000007800000078000000780000007800000078000000780000007 80000007800000078000000FC00003FFFF001E227EA123>I<0FE0001838003C0C003C0E 0018070000070000070000070000FF0007C7001E07003C0700780700700700F00708F007 08F00708F00F087817083C23900FC1E015157E9418>97 D<0E0000FE00001E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F000E61C0 0E80600F00300E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E 0E001C0E003C0E00380F00700C80600C41C0083F0017237FA21B>I<01FE000703000C07 801C0780380300780000700000F00000F00000F00000F00000F00000F00000F000007000 007800403800401C00800C010007060001F80012157E9416>I<0000E0000FE00001E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E001F8E007 04E00C02E01C01E03800E07800E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F0 00E07000E07800E03800E01801E00C02E0070CF001F0FE17237EA21B>I<01FC00070700 0C03801C01C03801C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000F00000 7000007800203800201C00400E008007030000FC0013157F9416>I<003C00C6018F038F 030F070007000700070007000700070007000700FFF80700070007000700070007000700 0700070007000700070007000700070007000700070007807FF8102380A20F>I<000070 01F198071E180E0E181C07001C07003C07803C07803C07803C07801C07001C07000E0E00 0F1C0019F0001000001000001800001800001FFE000FFFC00FFFE03800F0600030400018 C00018C00018C000186000306000303800E00E038003FE0015217F9518>I<0E0000FE00 001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E1F800E60C00E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C 001E003E001E001C00000000000000000000000000000000000E00FE001E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A227FA10E >I<01C003E003E003E001C00000000000000000000000000000000001E00FE001E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F18061803E000B2C82A10F>I<0E0000FE00001E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E03FC0E01 F00E01C00E01800E02000E04000E08000E10000E38000EF8000F1C000E1E000E0E000E07 000E07800E03C00E01C00E01E00E00F00E00F8FFE3FE17237FA21A>I<0E00FE001E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B237FA20E>I<0E1F C07F00FE60E183801E807201C00F003C00E00F003C00E00E003800E00E003800E00E0038 00E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800 E00E003800E00E003800E00E003800E00E003800E00E003800E0FFE3FF8FFE27157F942A >I<0E1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F941B>I< 01FC000707000C01801800C03800E0700070700070F00078F00078F00078F00078F00078 F00078F000787000707800F03800E01C01C00E038007070001FC0015157F9418>I<0E1F 00FE61C00E80600F00700E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E00 1E0E001E0E003C0E003C0E00380F00700E80E00E41C00E3F000E00000E00000E00000E00 000E00000E00000E00000E00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F06 0F000E000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF010157F94 13>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C80 0CC00CC008E018D0308FC00E157E9413>I<02000200020002000600060006000E001E00 3E00FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E04 0E040708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E0070 0E00700E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F0060170 03827800FC7F18157F941B>III121 D E /FF 20 118 df45 D68 D73 D77 D80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E0780000E0 700000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F8000000 7C0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC0000FFFF00 000FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F8000000F8 8000007880000078800000788000007880000078C0000078C0000070E00000F0E00000E0 F00000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E003E001E 0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E001F001E 003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E0478003E 047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700 040F00041E001E1C003E3C003E7C003E7C0008780000F80000F80000F80000F80000F800 00F80000F80000F80000F800007800007C00007C00003C00011E00011E00020F00020700 0403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E00000 01E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E00000 01E0000001E0000001E0000001E0000001E0000001E0001F81E000F061E001C019E00780 05E00F0003E00E0003E01E0001E03C0001E03C0001E07C0001E0780001E0F80001E0F800 01E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0780001E07800 01E03C0001E03C0001E01C0001E01E0003E00E0005E0070009E0038011F000E061FF003F 81FF20327DB125>I<003F800000E0E0000380380007003C000E001E001E001E001C000F 003C000F007C000F0078000F8078000780F8000780F8000780FFFFFF80F8000000F80000 00F8000000F8000000F8000000F8000000780000007C0000003C0000003C0000801E0000 800E0001000F0002000780020001C00C0000F03000001FC000191F7E9E1D>I<0007E000 1C1000383800707C00E07C01E07C01C03803C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C000FFFFC0FFFFC003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00007E0007FFF007F FF0016327FB114>I<000000F0007F030801C1C41C0380E81C070070080F0078001E003C 001E003C003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E003C 000F007800070070000780E00009C1C000087F0000180000001800000018000000180000 00180000001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E0300000 F070000070E0000038E0000038E0000038E0000038E00000387000007070000070380000 E01C0001C00700070001C01C00003FE0001E2F7E9F21>I<07000F801F801F800F800700 000000000000000000000000000000000000000000000780FF80FF800F80078007800780 078007800780078007800780078007800780078007800780078007800780078007800780 0780078007800FC0FFF8FFF80D307EAF12>105 D<0780FE001FC000FF83078060F000FF 8C03C18078000F9001E2003C0007A001E4003C0007A000F4001E0007C000F8001E0007C0 00F8001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000 F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0 001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F000 1E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E 000FC001F8003F00FFFC1FFF83FFF0FFFC1FFF83FFF0341F7E9E38>109 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C000F000 07C000F000078000F000078000F000078000F000078000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F0000FC001 F800FFFC1FFF80FFFC1FFF80211F7E9E25>I<001FC00000F0780001C01C00070007000F 0007801E0003C01C0001C03C0001E03C0001E0780000F0780000F0780000F0F80000F8F8 0000F8F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8780000F07C0001F03C 0001E03C0001E01E0003C01E0003C00F00078007800F0001C01C0000F07800001FC0001D 1F7E9E21>I<0783E0FF8C18FF907C0F907C07A07C07C03807C00007C00007C000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0780000780000780000780000780000780000FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E00010E00010F00010F800007E00 003FF0001FFF000FFFC003FFE0003FF00001F80000F880003C80003C80001CC0001CC000 1CE0001CE00018F00038F00030CC0060C301C080FE00161F7E9E1A>I<00400000400000 400000400000400000C00000C00000C00001C00001C00003C00007C0000FC0001FFFE0FF FFE003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C01003C01003C01003C01003C01003C01003C01003 C01001C02001E02000E0400078C0001F00142C7FAB19>I<078000F000FF801FF000FF80 1FF0000F8001F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078001F000 078001F000078001F000038002F00003C004F00001C008F800007030FF80001FC0FF8021 1F7E9E25>I E /FG 5 85 df<00000000600000000000700000000000F00000000001F0 0000000001F00000000003F00000000003F00000000007F00000000007F0000000000FF0 000000000FF0000000001BF00000000033F00000000033F00000000063F00000000063F8 00000000C1F800000000C1F80000000181F80000000381F80000000301F80000000601F8 0000000601F80000000C01F80000000C01F80000001801F80000001801F80000003001F8 0000006001F80000006001F8000000C001F8000000C001FC000001FFFFFC000001FFFFFC 0000030000FC0000070000FC0000060000FC00000C0000FC00000C0000FC0000180000FC 0000180000FC0000300000FC0000700000FC0000600000FC0000E00000FC0001E00000FC 0003E00000FE000FF00001FE00FFFE003FFFF0FFFE003FFFF02C327CB135>65 D<000FFFFFFF0000000FFFFFFFC00000003F8007F00000003F8001F80000003F00007C00 00003F00007E0000007F00003E0000007F00001F0000007E00001F0000007E00001F8000 00FE00000F800000FE00000F800000FC00000FC00000FC00000FC00001FC00000FC00001 FC00000FC00001F800000FC00001F800000FC00003F800000FC00003F800001FC00003F0 00001FC00003F000001FC00007F000001FC00007F000001F800007E000003F800007E000 003F80000FE000003F80000FE000003F00000FC000007F00000FC000007F00001FC00000 7E00001FC00000FE00001F800000FC00001F800000FC00003F800001F800003F800001F0 00003F000003F000003F000007E000007F000007C000007F00000FC000007E00001F8000 007E00003F000000FE00007E000000FE0000F8000000FC0001F0000000FC0007E0000001 FC003F800000FFFFFFFE000000FFFFFFF000000032317CB036>68 D<000FFFFFFFFE000FFFFFFFFE00003F8000FE00003F80003E00003F00001E00003F0000 1E00007F00000C00007F00000C00007E00000C00007E00000C0000FE00000C0000FE0000 0C0000FC00000C0000FC00000C0001FC00001C0001FC00C0180001F800C0000001F800C0 000003F801C0000003F801C0000003F00180000003F00380000007F00F80000007FFFF80 000007FFFF00000007E00F0000000FE0070000000FE0070000000FC0060000000FC00600 00001FC00E0000001FC00E0000001F800C0000001F80000000003F80000000003F800000 00003F00000000003F00000000007F00000000007F00000000007E00000000007E000000 0000FE0000000000FE0000000000FC0000000000FC0000000001FC00000000FFFFFC0000 00FFFFFC0000002F317CB02F>70 D<000FFFFFF800000FFFFFFF0000003F801FC000003F 8007E000003F0003F000003F0001F800007F0000FC00007F0000FC00007E0000FC00007E 0000FC0000FE0000FC0000FE0001FC0000FC0001FC0000FC0001FC0001FC0001F80001FC 0003F80001F80003F00001F80007E00003F80007E00003F8000F800003F0003F000003F0 007E000007F003F8000007FFFFE0000007FFFF80000007E007C000000FE003F000000FE0 01F000000FC000F800000FC000F800001FC000FC00001FC000FC00001F8000FC00001F80 00FC00003F8001FC00003F8001FC00003F0001FC00003F0001FC00007F0003F800007F00 03F800007E0003F800007E0003F80600FE0003F80E00FE0003F80C00FC0003F80C00FC00 03F81C01FC0001F838FFFFF000FC70FFFFF0007FE0000000001F802F327CB034>82 D<07FFFFFFFFF807FFFFFFFFF80FE007F001F80F8007F000F80E0007E000701E0007E000 701C000FE0007018000FE0007038000FC0007038000FC0007030001FC0006070001FC000 6060001F80006060001F80006060003F8000E0E0003F8000C000003F00000000003F0000 0000007F00000000007F00000000007E00000000007E0000000000FE0000000000FE0000 000000FC0000000000FC0000000001FC0000000001FC0000000001F80000000001F80000 000003F80000000003F80000000003F00000000003F00000000007F00000000007F00000 000007E00000000007E0000000000FE0000000000FE0000000000FC0000000000FC00000 00001FC0000000001FC0000000001F80000000003F80000000007FC00000007FFFFFC000 007FFFFFC000002D3174B033>84 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 795 908 a FG(D)26 b(R)g(A)f(F)h(T)225 999 y FF(Do)r(cumen)n(t) 20 b(for)i(a)f(Standard)g(Message-P)n(assing)f(In)n(terface)621 1194 y FE(Message)c(P)o(assing)h(In)o(terface)e(F)l(orum)783 1320 y(F)l(ebruary)h(20,)g(1994)77 1378 y(This)h(w)o(ork)f(w)o(as)h (supp)q(orted)g(in)f(part)h(b)o(y)e(ARP)l(A)h(and)h(NSF)e(under)h(gran) o(t)h(ASC-9310330,)i(the)192 1436 y(National)d(Science)f(F)l(oundation) i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q(erativ)o(e) 76 1494 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o(y)e(the)h (Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1552 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 166 45 a FD(The)14 b(Message)e(P)o(assing)h(In)o(terface)h(F)l (orum)f(\(MPIF\),)f(with)i(participation)g(from)f(o)o(v)o(er)f(40)h (organi-)75 102 y(zations,)h(has)f(b)q(een)i(meeting)f(since)g(Jan)o (uary)g(1993)e(to)h(discuss)h(and)g(de\014ne)h(a)e(set)g(of)g(library)i (in)o(terface)75 158 y(standards)h(for)h(message)f(passing.)25 b(MPIF)16 b(is)i(not)e(sanctioned)i(or)e(supp)q(orted)h(b)o(y)g(an)o(y) g(o\016cial)g(stan-)75 214 y(dards)e(organization.)166 271 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o(terface,)g (simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g(used)75 327 y(standard)h(for)g(writing)h(message-passing)g(programs.)29 b(As)19 b(suc)o(h)g(the)g(in)o(terface)g(should)g(establish)h(a)75 384 y(practical,)c(p)q(ortable,)f(e\016cien)o(t,)g(and)h(\015exible)h (standard)e(for)g(message)f(passing.)166 440 y(This)22 b(is)f(a)g(draft)f(of)h(what)g(will)h(b)q(ecome)g(the)f(Final)h(Rep)q (ort,)h(V)l(ersion)f(1.0,)f(of)f(the)i(Message)75 497 y(P)o(assing)c(In)o(terface)h(F)l(orum.)29 b(This)19 b(do)q(cumen)o(t)g(con)o(tains)g(all)g(the)g(tec)o(hnical)h(features)e (prop)q(osed)h(for)75 553 y(the)c(in)o(terface.)20 b(This)c(cop)o(y)f (of)g(the)g(draft)g(w)o(as)f(pro)q(cessed)i(b)o(y)f(L)1174 547 y FC(a)1195 553 y FD(T)1220 567 y(E)1246 553 y(X)g(on)g(F)l (ebruary)g(20,)g(1994.)166 610 y(MPIF)d(in)o(vites)h(commen)o(ts)e(on)h (the)g(tec)o(hnical)i(con)o(ten)o(t)d(of)h(MPI,)f(as)h(w)o(ell)h(as)e (on)h(the)g(editorial)i(pre-)75 666 y(sen)o(tation)h(in)h(the)f(do)q (cumen)o(t.)20 b(Commen)o(ts)14 b(receiv)o(ed)j(b)q(efore)e(Jan)o(uary) g(15,)f(1994)g(will)j(b)q(e)f(considered)75 723 y(in)i(pro)q(ducing)h (the)f(\014nal)g(draft)f(of)g(V)l(ersion)h(1.0)f(of)g(the)g(Message)g (P)o(assing)h(In)o(terface)f(Sp)q(eci\014cation.)75 779 y(Please)g(send)g(commen)o(ts)e(on)i(MPI)f(to)g FB (mpi-comments@cs.utk.edu)o FD(.)k(Y)l(our)d(commen)o(t)f(will)h(b)q(e)g (for-)75 835 y(w)o(arded)e(to)g(MPIF)f(committee)i(mem)o(b)q(ers)f(who) g(will)i(attempt)d(to)g(resp)q(ond.)179 2091 y(c)166 2092 y FA(\015)p FD(1993)h(Univ)o(ersit)o(y)h(of)f(T)l(ennessee,)h(Kno) o(xville,)h(T)l(ennessee.)22 b(P)o(ermission)16 b(to)f(cop)o(y)g (without)h(fee)75 2148 y(all)e(or)f(part)f(of)h(this)h(material)f(is)h (gran)o(ted,)e(pro)o(vided)i(the)g(Univ)o(ersit)o(y)f(of)g(T)l (ennessee)h(cop)o(yrigh)o(t)f(notice)75 2205 y(and)i(the)h(title)g(of)f (this)g(do)q(cumen)o(t)h(app)q(ear,)f(and)g(notice)h(is)g(giv)o(en)g (that)e(cop)o(ying)i(is)g(b)o(y)f(p)q(ermission)h(of)75 2261 y(the)f(Univ)o(ersit)o(y)h(of)f(T)l(ennessee.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 3 2 2 bop 75 377 a Fy(Con)m(ten)m(ts)75 645 y Fx(Ac)o(kno)o(wledgmen)o (ts)1346 b(vi)75 747 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(1)143 804 y FD(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(1)143 860 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(2)143 917 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(3)143 973 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(3)143 1030 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b Fw(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b FD(3)143 1086 y(1.6)46 b(Organization)16 b(of)e(this)i(Do)q(cumen)o(t)34 b Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(4)75 1188 y Fx(2)42 b(MPI)17 b(T)l(erms)f(and)i(Con)o(v)o(en)o (tions)1042 b(6)143 1245 y FD(2.1)46 b(Pro)q(cedure)16 b(Sp)q(eci\014cation)44 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b FD(6)143 1302 y(2.2)46 b(Seman)o(tic)15 b(T)l(erms)g Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 91 b FD(6)143 1358 y(2.3)46 b(Data)14 b(T)o(yp)q(es)30 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b FD(7)248 1415 y(2.3.1)50 b(Opaque)16 b(ob)s(jects)21 b Fw(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(7)248 1471 y(2.3.2)50 b(Arra)o(y)14 b(argumen)o(ts)28 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(8)248 1528 y(2.3.3)50 b(State)14 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(8)248 1584 y(2.3.4)50 b(Named)15 b(constan)o(ts)23 b Fw(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(8)248 1641 y(2.3.5)50 b(Choice)20 b Fw(:)i(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(9)248 1697 y(2.3.6)50 b(Addresses)28 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FD(9)143 1754 y(2.4)46 b(Language)15 b(Binding)45 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)91 b FD(9)248 1810 y(2.4.1)50 b(Data)14 b(T)o(yp)q(es)i(and)f(Naming)g(Con)o(v)o(en)o(tions)39 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)69 b FD(10)248 1867 y(2.4.2)50 b(F)l(ortran)14 b(77)h(Binding)i (Issues)30 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(10)248 1923 y(2.4.3)50 b(C)15 b(Binding)i(Issues)31 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(10)143 1980 y(2.5)46 b(Pro)q(cesses)40 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(11)143 2036 y(2.6)46 b(Error)14 b(Handling)32 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(12)143 2093 y(2.7)46 b(Implemen)o(tation)16 b(issues)36 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(13)248 2149 y(2.7.1)50 b(Indep)q(endence)19 b(of)14 b(basic)i(run)o(time)g (routines)44 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b FD(13)248 2206 y(2.7.2)50 b(In)o(teraction)16 b(with)f(signals)h(in)g(POSIX)42 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(13)75 2308 y Fx(3)42 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)g(Comm)o(unication) 975 b(15)143 2364 y FD(3.1)46 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FD(15)143 2421 y(3.2)46 b(Basic)16 b(send)f(op)q(eration) 27 b Fw(:)c(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(15)248 2477 y(3.2.1)50 b(Message)15 b(data)26 b Fw(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(16)248 2534 y(3.2.2)50 b(Message)15 b(en)o(v)o(elop)q(e)j Fw(:)k(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(17)143 2591 y(3.3)46 b(Basic)16 b(receiv)o(e)g(op)q(eration)h Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(18)248 2647 y(3.3.1)50 b(Return)16 b(status)j Fw(:)k(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(20)143 2704 y(3.4)46 b(Seman)o(tics)15 b(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)40 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FD(23)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 4 3 3 bop 143 45 a FD(3.5)46 b(Data)14 b(T)o(yp)q(e)h(Matc)o(hing)22 b Fw(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(26)248 99 y(3.5.1)50 b(T)o(yp)q(e)16 b Fv(MPI)p 588 99 13 2 v 14 w(CHARA)o(CTER)48 b Fu(:)20 b(:)g(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)g(:) h(:)f(:)g(:)57 b Ft(28)143 160 y FD(3.6)46 b(Data)14 b(con)o(v)o(ersion)g Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b FD(29)143 221 y(3.7)46 b(Comm)o(unication)15 b(Mo)q(des)i Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(31)143 282 y(3.8)46 b(Non)o(blo)q(c)o(king)16 b(comm)o(unication)j Fw(:)j(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(33)248 343 y(3.8.1)50 b(Comm)o(unication)16 b(Ob)s(jects)23 b Fw(:)g(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(34)248 404 y(3.8.2)50 b(Comm)o(unication)16 b(initiation)32 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FD(34)248 464 y(3.8.3)50 b(Comm)o(unication)16 b(Completion)i Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(37)248 525 y(3.8.4)50 b(Seman)o(tics)16 b(of)f(Non)o(blo)q(c)o(king)h(Comm)o(unications)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(39)248 586 y(3.8.5)50 b(Multiple)17 b(Completions)41 b Fw(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(40)143 647 y(3.9)46 b(Prob)q(e)15 b(and)g(Cancel)g Fw(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(48)143 708 y(3.10)23 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(requests)33 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(51)143 769 y(3.11)23 b(Send-receiv)o(e)d Fw(:)i(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(56)143 830 y(3.12)23 b(Null)16 b(pro)q(cesses)i Fw(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(58)143 890 y(3.13)23 b(Deriv)o(ed)15 b(datat)o(yp)q(es)42 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(58)248 951 y(3.13.1)27 b(Datat)o(yp)q(e)14 b(constructors)27 b Fw(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FD(60)248 1012 y(3.13.2)27 b(Address)16 b(and)f(exten)o(t)g(functions)32 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(66)248 1073 y(3.13.3)27 b(Lo)o(w)o(er-b)q(ound)16 b(and)f(upp)q(er-b)q(ound)j(mark)o(ers)33 b Fw(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(67)248 1134 y(3.13.4)27 b(Commit)15 b(and)g(free)28 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(68)248 1195 y(3.13.5)27 b(Use)16 b(of)e(general)i(datat)o(yp)q(es)f(in)h(comm) o(unication)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)69 b FD(69)248 1256 y(3.13.6)27 b(Correct)15 b(use)g(of)g (addresses)26 b Fw(:)d(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(72)248 1316 y(3.13.7)27 b(Examples)33 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(73)143 1377 y(3.14)23 b(New)15 b(prop)q(osal)g(for)g(pac)o(k)g(and)g(unpac)o(k)32 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(81)75 1487 y Fx(4)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(87)143 1547 y FD(4.1)46 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(87)143 1608 y(4.2)46 b(Comm)o(unication)15 b(F)l(unctions)24 b Fw(:)f(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(88)143 1669 y(4.3)46 b(Barrier)15 b(sync)o(hronization)44 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(88)143 1730 y(4.4)46 b(Data)14 b(mo)o(v)o(e)g(functions)29 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(89)248 1791 y(4.4.1)50 b(Broadcast)24 b Fw(:)e(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(89)248 1852 y(4.4.2)50 b(Gather)13 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(89)248 1912 y(4.4.3)50 b(Examples)16 b(of)f(Usage)f(of)h Fs(MPI)p 921 1912 14 2 v 16 w(GA)l(THER)p FD(,)h Fs(MPI)p 1223 1912 V 16 w(GA)l(THERV)24 b Fw(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(93)248 1973 y(4.4.4)50 b(Scatter)c Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FD(99)248 2034 y(4.4.5)50 b(Examples)16 b(of)f(Usage)f(of)h Fs(MPI)p 921 2034 V 16 w(SCA)l(TTER)p FD(,)h Fs(MPI)p 1246 2034 V 16 w(SCA)l(TTERV)d Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(101)248 2095 y(4.4.6)k(Gather-to-all)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FD(104)248 2156 y(4.4.7)k(All-to-All)18 b(Scatter/Gather)32 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(106)143 2217 y(4.5)g(Global)16 b(Compute)e(Op)q(erations)20 b Fw(:)i(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(108)248 2278 y(4.5.1)k(Reduce)45 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(109)248 2338 y(4.5.2)k(User-Reduce)c Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(112)248 2399 y(4.5.3)k(All-Reduce)43 b Fw(:)22 b(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(115)248 2460 y(4.5.4)k(Reduce-Scatter)30 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b FD(116)248 2521 y(4.5.5)k(Scan)25 b Fw(:)d(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FD(118)143 2582 y(4.6)g(Correctness)32 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b FD(119)248 2643 y(4.6.1)k(Sync)o(hronization)17 b(Side-E\013ects)12 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(119)248 2704 y(4.6.2)k(Multiple)17 b(Calls)f(to)f(Collectiv)o(e)31 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(120)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 5 4 4 bop 75 45 a Fx(5)42 b(Groups,)17 b(Con)o(texts,)f(Comm)o (unicators,)g(and)i(Cac)o(hing)543 b(123)143 102 y FD(5.1)46 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(123)248 159 y(5.1.1)k(Wh)o(y)15 b(w)o(e)g(need)h(libraries)30 b Fw(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b FD(123)248 216 y(5.1.2)k(Wh)o(y)15 b(existing)h(systems)f(do)g (not)g(supp)q(ort)g(libraries)f Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FD(123)248 272 y(5.1.3)k(What)15 b(features)g(are)f(needed)j(to)d(supp)q(ort)i(libraries)23 b Fw(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(124)248 329 y(5.1.4)k(What)15 b(these)g(features)g(are)g(called)i (in)f(MPI)27 b Fw(:)c(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FD(124)143 386 y(5.2)g(Basic)16 b(Concepts)30 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b FD(125)248 443 y(5.2.1)k(Groups)43 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(125)248 500 y(5.2.2)k(Con)o(texts)c Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(125)248 556 y(5.2.3)k(Comm)o(unicators) 17 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(126)248 613 y(5.2.4)k(Prede\014ned)17 b(Comm)o(unicators)38 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(126)143 670 y(5.3)g(Group)15 b(Managemen)o(t)41 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FD(127)248 727 y(5.3.1)k(Group)15 b(Accessors)34 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(127)248 784 y(5.3.2)k(Group)15 b(Constructors)37 b Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(127)248 840 y(5.3.3)k(Group)15 b(Destructors)27 b Fw(:)c(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FD(131)143 897 y(5.4)g(Comm)o(unicator)14 b(Managemen)o(t)i Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(131)248 954 y(5.4.1)k(Comm)o(unicator)15 b(Accessors)43 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(131)248 1011 y(5.4.2)k(Comm)o (unicator)15 b(Constructors)10 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(132)248 1068 y(5.4.3)k(Comm)o(unicator)15 b(Destructors)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(134)143 1124 y(5.5)g(In)o(ter-Comm)o (unication)14 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FD(135)248 1181 y(5.5.1)k(Comm)o(unicator)15 b(Accessors)43 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(136)248 1238 y(5.5.2)k(In)o(ter-Comm)o(unicator)15 b(Constructors)f(and)h (Destructors)26 b Fw(:)c(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(137)248 1295 y(5.5.3)k(Name)15 b(Service)25 b Fw(:)e(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(142)143 1352 y(5.6)g(Cac)o(hing)32 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(144)248 1408 y(5.6.1)k(F)l(unctionalit)o(y)32 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(144)143 1465 y(5.7)g(F)l (ormalizing)16 b(the)f(Lo)q(osely)h(Sync)o(hronous)g(Mo)q(del)g (\(Usage,)e(Safet)o(y\))j Fw(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(149)248 1522 y(5.7.1)k(Basic)16 b(Statemen)o(ts)22 b Fw(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(149)248 1579 y(5.7.2)k(Mo)q(dels)16 b(of)f(Execution)29 b Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(149)143 1636 y(5.8)g(Motiv)m(ating)15 b(Examples)h Fw(:)23 b(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(150)248 1692 y(5.8.1)k(Curren)o (t)15 b(Practice)g(#1)27 b Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(150)248 1749 y(5.8.2)k(Curren)o(t)15 b(Practice)g(#2)27 b Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(151)248 1806 y(5.8.3)k(\(Appro)o(ximate\))15 b(Curren)o(t)f(Practice)i(#3)35 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b FD(152)248 1863 y(5.8.4)k(Example)16 b(#4)44 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(153)248 1920 y(5.8.5)k(Library)16 b(Example)g(#1)22 b Fw(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(154)248 1976 y(5.8.6)k(Library)16 b(Example)g(#2)22 b Fw(:)g(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FD(155)248 2033 y(5.8.7)k(In)o(ter-Comm)o (unication)16 b(Examples)g Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(157)75 2136 y Fx(6)c(Pro)q(cess)17 b(T)l(op)q(ologies)1224 b(163)143 2192 y FD(6.1)46 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(163)143 2249 y(6.2)g(Virtual)16 b(T)l(op)q(ologies)37 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(163)143 2306 y(6.3)g(Em)o(b)q(edding)16 b(in)g(MPI)22 b Fw(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(164)143 2363 y(6.4)g(Ov)o(erview)16 b(of)e(the)i(prop)q(osed)f (MPI)g(functions)20 b Fw(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(164)143 2420 y(6.5)g(T)l(op)q(ology)15 b(Constructors)20 b Fw(:)i(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(165)248 2476 y(6.5.1)k(T)l(op)q (ology)15 b(inquiry)i(functions)42 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(168)248 2533 y(6.5.2)k(Cartesian)15 b(shift)h(co)q(ordinates)43 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(173)248 2590 y(6.5.3)k(P)o (artitioning)16 b(of)e(Cartesian)h(structures)45 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(173)248 2647 y(6.5.4)k(Lo)o(w{lev)o(el)16 b(top)q(ology)f (functions)37 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(174)143 2704 y(6.6)g(A)15 b(simple)i(example)f Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(176)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 6 5 5 bop 75 45 a Fx(7)42 b(MPI)17 b(En)o(vironmen)o(tal)g(Managemen)o(t) 883 b(178)143 102 y FD(7.1)46 b(Implemen)o(tation)16 b(information)22 b Fw(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(178)248 158 y(7.1.1)k(En)o(vironmen)o(tal)16 b(Inquiries)28 b Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(178)248 214 y(7.1.2)k(Bu\013ering)42 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b FD(180)143 271 y(7.2)g(Error)14 b(handling)41 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FD(185)143 327 y(7.3)g(Error)14 b(co)q(des)34 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(187)143 384 y(7.4)g(Timers)18 b Fw(:)k(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FD(188)143 440 y(7.5)g(Startup)40 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FD(189)75 542 y Fx(8)c(Pro\014ling)18 b(In)o(terface)1238 b(191)143 599 y FD(8.1)46 b(Requiremen)o(ts)28 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FD(191)143 655 y(8.2)g(Discussion)21 b Fw(:)i(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(191)143 712 y(8.3)g(Logic)16 b(of)e(the)i(design)29 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(192)143 768 y(8.4)g(Examples)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(192)248 824 y(8.4.1)k(Pro\014ler)16 b(implemen)o(tation)35 b Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(192)248 881 y(8.4.2)k(MPI)15 b(library)h(implemen)o(tation)h Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FD(193)248 937 y(8.4.3)k(Complications)e Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(194)143 994 y(8.5)g(Multiple)17 b(lev)o(els)f(of)f(in)o (terception)25 b Fw(:)d(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(195)75 1096 y Fx(9)c(Initial)20 b(Implemen)o(tation)e(Subset)967 b(196)143 1152 y FD(9.1)46 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FD(196)143 1209 y(9.2)g(Criteria)15 b(and)h(Rationale)28 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(196)143 1265 y(9.3)g(Subset)15 b(F)l(unctionalit)o(y)31 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(197)248 1322 y(9.3.1)k(P)o(oin)o(t)15 b(to)g(P)o(oin)o(t)f(F)l (unctionalit)o(y)i Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(197)248 1378 y(9.3.2)k(Collectiv)o(e)17 b(Comm)o(unication)e(F)l(unctionalit)o (y)24 b Fw(:)e(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)46 b FD(199)248 1434 y(9.3.3)k(Groups,)15 b(Con)o(texts,)e(Comm)o (unicators,)h(and)i(Cac)o(heing)f Fw(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)46 b FD(199)248 1491 y(9.3.4)k(T)l(op)q(ology)15 b(F)l(unctionalit)o(y)46 b Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(200)248 1547 y(9.3.5)k(Language)15 b(Binding)41 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(201)248 1604 y(9.3.6)k(MPI)15 b(En)o(vironmen)o(tal)h(Managemen)o(t)f Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FD(201)248 1660 y(9.3.7)k(Pro\014ling)16 b(F)l(unctionalit)o(y)24 b Fw(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(201)143 1717 y(9.4)g(Subset)15 b(T)l(esting)42 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b FD(202)75 1819 y Fx(Bibliograph)o(y)1430 b(203)75 1921 y(A)28 b(Language)19 b(Binding)1242 b(205)143 1977 y FD(A.1)35 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(205)143 2033 y(A.2)35 b(De\014ned)16 b(Constan)o(ts)e(for)g(C)h(and)h(F)l (ortran)21 b Fw(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(205)143 2090 y(A.3)35 b(C)15 b(bindings)i(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o (unication)21 b Fw(:)i(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FD(207)143 2146 y(A.4)35 b(C)15 b(Bindings)i(for)d (Collectiv)o(e)j(Comm)o(unication)34 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(210)143 2203 y(A.5)35 b(C)15 b(Bindings)i(for)d(Groups,)h(Con)o (texts,)f(and)h(Comm)o(unicators)26 b Fw(:)c(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FD(211)143 2259 y(A.6)35 b(C)15 b(Bindings)i(for)d(Pro)q(cess)h(T)l(op)q(ologies)43 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(212)143 2316 y(A.7)35 b(C)15 b(bindings)i(for)d(En)o(vironmen)o(tal)i(Inquiry)42 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FD(213)143 2372 y(A.8)35 b(F)l(ortran)14 b(Bindings)j(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o(unication)39 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(213)143 2429 y(A.9)35 b(F)l(ortran)14 b(Bindings)j(for)d (Collectiv)o(e)j(Comm)o(unication)23 b Fw(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(216)143 2485 y(A.10)12 b(F)l(ortran)i(Bindings)j(for)d(Groups,)h(Con)o(texts,)f(and) h(Comm)o(unicators)g Fw(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(219)143 2542 y(A.11)12 b(F)l(ortran)i(Bindings)j(for)d(Pro)q(cess) h(T)l(op)q(ologies)32 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FD(220)143 2598 y(A.12)12 b(F)l(ortran)i(Bindings)j(for)d(En)o(vironmen)o(tal)i (Inquiry)24 b Fw(:)e(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FD(221)75 2700 y Fx(MPI)17 b(F)l(unction)h(Index) 1252 b(223)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 7 6 6 bop 75 75 a Fy(Ac)m(kno)m(wledgmen)m(ts)166 282 y FD(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f(w)o(as)e(carried)i(out)f (b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o(as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d) h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P)o(assing)g(In)o (terface)75 395 y(\(MPI\),)f(man)o(y)h(p)q(eople)i(serv)o(ed)e(in)h(p)q (ositions)g(of)f(resp)q(onsibilit)o(y:)143 483 y FA(\017)23 b FD(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o(er,)g(Con)o(v)o (eners)g(and)g(Meeting)h(Chairs)143 572 y FA(\017)23 b FD(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes)143 661 y FA(\017)23 b FD(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i (Lusk,)f(P)o(oin)o(t)g(to)g(P)o(oin)o(t)f(Comm)o(unications)143 749 y FA(\017)23 b FD(Al)16 b(Geist,)e(Marc)h(Snir,)h(Stev)o(e)f(Otto,) f(Collectiv)o(e)j(Comm)o(unications)143 838 y FA(\017)23 b FD(Rolf)15 b(Hemp)q(el,)i(Pro)q(cess)e(T)l(op)q(ologies)143 927 y FA(\017)23 b FD(Ewing)15 b(Lusk,)h(Language)f(Binding)143 1015 y FA(\017)23 b FD(William)17 b(Gropp,)d(En)o(vironmen)o(tal)i (Managemen)o(t)e(and)h(Inquiry)143 1104 y FA(\017)23 b FD(James)15 b(Co)o(wnie,)g(Pro\014ling)143 1193 y FA(\017)23 b FD(T)l(on)o(y)16 b(Skjellum,)j(Lyndon)f(Clark)o(e,)f(Marc)g(Snir,)h (Ric)o(hard)g(Little\014eld,)h(Mark)e(Sears,)g(Groups,)189 1249 y(Con)o(texts,)c(and)j(Comm)o(unicators)143 1338 y FA(\017)23 b FD(Stev)o(en)15 b(Huss-Lederman,)h(Initial)h(Implemen)o (tation)f(Subset)143 1427 y FA(\017)23 b FD(Stev)o(e)15 b(Otto,)f(Editor)166 1515 y(The)22 b(follo)o(wing)h(list)f(includes)i (some)e(of)f(the)h(activ)o(e)g(participan)o(ts)h(in)f(the)g(MPI)g(pro)q (cess)g(not)75 1572 y(men)o(tioned)16 b(ab)q(o)o(v)o(e:)100 1705 y(Ed)f(Anderson)134 b(Jo)q(e)16 b(Baron)171 b(Eric)15 b(Barszcz)162 b(Scott)15 b(Berryman)100 1762 y(Rob)g(Bjornson)119 b(Anne)16 b(Elster)138 b(Jim)16 b(F)l(eeney)185 b(Vince)17 b(F)l(ernando)100 1818 y(Sam)e(Fineb)q(erg)118 b(Jon)16 b(Flo)o(w)o(er)154 b(Daniel)16 b(F)l(ry)o(e)179 b(Ian)16 b(Glendinning)100 1875 y(Adam)f(Green)o(b)q(erg)53 b(Rob)q(ert)16 b(Harrison)50 b(Leslie)17 b(Hart)189 b(T)l(om)15 b(Haupt)100 1931 y(Don)g(Heller)179 b(T)l(om)15 b(Henderson)62 b(Alex)16 b(Ho)246 b(C.T.)14 b(Ho)o(w)o(ard)g(Ho)100 1988 y(John)i(Kap)q(enga)106 b(Bob)16 b(Leary)166 b(Arth)o(ur)14 b(Maccab)q(e)84 b(P)o(eter)15 b(Madams)100 2044 y(Alan)h(Main)o(w)o(aring)49 b(Oliv)o(er)17 b(McBry)o(an)55 b(Phil)16 b(McKinley)123 b(Charles)16 b(Mosher)100 2100 y(Dan)f(Nessett)149 b(P)o(eter)15 b(P)o(ac)o(heco)94 b(Ho)o(w)o(ard)14 b(P)o(almer)100 b(P)o(aul)15 b(Pierce)100 2157 y(Sanja)o(y)g(Rank)m(a)120 b(P)o(eter)15 b(Rigsb)q(ee)102 b(Arc)o(h)15 b(Robison)139 b(Eric)o(h)16 b(Sc)o(hikuta)100 2213 y(Am)o(buj)f(Singh)133 b(Alan)16 b(Sussman)93 b(Rob)q(ert)15 b(T)l(omlinson)51 b(Rob)q(ert)16 b(G.)e(V)l(oigt)100 2270 y(Dennis)i(W)l(eeks)117 b(Stephen)17 b(Wheat)67 b(Stev)o(e)15 b(Zenith)166 2412 y(Oak)21 b(Ridge)h(National)g(Lab)q (oratory)e(made)h(the)g(draft)f(a)o(v)m(ailable)j(b)o(y)e(anon)o(ymous) g(FTP)f(mail)75 2469 y(serv)o(ers)15 b(and)g(w)o(as)g(instrumen)o(tal)g (in)h(distributing)h(the)e(do)q(cumen)o(t.)166 2525 y(MPI)22 b(op)q(erated)h(on)f(a)g(v)o(ery)g(tigh)o(t)g(budget)h(\(in)g(realit)o (y)l(,)h(it)f(had)f(no)g(budget)h(when)g(the)f(\014rst)75 2582 y(meeting)14 b(w)o(as)e(announced\).)20 b(D)o(ARP)l(A)13 b(and)g(NSF)g(ha)o(v)o(e)g(supp)q(orted)g(researc)o(h)g(at)f(v)m (arious)i(institutions)75 2638 y(that)i(ha)o(v)o(e)g(made)h(a)f(con)o (tribution)i(to)o(w)o(ards)d(tra)o(v)o(el)h(for)g(the)h(U.S.)f (academics.)25 b(Supp)q(ort)17 b(for)f(sev)o(eral)75 2695 y(Europ)q(ean)g(participan)o(ts)f(w)o(as)g(pro)o(vided)h(b)o(y)f (ESPRIT.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 1 8 1 7 bop 75 362 a Fr(Chapter)34 b(1)75 576 y Fy(In)m(tro)s(duction)41 b(to)g(MPI)75 823 y Fq(1.1)59 b(Overview)19 b(and)g(Goals)75 936 y FD(Message)11 b(passing)i(is)f(a)g(paradigm)f(used)i(widely)g(on) f(certain)h(classes)f(of)f(parallel)j(mac)o(hines;)f(esp)q(ecially)75 992 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1049 y(pro)q(cesses)j(comm)o(unicating)h(through)e(messages)h(is)g (w)o(ell)h(understo)q(o)q(d.)34 b(Ov)o(er)20 b(the)g(last)g(ten)g(y)o (ears,)75 1105 y(substan)o(tial)11 b(progress)f(has)g(b)q(een)h(made)g (in)g(casting)f(signi\014can)o(t)i(applications)g(in)f(this)g (paradigm.)18 b(Eac)o(h)75 1161 y(v)o(endor)13 b(has)g(implemen)o(ted)i (its)e(o)o(wn)g(v)m(arian)o(t.)19 b(More)13 b(recen)o(tly)l(,)h(sev)o (eral)f(systems)g(ha)o(v)o(e)f(demonstrated)75 1218 y(that)k(a)h (message)f(passing)h(system)g(can)g(b)q(e)g(e\016cien)o(tly)h(and)f(p)q (ortably)h(implemen)o(ted.)26 b(It)17 b(is)g(th)o(us)g(an)75 1274 y(appropriate)c(time)g(to)f(try)h(to)f(de\014ne)i(b)q(oth)f(the)g (syn)o(tax)f(and)h(seman)o(tics)g(of)g(a)g(core)f(of)h(library)h (routines)75 1331 y(that)i(will)i(b)q(e)f(useful)h(to)e(a)g(wide)h (range)f(of)h(users)f(and)h(e\016cien)o(tly)h(implemen)o(table)g(on)f (a)f(wide)h(range)75 1387 y(of)e(computers.)166 1450 y(In)21 b(designing)i(MPI)e(w)o(e)f(ha)o(v)o(e)h(sough)o(t)f(to)g(mak)o (e)g(use)h(of)g(the)g(most)f(attractiv)o(e)f(features)i(of)f(a)75 1506 y(n)o(um)o(b)q(er)12 b(of)g(existing)h(message)e(passing)h (systems,)g(rather)f(than)h(selecting)h(one)f(of)g(them)g(and)g (adopting)75 1563 y(it)f(as)f(the)h(standard.)18 b(Th)o(us,)11 b(MPI)g(has)f(b)q(een)i(strongly)f(in\015uenced)i(b)o(y)d(w)o(ork)g(at) g(the)h(IBM)g(T.)f(J.)h(W)l(atson)75 1619 y(Researc)o(h)18 b(Cen)o(ter)g([1)o(,)g(2)o(],)g(In)o(tel's)g(NX/2)g([20)o(],)f(Express) h([19)o(],)g(nCUBE's)g(V)l(ertex)g([18)o(],)f(P4)h([6)o(],)g(and)75 1675 y(P)l(ARMA)o(CS)d([5,)f(7].)19 b(Other)c(imp)q(ortan)o(t)g(con)o (tributions)h(ha)o(v)o(e)e(come)h(from)f(Zip)q(co)q(de)j([21)o(,)d (22],)g(Chimp)75 1732 y([12)o(,)h(13)o(],)g(PVM)g([4)o(,)g(10)o(],)g (and)g(PICL)h([16)o(].)166 1794 y(The)d(MPI)g(standardization)g (e\013ort)f(in)o(v)o(olv)o(ed)i(ab)q(out)f(60)f(p)q(eople)i(from)f(40)f (organizations)h(mainly)75 1851 y(from)k(the)g(United)i(States)e(and)h (Europ)q(e.)27 b(Most)16 b(of)h(the)h(ma)s(jor)e(v)o(endors)h(of)g (concurren)o(t)h(computers)75 1907 y(w)o(ere)10 b(in)o(v)o(olv)o(ed)i (in)f(MPI,)g(along)f(with)i(researc)o(hers)e(from)g(univ)o(ersities,)j (go)o(v)o(ernmen)o(t)d(lab)q(oratories,)h(and)75 1964 y(industry)l(.)21 b(The)14 b(standardization)h(pro)q(cess)f(b)q(egan)h (with)f(the)h(W)l(orkshop)f(on)g(Standards)g(for)g(Message)75 2020 y(P)o(assing)j(in)i(a)e(Distributed)h(Memory)f(En)o(vironmen)o(t,) h(sp)q(onsored)g(b)o(y)f(the)h(Cen)o(ter)f(for)g(Researc)o(h)h(on)75 2077 y(P)o(arallel)11 b(Computing,)g(held)h(April)f(29-30,)f(1992,)g (in)h(Williamsburg,)h(Virginia)g([23)o(].)18 b(A)o(t)9 b(this)i(w)o(orkshop)75 2133 y(the)17 b(basic)h(features)f(essen)o (tial)h(to)f(a)g(standard)g(message)g(passing)g(in)o(terface)h(w)o(ere) f(discussed,)i(and)e(a)75 2190 y(w)o(orking)e(group)g(established)i(to) d(con)o(tin)o(ue)i(the)f(standardization)h(pro)q(cess.)166 2252 y(A)e(preliminary)i(draft)e(prop)q(osal,)g(kno)o(wn)g(as)g(MPI1,)g (w)o(as)f(put)h(forw)o(ard)f(b)o(y)h(Dongarra,)f(Hemp)q(el,)75 2308 y(Hey)l(,)i(and)g(W)l(alk)o(er)f(in)i(No)o(v)o(em)o(b)q(er)e (1992,)f(and)i(a)f(revised)i(v)o(ersion)f(w)o(as)f(completed)h(in)h(F)l (ebruary)e(1993)75 2365 y([11)o(].)19 b(MPI1)c(em)o(b)q(o)q(died)h(the) f(main)h(features)e(that)g(w)o(ere)h(iden)o(ti\014ed)i(at)d(the)h (Williamsburg)i(w)o(orkshop)75 2421 y(as)g(b)q(eing)h(necessary)g(in)g (a)f(message)f(passing)i(standard.)26 b(This)17 b(prop)q(osal)h(w)o(as) e(in)o(tended)j(to)e(initiate)75 2478 y(discussion)i(of)e (standardization)h(issues)h(within)f(the)g(distributed)h(memory)e (concurren)o(t)g(computing)75 2534 y(comm)o(unit)o(y)l(.)30 b(Since)20 b(MPI1)e(w)o(as)g(primarily)i(in)o(tended)g(to)e(promote)f (discussion)k(and)d(\\get)g(the)h(ball)75 2591 y(rolling,")h(it)f(fo)q (cused)g(mainly)g(on)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unications.) 31 b(MPI1)18 b(did)h(not)f(include)j(an)o(y)75 2647 y(collectiv)o(e)e (comm)o(unication)e(routines.)26 b(MPI1)16 b(brough)o(t)g(to)h(the)g (forefron)o(t)e(a)i(n)o(um)o(b)q(er)g(of)f(imp)q(ortan)o(t)75 2704 y(standardization)e(issues;)h(ho)o(w)o(ev)o(er,)d(its)i(ma)s(jor)e (de\014ciency)k(w)o(as)d(that)f(the)i(managemen)o(t)f(of)g(resources) -32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 9 2 8 bop 75 -100 a FD(2)903 b Fp(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)75 45 y FD(is)h(not)f(thread-safe.)166 102 y(In)j(No)o(v)o(em)o(b)q(er)e(1992,)g(a)h(meeting)h(of)e(the)h(MPI)g(w) o(orking)g(group)g(w)o(as)f(held)i(in)g(Minneap)q(olis,)i(at)75 158 y(whic)o(h)15 b(it)g(w)o(as)e(decided)k(to)c(place)j(the)e (standardization)h(pro)q(cess)g(on)f(a)g(more)g(formal)g(fo)q(oting,)g (and)h(to)75 214 y(generally)i(adopt)f(the)g(pro)q(cedures)i(and)e (organization)g(of)g(the)g(High)h(P)o(erformance)f(F)l(ortran)f(F)l (orum.)75 271 y(Sub)q(committees)i(w)o(ere)e(formed)g(for)g(the)h(ma)s (jor)e(comp)q(onen)o(t)i(areas)f(of)g(the)h(standard,)f(and)g(an)h (email)75 327 y(discussion)23 b(service)f(established)g(for)f(eac)o(h.) 37 b(In)22 b(addition,)h(the)e(goal)g(of)g(pro)q(ducing)h(a)f(draft)f (MPI)75 384 y(standard)f(b)o(y)h(the)f(F)l(all)h(of)g(1993)e(w)o(as)h (set.)32 b(T)l(o)19 b(ac)o(hiev)o(e)i(this)f(goal)f(the)g(MPI)h(w)o (orking)f(group)g(met)75 440 y(ev)o(ery)d(6)f(w)o(eeks)g(for)g(t)o(w)o (o)g(da)o(ys)g(throughout)g(the)h(\014rst)f(9)g(mon)o(ths)g(of)h(1993,) e(and)i(presen)o(ted)g(the)g(draft)75 497 y(MPI)j(standard)g(at)g(the)g (Sup)q(ercomputing)i(93)e(conference)h(in)g(No)o(v)o(em)o(b)q(er)f (1993.)31 b(These)20 b(meetings)75 553 y(and)f(the)f(email)h (discussion)i(together)c(constituted)i(the)g(MPI)f(F)l(orum,)g(mem)o(b) q(ership)i(of)e(whic)o(h)h(has)75 610 y(b)q(een)d(op)q(en)g(to)f(all)h (mem)o(b)q(ers)f(of)g(the)g(high)h(p)q(erformance)g(computing)f(comm)o (unit)o(y)l(.)166 666 y(The)j(main)h(adv)m(an)o(tages)e(of)h (establishing)i(a)e(message)g(passing)g(standard)g(are)g(p)q(ortabilit) o(y)h(and)75 723 y(ease-of-use.)h(In)14 b(a)g(distributed)h(memory)e (comm)o(unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f(higher)h (lev)o(el)75 779 y(routines)c(and/or)g(abstractions)f(are)h(build)i(up) q(on)e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i(routines)f(the)g(b) q(ene\014ts)75 835 y(of)22 b(standardization)h(are)f(particularly)h (apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g (message)75 892 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop)q (osed)g(here,)h(pro)o(vides)f(v)o(endors)h(with)f(a)g(clearly)h (de\014ned)h(base)75 948 y(set)10 b(of)g(routines)g(that)g(they)g(can)h (implemen)o(t)g(e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o (vide)f(hardw)o(are)g(supp)q(ort)75 1005 y(for,)k(thereb)o(y)h (enhancing)i(scalabilit)o(y)l(.)166 1061 y(The)h(goal)g(of)g(the)g (Message)g(P)o(assing)g(In)o(terface)g(simply)i(stated)d(is)i(to)f(dev) o(elop)h(a)f(widely)h(used)75 1118 y(standard)e(for)f(writing)i (message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface) g(should)h(establishing)75 1174 y(a)d(practical,)h(p)q(ortable,)f (e\016cien)o(t,)g(and)h(\015exible)h(standard)e(for)f(message)h (passing.)166 1231 y(A)g(complete)h(list)g(of)f(goals)g(follo)o(ws.)143 1330 y FA(\017)23 b FD(Design)11 b(an)f(application)i(programming)e(in) o(terface)h(\(not)f(necessarily)i(for)d(compilers)j(or)e(a)g(system)189 1386 y(implemen)o(tation)16 b(library\).)143 1478 y FA(\017)23 b FD(Allo)o(w)15 b(e\016cien)o(t)h(comm)o(unication:)21 b(Av)o(oid)16 b(memory)e(to)h(memory)g(cop)o(ying)g(and)h(allo)o(w)f(o) o(v)o(erlap)189 1535 y(of)c(computation)h(and)g(comm)o(unication)h(and) f(o\017oad)f(to)g(comm)o(unication)i(co-pro)q(cessor,)f(where)189 1591 y(a)o(v)m(ailable.)143 1683 y FA(\017)23 b FD(Allo)o(w)15 b(for)g(implemen)o(tation)h(that)f(can)g(b)q(e)h(used)g(in)g(a)f (heterogeneous)g(en)o(vironmen)o(t.)143 1775 y FA(\017)23 b FD(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C)f(and)h(F)l(ortran)e(77)g (bindings)j(for)e(the)g(in)o(terface.)143 1867 y FA(\017)23 b FD(Assume)15 b(a)g(reliable)i(comm)o(unication)f(in)o(terface:)k (User)15 b(need)h(not)f(cop)q(e)h(with)f(comm)o(unication)189 1923 y(failures.)21 b(Suc)o(h)15 b(failures)i(are)d(dealt)i(with)g(b)o (y)f(the)g(underlying)i(comm)o(unication)f(subsystem.)143 2015 y FA(\017)23 b FD(De\014ne)f(an)g(in)o(terface)g(that)f(is)h(not)f (to)q(o)g(di\013eren)o(t)h(from)f(curren)o(t)h(practice,)h(suc)o(h)f (as)g(PVM,)189 2071 y(Express,)14 b(P4,)h(etc)g(and)h(pro)o(vides)f (extension)h(that)f(allo)o(w)g(greater)f(\015exibili)q(t)o(y)l(.)143 2163 y FA(\017)23 b FD(De\014ne)d(an)f(in)o(terface)h(that)e(can)i(b)q (e)g(implemen)o(ted)h(on)e(man)o(y)g(v)o(endor's)g(platforms,)g(with)h (no)189 2220 y(signi\014can)o(t)c(c)o(hanges)f(in)h(the)f(underlying)i (comm)o(unication)f(and)g(system)e(soft)o(w)o(are.)143 2312 y FA(\017)23 b FD(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h (should)g(b)q(e)g(language)f(indep)q(enden)o(t.)143 2403 y FA(\017)23 b FD(The)15 b(in)o(terface)h(should)g(b)q(e)g(designed)g (to)f(allo)o(w)g(for)g(thread-safet)o(y)l(.)75 2546 y Fq(1.2)59 b(Who)20 b(Should)g(Use)g(This)f(Standa)n(rd?)75 2647 y FD(This)d(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h(all) g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 2704 y(programs)9 b(in)j(F)l(ortran)d(77)i(and)f (C.)h(This)g(includes)i(individual)h(application)e(programmers,)e(dev)o (elop)q(ers)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 10 3 9 bop 75 -100 a Fp(1.3.)34 b(WHA)l(T)15 b(PLA)l(TF)o(ORMS)h(ARE)g(T)l (AR)o(GETS)f(F)o(OR)g(IMPLEMENT)l(A)l(TION?)296 b FD(3)75 45 y(of)17 b(soft)o(w)o(are)e(designed)j(to)f(run)g(on)g(parallel)i (mac)o(hines,)f(and)f(creators)f(of)h(en)o(vironmen)o(ts)g(and)g(to)q (ols.)75 102 y(In)g(order)f(to)f(b)q(e)i(attractiv)o(e)e(to)h(this)g (wide)i(audience,)f(the)g(standard)e(m)o(ust)h(pro)o(vide)h(a)e (simple,)j(easy-)75 158 y(to-use)f(in)o(terface)g(for)f(the)g(basic)i (user)f(while)h(not)e(seman)o(tically)i(precluding)h(the)e(high-p)q (erformance)75 214 y(message-passing)e(op)q(erations)h(a)o(v)m(ailable) h(on)e(adv)m(anced)h(mac)o(hines.)75 359 y Fq(1.3)59 b(What)20 b(Platfo)n(rms)g(Are)g(T)-5 b(a)n(rgets)19 b(F)n(o)n(r)i(Implementat)o(ion?)75 461 y FD(The)c(attractiv)o(eness)f (of)g(the)h(message-passing)g(paradigm)f(at)g(least)h(partially)h (stems)e(from)g(its)h(wide)75 518 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o(y)f(ma)o(y)g(run)h(on)g (distributed-memory)h(m)o(ultipro)q(cessors,)75 574 y(net)o(w)o(orks)i (of)i(w)o(orkstations,)e(and)i(com)o(binations)g(of)g(all)g(of)f (these.)34 b(In)20 b(addition,)h(shared-memory)75 631 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 687 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 744 y(th)o(us)d(should)i(b)q(e)f(b)q(oth)g (p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f(on) h(a)f(great)g(v)m(ariet)o(y)g(of)75 800 y(mac)o(hines,)i(including)h (those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h(of)e(other)h (mac)o(hines,)g(parallel)75 857 y(or)d(not,)f(connected)i(b)o(y)f(a)g (comm)o(unication)h(net)o(w)o(ork.)166 913 y(The)21 b(in)o(terface)f (is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i(general)f(MIMD)f (programs,)g(as)g(w)o(ell)h(as)f(those)75 970 y(written)d(in)g(the)g (more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g(Although)h(no)f(explicit)j (supp)q(ort)e(for)f(threads)g(is)75 1026 y(pro)o(vided,)e(the)f(in)o (terface)h(has)f(b)q(een)h(designed)g(so)f(as)g(not)f(to)h(prejudice)i (their)e(use.)20 b(With)13 b(this)h(v)o(ersion)75 1083 y(of)h(MPI)g(no)g(supp)q(ort)g(is)h(pro)o(vided)g(for)f(dynamic)h(spa)o (wning)f(of)g(tasks.)75 1228 y Fq(1.4)59 b(What)20 b(Is)f(Included)g (In)g(The)g(Standa)n(rd?)75 1330 y FD(The)c(standard)g(includes:)143 1437 y FA(\017)23 b FD(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 1532 y FA(\017)23 b FD(Collectiv)o(e)16 b(op)q(erations)143 1627 y FA(\017)23 b FD(Pro)q(cess)15 b(groups)143 1722 y FA(\017)23 b FD(Comm)o(unication)15 b(con)o(texts)143 1817 y FA(\017)23 b FD(Pro)q(cess)15 b(top)q(ology)143 1912 y FA(\017)23 b FD(Bindings)17 b(for)d(F)l(ortran)g(77)h(and)g(C)143 2007 y FA(\017)23 b FD(En)o(vironmen)o(tal)15 b(Managemen)o(t)f(and)i(inquiry)143 2102 y FA(\017)23 b FD(Pro\014ling)16 b(in)o(terface)143 2197 y FA(\017)23 b FD(Subset)15 b(sp)q(eci\014cation)75 2342 y Fq(1.5)59 b(What)20 b(Is)f(Not)h(Included)e(In)i(The)f(Standa)n (rd?)75 2444 y FD(The)c(standard)g(do)q(es)h(not)f(sp)q(ecify:)143 2552 y FA(\017)23 b FD(Explicit)17 b(shared-memory)e(op)q(erations)143 2647 y FA(\017)23 b FD(Op)q(erations)c(that)e(require)i(more)f(op)q (erating)h(system)f(supp)q(ort)g(than)g(is)h(curren)o(tly)g(standard;) 189 2704 y(for)14 b(example,)i(in)o(terrupt-driv)o(en)g(receiv)o(es,)g (remote)f(execution,)h(or)e(activ)o(e)i(messages)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 11 4 10 bop 75 -100 a FD(4)903 b Fp(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION) 17 b(TO)e(MPI)143 45 y FA(\017)23 b FD(Program)13 b(construction)j(to)q (ols)143 135 y FA(\017)23 b FD(Debugging)15 b(facilities)143 225 y FA(\017)23 b FD(Auxiliary)17 b(functions)e(suc)o(h)h(as)f(timers) 143 315 y FA(\017)23 b FD(Explicit)17 b(supp)q(ort)e(for)g(threads)143 405 y FA(\017)23 b FD(Supp)q(ort)15 b(for)g(task)f(managemen)o(t)143 494 y FA(\017)23 b FD(I/O)15 b(functions)166 587 y(There)g(are)f(man)o (y)h(features)f(that)g(ha)o(v)o(e)g(b)q(een)i(considered)g(and)f(not)g (included)i(in)f(this)f(standard.)75 643 y(This)j(happ)q(ened)h(for)e (a)g(n)o(um)o(b)q(er)h(of)f(reasons,)g(one)g(of)g(whic)o(h)i(is)f(the)f (time)h(constrain)o(t)f(that)g(w)o(as)f(self)75 700 y(imp)q(osed)g(in)h (\014nishing)g(the)e(standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g (included)j(can)e(alw)o(a)o(ys)e(b)q(e)i(o\013ered)f(as)75 756 y(extensions)i(b)o(y)f(sp)q(eci\014c)j(implemen)o(tations.)24 b(P)o(erhaps)17 b(future)f(v)o(ersions)g(of)g(MPI)h(will)h(address)e (some)75 813 y(of)f(these)g(issues.)75 954 y Fq(1.6)59 b(Organization)20 b(of)g(this)f(Do)r(cument)75 1056 y FD(The)f(follo)o(wing)h(is)f(a)g(list)g(of)g(the)g(c)o(hapters)f(in)i (this)f(do)q(cumen)o(t,)h(along)f(with)g(a)f(brief)i(description)g(of) 75 1112 y(eac)o(h.)143 1205 y FA(\017)k FD(Chapter)c(2,)i Fs(MPI)e(T)l(erms)f(and)i(Conventions)p FD(,)j(explains)e(notational)f (terms)g(and)g(con)o(v)o(en)o(tions)189 1261 y(used)15 b(throughout)g(the)g(MPI)g(do)q(cumen)o(t.)143 1351 y FA(\017)23 b FD(Chapter)16 b(3,)g Fs(P)o(oint)h(to)f(P)o(oint)h(Comm)m (unication)p FD(,)e(de\014nes)i(the)g(basic,)g(pairwise)g(comm)o (unication)189 1407 y(subset)c(of)h(MPI.)f Fo(send)g FD(and)h Fo(r)n(e)n(c)n(eive)e FD(are)h(found)h(here,)g(along)g(with)g (man)o(y)f(asso)q(ciated)h(functions)189 1464 y(designed)i(to)f(mak)o (e)f(basic)i(comm)o(unication)g(p)q(o)o(w)o(erful)g(and)f(e\016cien)o (t.)143 1554 y FA(\017)23 b FD(Chapter)12 b(4,)h Fs(Collective)g(Com)o (m)n(unications)p FD(,)e(de\014nes)j(pro)q(cess-group)f(collectiv)o(e)h (comm)o(unication)189 1610 y(op)q(erations.)28 b(W)l(ell)19 b(kno)o(wn)f(examples)h(of)e(this)i(are)e(barrier)i(and)f(broadcast)f (o)o(v)o(er)g(a)h(group)f(of)189 1667 y(pro)q(cesses)e(\(not)g (necessarily)h(all)g(the)g(pro)q(cesses\).)143 1756 y FA(\017)23 b FD(Chapter)16 b(5,)g Fs(Groups,)h(Contexts,)i(and)e(Comm)m (unicato)o(rs)p FD(,)d(sho)o(ws)i(ho)o(w)g(groups)g(of)g(pro)q(cesses)h (are)189 1813 y(formed)c(and)h(manipulated,)i(ho)o(w)d(unique)i(comm)o (unication)g(con)o(texts)e(are)h(obtained,)g(and)g(ho)o(w)189 1869 y(the)h(t)o(w)o(o)f(are)h(b)q(ound)h(together)e(in)o(to)h(a)g Fo(c)n(ommunic)n(ator)p FD(.)143 1959 y FA(\017)23 b FD(Chapter)18 b(6,)h Fs(Pro)q(cess)h(T)l(op)q(ologies)p FD(,)f(explains)h(a)f(set)f(of)h(utilit)o(y)h(functions)f(mean)o(t)f (to)h(assist)f(in)189 2016 y(the)13 b(mapping)i(of)e(pro)q(cess)g (groups)h(\(a)f(linearly)i(ordered)f(set\))f(to)g(ric)o(her)h(top)q (ological)g(structures)189 2072 y(suc)o(h)h(as)g(m)o(ulti-dimensional)j (grids.)143 2162 y FA(\017)23 b FD(Chapter)10 b(7,)h Fs(MPI)g(Environmental)e(Management)p FD(,)g(explains)j(ho)o(w)f(the)g (programmer)e(can)i(manage)189 2219 y(and)22 b(mak)o(e)f(inquiries)j (of)d(the)h(curren)o(t)f(MPI)h(en)o(vironmen)o(t.)40 b(These)22 b(functions)g(are)g(needed)189 2275 y(for)g(the)h(writing)g (of)f(correct,)i(robust)e(programs,)i(and)f(are)f(esp)q(ecially)j(imp)q (ortan)o(t)e(for)f(the)189 2331 y(construction)15 b(of)g(highly-p)q (ortable)i(message-passing)f(programs.)143 2421 y FA(\017)23 b FD(Chapter)c(8,)i Fs(Pro\014ling)f(Interface)p FD(,)h(explains)g(a)f (simple)i(name)e(shifting)h(con)o(v)o(en)o(tion)f(that)f(an)o(y)189 2478 y(MPI)i(implemen)o(tation)i(m)o(ust)e(supp)q(ort.)38 b(One)22 b(motiv)m(ation)g(for)f(this)h(is)g(the)f(abilit)o(y)i(to)e (put)189 2534 y(p)q(erformance)c(pro\014ling)i(calls)f(in)o(to)g(MPI)f (without)h(the)f(need)i(for)d(access)i(to)f(the)g(MPI)h(source)189 2591 y(co)q(de.)31 b(The)19 b(name)g(shift)g(is)h(merely)f(an)g(in)o (terface,)h(it)f(sa)o(ys)f(nothing)i(ab)q(out)e(ho)o(w)h(the)g(actual) 189 2647 y(pro\014ling)h(should)g(b)q(e)f(done)h(and)f(in)h(fact,)f (the)g(name)g(shift)g(can)g(usefully)i(b)q(e)e(used)h(for)e(other)189 2704 y(purp)q(oses.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 12 5 11 bop 75 -100 a Fp(1.6.)34 b(OR)o(GANIZA)l(TION)17 b(OF)e(THIS)h(DOCUMENT)810 b FD(5)143 45 y FA(\017)23 b FD(Chapter)10 b(9,)h Fs(Initial)f(Implem)m(entation)f(Subset)p FD(,)k(suggests)d(to)g(MPI)h(implemen)o(tors)g(a)f(\\core")g(subset)189 102 y(of)16 b(MPI)h(that)g(is)g(useful,)h(in)o(ternally)h(consisten)o (t,)e(and)g(should)h(app)q(ear)g(\014rst)e(in)i(the)f(ev)o(olution)189 158 y(of)d(an)h(MPI)g(implemen)o(tation.)21 b(The)15 b(subset)g(is)h(de\014ned)g(so)e(that)h(consisten)o(t)g(implemen)o (tations)189 214 y(can)d(app)q(ear)g(rapidly)l(,)i(and)e(p)q(ortable)h (parallel)h(programming)d(with)i(MPI)f(can)g(start)f(in)i(a)f(timely) 189 271 y(manner.)143 365 y FA(\017)23 b FD(Annex)18 b(A,)f Fs(Language)g(Bindings)p FD(,)i(giv)o(es)f(sp)q(eci\014c)h(syn)o (tax)d(in)i(F)l(ortran)e(77)h(and)h(C,)e(for)h(all)h(MPI)189 421 y(functions,)d(constan)o(ts,)f(and)h(t)o(yp)q(es.)143 515 y FA(\017)23 b FD(The)g Fs(MPI)g(F)o(unction)i(Index)f FD(is)g(a)g(simple)h(index)f(sho)o(wing)g(the)g(lo)q(cation)g(of)f(the) h(language-)189 571 y(indep)q(enden)o(t)17 b(de\014nition)g(of)e(eac)o (h)g(MPI)g(function.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 13 6 12 bop 75 356 a Fr(Chapter)34 b(2)75 564 y Fy(MPI)40 b(T)-10 b(erms)39 b(and)g(Con)m(v)m(en)m(tions)75 786 y FD(This)11 b(c)o(hapter)g(explains)h(notational)e(terms)g(and)h(con)o (v)o(en)o(tions)g(used)g(throughout)f(the)g(MPI)h(do)q(cumen)o(t,)75 843 y(some)k(of)g(the)g(c)o(hoices)h(that)e(ha)o(v)o(e)h(b)q(een)i (made,)d(and)i(the)f(rationale)h(b)q(ehind)h(those)e(c)o(hoices.)75 984 y Fq(2.1)59 b(Pro)r(cedure)19 b(Sp)r(eci\014cation)75 1086 y FD(MPI)g(pro)q(cedures)h(are)f(sp)q(eci\014ed)j(using)e(a)f (language)g(indep)q(enden)o(t)j(notation.)31 b(The)20 b(argumen)o(ts)e(of)75 1142 y(pro)q(cedure)e(calls)g(are)f(mark)o(ed)g (as)g Ft(IN)p FD(,)g Ft(OUT)g FD(or)g Ft(INOUT)p FD(.)g(The)h(meanings) f(of)g(these)h(are:)143 1226 y FA(\017)23 b FD(the)15 b(call)h(uses)g(but)f(do)q(es)h(not)e(up)q(date)i(an)f(argumen)o(t)g (mark)o(ed)f Ft(IN)p FD(,)143 1315 y FA(\017)23 b FD(the)15 b(call)h(ma)o(y)f(up)q(date)h(an)f(argumen)o(t)f(mark)o(ed)h Ft(OUT)p FD(,)143 1405 y FA(\017)23 b FD(the)15 b(call)h(b)q(oth)g (uses)f(and)g(up)q(dates)h(an)f(argumen)o(t)g(mark)o(ed)f Ft(INOUT)p FD(.)166 1489 y(There)j(is)h(one)f(sp)q(ecial)h(case)f(|)h (if)f(an)g(argumen)o(t)f(is)h(a)g(handle)h(to)e(an)h(opaque)g(ob)s (ject,)g(and)g(the)75 1545 y(ob)s(ject)g(is)h(up)q(dated)g(b)o(y)f(the) h(pro)q(cedure)g(call,)h(then)e(the)h(argumen)o(t)f(is)g(mark)o(ed)g Ft(OUT)p FD(.)h(This)g(is)g(done)75 1602 y(ev)o(en)f(though)g(the)g (handle)h(itself)f(is)h(not)e(mo)q(di\014ed)i(|)f(w)o(e)g(use)g(the)g Ft(OUT)g FD(attribute)g(to)f(denote)h(that)75 1658 y(what)e(the)g (handle)h Fo(r)n(efer)n(enc)n(es)e FD(is)h(up)q(dated.)166 1715 y(The)h(de\014nition)j(of)c(MPI)i(tries)f(to)g(a)o(v)o(oid,)g(to)f (the)i(largest)f(p)q(ossible)i(exten)o(t,)e(the)g(use)h(of)e Ft(INOUT)75 1771 y FD(argumen)o(ts,)f(b)q(ecause)i(suc)o(h)g(use)f(is)h (error-prone,)e(esp)q(ecially)k(for)d(scalar)g(argumen)o(ts.)166 1828 y(A)d(common)f(o)q(ccurrence)i(for)f(MPI)f(functions)i(is)f(an)g (argumen)o(t)f(that)g(is)i(used)f(as)g Ft(IN)g FD(b)o(y)g(some)f(pro-) 75 1884 y(cesses)h(and)h Ft(OUT)f FD(b)o(y)g(other)g(pro)q(cesses.)19 b(Suc)o(h)13 b(argumen)o(ts)e(are,)h(syn)o(tactically)l(,)i(an)e Ft(INOUT)g FD(argumen)o(t,)75 1940 y(although,)j(seman)o(tically)l(,)h (they)f(are)g(not)g(used)h(in)g(one)f(call)i(b)q(oth)e(for)g(input)h (and)f(for)g(output.)166 1997 y(Another)h(frequen)o(t)g(situation)h (arises)g(when)f(an)g(argumen)o(t)g(v)m(alue)h(is)g(needed)h(only)f(b)o (y)f(a)g(subset)75 2053 y(of)f(the)g(pro)q(cesses.)20 b(When)c(an)f(argumen)o(t)f(is)i(not)f(signi\014can)o(t)h(at)e(a)h(pro) q(cess)h(then)f(an)g(arbitrary)g(v)m(alue)75 2110 y(can)g(b)q(e)h (passed)g(as)e(argumen)o(t.)75 2251 y Fq(2.2)59 b(Semantic)18 b(T)-5 b(erms)75 2353 y FD(When)12 b(discussing)i(p)q(oin)o(t-to-p)q (oin)o(t)f(and)f(collectiv)o(e)h(comm)o(unications)g(the)f(follo)o (wing)h(terms)e(are)h(used:)75 2445 y Fx(non)o(blo)q(c)o(king)24 b FD(if)e(the)h(pro)q(cedure)g(ma)o(y)f(return)g(b)q(efore)h(the)f (related)h(comm)o(unication)g(op)q(eration)189 2501 y(completes,)13 b(and)g(b)q(efore)f(the)h(user)f(is)h(allo)o(w)o(ed)g(to)f(re-use)h (resources)f(\(suc)o(h)h(as)f(bu\013ers\))g(sp)q(eci\014ed)189 2557 y(in)k(the)f(call;)75 2647 y Fx(lo)q(cally-blo)r(c)o(king)26 b FD(if)18 b(return)g(from)f(the)h(pro)q(cedure)h(indicates)g(the)f (user)g(is)h(allo)o(w)o(ed)f(to)f(re-use)i(re-)189 2704 y(sources)c(sp)q(eci\014ed)i(in)f(the)f(call;)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 14 7 13 bop 75 -100 a Fp(2.3.)34 b(D)o(A)l(T)l(A)15 b(TYPES)1375 b FD(7)75 45 y Fx(globally-blo)q(cking)25 b FD(if)c(return)f(from)g (the)g(pro)q(cedure)i(indicates)f(the)g(comm)o(unication)g(op)q (eration)189 102 y(has)15 b(completed.)166 201 y(When)i(discussing)i (the)e(b)q(eha)o(vior)h(of)e(an)o(y)h(MPI)g(function,)h(not)e (necessarily)j(a)d(comm)o(unication)75 257 y(function,)g(the)f(follo)o (wing)h(terms)f(are)f(used.)75 356 y Fx(lo)q(cal)25 b FD(An)16 b(op)q(eration)g(is)g Fx(lo)q(cal)i FD(if)e(its)g(completion)h (dep)q(ends)h(only)e(on)g(the)f(lo)q(cal)i(executing)g(pro)q(cess.)189 413 y(Suc)o(h)e(an)h(op)q(eration)f(do)q(es)h(not)e(require)i(comm)o (unication)g(with)g(another)f(user)g(pro)q(cess.)75 504 y Fx(non-lo)q(cal)26 b FD(An)c(op)q(eration)g(is)g Fx(non-lo)q(cal)j FD(if)d(its)g(completion)h(ma)o(y)e(dep)q(end)i(on)f(comm)o(unication) 189 561 y(o)q(ccurring)16 b(with)f(another)g(user)g(pro)q(cess.)75 653 y Fx(collectiv)o(e)24 b FD(An)17 b(op)q(eration)g(is)g Fx(collectiv)o(e)i FD(if)e(all)g(pro)q(cesses)g(in)h(a)e(pro)q(cess)h (group)g(need)g(to)f(p)q(erform)189 709 y(the)f(op)q(eration.)75 851 y Fq(2.3)59 b(Data)19 b(T)-5 b(yp)r(es)75 954 y Fn(2.3.1)49 b(Opaque)16 b(objects)75 1040 y FD(MPI)23 b(pro)q(cedures)g(use)g Fo(op)n(aque)h(obje)n(cts)p FD(.)42 b(These)23 b(hide)h(the)f(in)o (ternal)h(represen)o(tation)e(used,)j(th)o(us)75 1097 y(allo)o(wing)17 b(similar)h(calls)f(in)h(C)e(and)g(F)l(ortran;)g(the)g (a)o(v)o(oidance)h(of)f(con\015icts)h(with)g(the)f(t)o(yping)h(rules)g (in)75 1153 y(these)f(languages;)h(and)f(future)h(extension)g(of)f (functionalit)o(y)l(.)24 b(The)17 b(mec)o(hanism)g(for)e(opaque)i(ob)s (jects)75 1210 y(used)f(here)f(lo)q(osely)i(follo)o(ws)e(the)g(POSIX)h (F)l(ortran)e(binding)k(standard.)166 1266 y(An)g(opaque)g(ob)s(ject)f (can)h(b)q(e)h(allo)q(cated)g(in)g(user)f(storage)e(or)i(in)g(system)g (\(i.e.)28 b(MPI)18 b(managed\))75 1322 y(storage.)23 b(W)l(e)17 b(call)h(the)f(former)f(a)g Fo(user)i(obje)n(ct)e FD(and)h(the)g(latter)g(a)f Fo(system)h(obje)n(ct)p FD(.)24 b(User)17 b(ob)s(jects)f(are)75 1379 y(allo)q(cated)h(and)f(deallo)q (cated)h(explicitly)h(b)o(y)e(the)g(user)g(as)f(a)h(regular)f(v)m (ariable.)23 b(The)16 b(t)o(yp)q(e)g(de\014nitions)75 1435 y(for)g(suc)o(h)h(ob)s(jects)f(are)h(pro)o(vided)g(b)o(y)g(MPI.)f (The)h(ob)s(jects)f(can)h(b)q(e)g(accessed)g(and)g(mo)q(di\014ed)h (only)g(via)75 1492 y(MPI)d(functions)h(that)f(tak)o(e)f(suc)o(h)i(ob)s (jects)e(as)h(argumen)o(ts.)166 1548 y(System)d(ob)s(jects)g(are)g (accessed)i(only)f(via)f Fo(hand)r(les)p FD(.)19 b(In)13 b(C,)f(a)g(di\013eren)o(t)h(t)o(yp)q(e)g(of)f(handle)i(is)f(de\014ned) 75 1605 y(b)o(y)i(MPI)g(for)f(eac)o(h)h(t)o(yp)q(e)g(of)g(opaque)g (system)f(ob)s(ject.)20 b(In)15 b(F)l(ortran,)f(all)i(handles)g(ha)o(v) o(e)e(t)o(yp)q(e)h FB(INTEGER)p FD(.)75 1661 y(The)g(allo)q(cation)g (and)g(deallo)q(cation)h(of)e(suc)o(h)h(an)f(ob)s(ject,)g(and)g(an)o(y) h(access)f(or)g(up)q(date)h(to)f(it,)g(are)h(done)75 1718 y(only)h(via)f(MPI)g(calls)i(that)d(accept)i(as)e(argumen)o(ts)h (handles)h(to)f(suc)o(h)g(ob)s(jects.)166 1774 y(An)f(opaque)f(ob)s (ject,)g(and)h(the)g(handle)g(to)f(suc)o(h)h(an)g(ob)s(ject,)e(are)i (signi\014can)o(t)g(only)g(at)f(the)h(pro)q(cess)75 1831 y(where)h(the)h(ob)s(ject)e(w)o(as)h(created,)g(and)g(cannot)g(b)q(e)h (transferred)f(to)f(another)h(pro)q(cess.)166 1887 y(Opaque)h(system)e (ob)s(jects)h(are)g(allo)q(cated)h(and)f(deallo)q(cated)i(b)o(y)e (calls)h(that)f(are)f(sp)q(eci\014c)j(to)e(eac)o(h)75 1943 y(ob)s(ject)k(t)o(yp)q(e.)32 b(These)20 b(are)f(listed)i(in)f(the) f(sections)h(where)g(the)f(ob)s(jects)g(are)g(describ)q(ed.)35 b(A)19 b(call)i(to)75 2000 y(deallo)q(cate)15 b(an)f(opaque)h(system)e (ob)s(ject)h(do)q(es)g(not)g(necessarily)h(deallo)q(cate)h(the)e(ob)s (ject)f(immediately)l(,)75 2056 y(but)i(ma)o(y)e(only)i(mark)f(it)h(as) f(candidate)h(for)f(deallo)q(cation.)21 b(The)14 b(ob)s(ject)g(will)i (b)q(e)f(deallo)q(cated)h(as)e(so)q(on)75 2113 y(as)i(there)g(is)h(no)f (v)m(alid)i(handle)g(to)d(access)i(that)e(ob)s(ject)h(and)g(there)g(is) h(no)f(p)q(ending)i(op)q(eration)f(that)e(is)96 b Fm(CHANGE)75 2169 y FD(using)15 b(this)g(ob)s(ject;)f(the)g(handle)i(then)e(b)q (ecomes)h(n)o(ull.)21 b(F)l(or)14 b(example,)h(if)g(a)f(comm)o (unication)h(ob)s(ject)e(is)75 2226 y(freed)i(after)f(the)g(comm)o (unication)i(started)d(but)i(b)q(efore)g(it)g(completed,)g(then)g(the)f (p)q(ending)j(op)q(eration)75 2282 y(is)f(not)g(a\013ected;)g(one)g (can)g(use)g(the)g(handle)h(in)g(calls)g(to)e Fs(MPI)p 1155 2282 14 2 v 16 w(W)l(AIT)h FD(or)f Fs(MPI)p 1438 2282 V 16 w(ST)l(A)l(TUS)p FD(.)i(In)g(fact,)e(one)75 2339 y(m)o(ust)d(complete)h(the)f(comm)o(unication)h(with)f(suc)o(h)h (a)f(call.)20 b(Ho)o(w)o(ev)o(er,)11 b(once)i(the)f(op)q(eration)h (completes,)75 2395 y(it)i(is)h(erroneous)f(to)g(re-use)g(the)h(freed)f (handle)i(for)d(a)h(new)h(comm)o(unication.)166 2534 y Fl(Discussion:)31 b Ft(It)12 b(migh)o(t)d(b)q(e)j(cleaner)g(to)f(sp)q (ecify)h(that)f(the)p 1124 2534 13 2 v 27 w Fv(FREE)f Ft(op)q(eration)i(alw)o(a)o(ys)e(n)o(ullify)f(the)j(handle)95 b Fm(CHANGE)75 2591 y Ft(to)16 b(the)g(ob)r(ject)h(freed)g(b)q(efore)f (it)g(returns,)h(ev)o(en)g(though)o(t)e(the)i(ob)r(ject)g(ma)o(y)d(not) h(b)q(e)i(deallo)q(cated)f(immedia)o(tly)l(.)75 2647 y(Otherwise,)f(it)f(is)g(either)h(the)f(case)h(that)f(MPI)h(will)d(nev) o(er)j(n)o(ullify)e(the)h(p)q(oin)o(ter,)g(or)g(that)g(the)h(user)g(m)o (ust)e(nev)o(er)75 2704 y(deallo)q(cate)19 b(this)g(p)q(oin)o(ter)g(b)q (efore)g(it)g(is)f(n)o(ulli\014ed.)32 b(E.g.,)19 b(if)f(the)h(p)q(oin)o (ter)g(w)o(as)g(an)f(automatic)f(v)n(arialb)q(e)h(in)g(a)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 8 15 8 14 bop 75 -100 a FD(8)715 b Fp(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Ft(subroutine,)e(then)h(the)g(user) g(w)o(aits)f(for)f(MPI)i(to)f(n)o(ullify)e(the)i(p)q(oin)o(ter)h(b)q (efore)f(it)g(returns)i(from)c(the)j(subroutine.)166 318 y Fl(Implemen)o(tati)o(on)h(note:)47 b Ft(Some)15 b(ob)r(jects,)j(suc)o(h)f(as)g(datat)o(yp)q(e)f(ob)r(jects)i(\(see)g (section)f(3.13\),)f(can)g(b)q(e)75 368 y(used)f(b)o(y)f(sev)o(eral,)g (distinct,)g(concurren)o(t)i(comm)o(unicati)o(on)11 b(op)q(erations.)19 b(Th)o(us,)14 b(to)g(\014nd)g(out)g(whether)h(suc)o(h)g(an)75 418 y(ob)r(ject)g(can)f(b)q(e)g(freed,)h(one)f(needs)h(a)e(reference)k (coun)o(t)d(of)f(p)q(ending)h(op)q(erations.)166 475 y(MPI)d(do)q(es)h(not)f(k)o(eep)h(coun)o(t)f(of)f(references)k(to)d(an) g(opaque)g(MPI)g(ob)r(ject)h(that)f(are)h(created)g(b)o(y)f(assigning)f (to)75 532 y(one)j(handle)f(the)h(v)n(alue)e(of)h(another,)h(or)f(are)h (lost)f(when)h(a)f(handle)g(v)n(ariable)f(is)i(deallo)q(cated)f(\()p Fk(e.g.)p Ft(,)g(at)g(pro)q(cedure)75 588 y(exit\).)22 b(In)16 b(the)f(\014rst)h(case,)h(a)d(call)h(to)g(free)h(the)g(ob)r (ject)g(ma)o(y)d(lea)o(v)o(e)i(a)g(dangling)f(reference;)k(in)d(the)h (second)g(case)75 645 y(an)f(opaque)g(MPI)g(ob)r(ject)h(ma)o(y)d(b)q(e) j(left)f(with)f(no)h(handle)g(to)g(it,)g(and)g(no)f(p)q(ossibilit)o(y)g (of)h(deallo)q(cating)f(it.)21 b(It)15 b(is)75 701 y(the)h(user's)g (resp)q(onsibilit)o(y)f(to)g(a)o(v)o(oid)f(these)j(situations.)k(MPI)16 b(k)o(eeps)g(trac)o(k)g(only)e(of)h(c)o(hanges)h(in)f(the)h(n)o(um)o(b) q(er)75 757 y(of)d(activ)o(e)h(references)j(to)d(the)g(MPI)g(ob)r(ject) h(that)f(are)g(a)g(result)g(of)g(explicit)f(MPI)h(calls.)166 898 y FD(MPI)19 b(ma)o(y)g(pro)o(vide)h(prede\014ned)h(opaque)f(ob)s (jects)f(and)g(prede\014ned,)j(static)d(handles)i(to)e(these)75 954 y(ob)s(jects.)g(Suc)o(h)d(ob)s(jects)f(ma)o(y)f(not)h(b)q(e)h (destro)o(y)o(ed.)75 1083 y Fn(2.3.2)49 b(Arra)o(y)16 b(a)o(rguments)75 1171 y FD(An)21 b(MPI)f(call)i(ma)o(y)e(need)h(an)f (argumen)o(t)g(that)g(is)h(an)f(arra)o(y)f(of)h(opaque)h(ob)s(jects,)g (or)f(an)g(arra)o(y)f(of)75 1227 y(handles.)31 b(The)19 b(arra)o(y-of-handles)f(is)h(a)f(regular)g(arra)o(y)g(with)g(en)o (tries)h(that)f(are)g(handles)i(to)e(ob)s(jects)75 1284 y(of)g(the)h(same)g(t)o(yp)q(e)f(in)i(consecutiv)o(e)g(lo)q(cations)f (in)h(the)f(arra)o(y)l(.)29 b(Whenev)o(er)19 b(suc)o(h)g(an)g(arra)o(y) e(is)j(used,)75 1340 y(an)d(additional)h Fs(len)f FD(argumen)o(t)f(is)h (required)h(to)e(indicate)i(the)f(n)o(um)o(b)q(er)g(of)f(v)m(alid)j(en) o(tries)e(\(unless)h(this)75 1396 y(n)o(um)o(b)q(er)f(can)h(b)q(e)g (deriv)o(ed)g(otherwise\).)25 b(The)18 b(v)m(alid)h(en)o(tries)e(are)g (at)g(the)g(b)q(egining)i(of)e(the)g(arra)o(y;)f Fs(len)75 1453 y FD(indicates)j(ho)o(w)e(man)o(y)h(of)f(them)h(there)g(are,)f (and)h(need)h(not)e(b)q(e)h(the)g(en)o(tire)g(size)h(of)e(the)h(arra)o (y)l(.)27 b(The)75 1509 y(same)15 b(approac)o(h)g(is)g(follo)o(w)o(ed)h (for)f(other)f(arra)o(y)g(argumen)o(ts.)75 1638 y Fn(2.3.3)49 b(State)75 1726 y FD(MPI)15 b(pro)q(cedures)h(use)g(at)e(v)m(arious)i (places)g(argumen)o(ts)f(with)g Fo(state)g FD(t)o(yp)q(es.)20 b(The)c(v)m(alues)g(of)f(suc)o(h)h(data)75 1782 y(t)o(yp)q(e)i(are)h (all)g(iden)o(ti\014ed)h(b)o(y)f(names,)f(and)h(no)f(op)q(eration)h(is) g(de\014ned)h(on)e(them.)29 b(F)l(or)18 b(example,)i(the)75 1839 y Fs(MPI)p 160 1839 14 2 v 16 w(SET)p 259 1839 V 17 w(ERRORMODE)c FD(routine)f(has)f(a)g(state)g(t)o(yp)q(e)h(argumen)o (t)e(with)i(v)m(alues)h Fv(MPI)p 1572 1839 13 2 v 14 w(ERRORSF)m(A)m(T)m(AL)p FD(,)75 1895 y Fv(MPI)p 152 1895 V 14 w(ERRORSRETURN)p FD(,)e(etc.)75 2024 y Fn(2.3.4)49 b(Named)14 b(constants)75 2112 y FD(MPI)g(pro)q(cedures)g(sometimes)g (assign)f(a)h(sp)q(ecial)h(meaning)f(to)f(a)h(sp)q(ecial)h(v)m(alue)g (of)e(a)g(basic)i(t)o(yp)q(e)e(argu-)75 2168 y(men)o(t;)k(e.g.)24 b Fs(tag)17 b FD(is)g(an)g(in)o(teger)g(v)m(alued)i(argumen)o(t)d(of)g (p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)f(op)q(erations,)75 2225 y(with)f(a)g(sp)q(ecial)i(wild-card)f(v)m(alue,)g Fv(MPI)p 775 2225 V 15 w(ANY)p 875 2225 V 14 w(T)m(A)o(G)p FD(.)e(Suc)o(h)i(argumen)o(ts)e(will)i(ha)o(v)o(e)f(a)g(range)f(of)h (regular)75 2281 y(v)m(alues,)i(whic)o(h)g(is)f(a)f(prop)q(er)h (subrange)g(of)f(the)h(range)g(of)f(v)m(alues)i(of)e(the)h(corresp)q (onding)h(basic)f(t)o(yp)q(e;)75 2337 y(sp)q(ecial)f(v)m(alues)g(\(suc) o(h)f(as)f Fv(MPI)p 609 2337 V 14 w(ANY)p 708 2337 V 15 w(T)m(A)o(G)p FD(\))f(will)j(b)q(e)f(outside)g(the)g(regular)g (range.)k(The)c(range)f(of)g(regu-)75 2394 y(lar)f(v)m(alues)i(can)e(b) q(e)h(queried,)g(and)f(sometimes)g(set,)g(using)h(en)o(vironmen)o(tal)g (inquiry)h(or)d(en)o(vironmen)o(tal)75 2450 y(managemen)o(t)i (functions)i(\(Chapter)f(7\).)166 2591 y Fl(Implemen)o(tati)o(on)d (note:)35 b Ft(Implemen)o(tors)11 b(should)i(detect,)i(whenev)o(er)g(p) q(ossible,)e(illegal)e(uses)k(of)d(\\sp)q(e-)75 2647 y(cial)i(v)n(alues".)22 b(Th)o(us,)16 b(the)f(use)i(of)d(the)i Fv(MPI)p 780 2647 V 14 w(ANY)p 879 2647 V 15 w(T)m(A)o(G)e Ft(v)n(alue)h(to)g(tag)g(a)g(message)g(sen)o(t)h(should)f(b)q(e)h (\015agged)f(as)75 2704 y(an)f(error.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 9 16 9 15 bop 75 -100 a Fp(2.4.)34 b(LANGUA)o(GE)15 b(BINDING)1187 b FD(9)75 45 y Fn(2.3.5)49 b(Choice)75 149 y FD(MPI)11 b(functions)h(sometimes)f(use)h(argumen)o(ts)e(with)h(a)g Fo(choic)n(e)g FD(\(or)g(union\))g(data)g(t)o(yp)q(e.)19 b(Distinct)11 b(calls)i(to)75 206 y(the)f(same)g(routine)g(ma)o(y)g (pass)f(b)o(y)h(reference)h(actual)f(argumen)o(ts)f(of)h(di\013eren)o (t)g(t)o(yp)q(es.)19 b(The)12 b(mec)o(hanism)75 262 y(for)j(pro)o (viding)h(suc)o(h)f(argumen)o(ts)g(will)h(di\013er)g(from)e(language)i (to)e(language.)166 404 y Fl(Discussion:)47 b Ft(The)18 b(F)m(ortran)f(77)f(standard)i(sp)q(eci\014es)h(that)e(the)h(t)o(yp)q (e)g(of)f(actual)f(argumen)o(ts)h(need)h(to)75 454 y(agree)e(with)g (the)g(t)o(yp)q(e)h(of)e(dumm)o(y)e(argumen)o(ts;)i(no)h(construct)h (equiv)n(alen)o(t)e(to)h(C)g(v)o(oid)f(p)q(oin)o(ters)h(is)g(a)o(v)n (ailable.)75 503 y(Th)o(us,)d(it)g(w)o(ould)g(seem)g(that)h(there)g(is) g(no)f(standard)h(conforming)d(mec)o(hanism)g(to)i(supp)q(ort)h(c)o (hoice)g(argumen)o(ts.)75 553 y(Ho)o(w)o(ev)o(er,)e(most)f(F)m(ortran)h (compilers)f(either)i(don't)f(c)o(hec)o(k)h(t)o(yp)q(e)f(consistency)i (of)d(calls)h(to)g(external)g(routines,)h(or)75 603 y(supp)q(ort)i(a)f (sp)q(ecial)g(mec)o(hanism)d(to)j(link)f(foreign)h(\(e.g.,)f(C\))h (routines.)19 b(W)m(e)13 b(accept)j(this)e(non-conformit)o(y)d(with)75 653 y(the)16 b(F)m(ortran)e(77)h(standard.)22 b(I.e.,)14 b(w)o(e)h(accept)h(that)f(the)h(same)e(routine)h(ma)o(y)e(b)q(e)j (passed)g(an)e(actual)h(argumen)o(t)75 703 y(of)e(a)h(di\013eren)o(t)h (t)o(yp)q(e)f(at)g(distinct)g(calls.)166 762 y(Generic)j(routines)g (can)f(b)q(e)h(used)h(in)d(F)m(ortran)i(90)e(to)h(pro)o(vide)h(a)f (standard)g(conforming)f(solution.)24 b(This)75 812 y(solution)13 b(will)f(b)q(e)j(consisten)o(t)g(with)f(our)f(nonstandard)i(conforming) c(F)m(ortran)j(77)f(solution.)75 1070 y Fn(2.3.6)49 b(Addresses)75 1174 y FD(Some)21 b(MPI)g(pro)q(cedures)h(use)f Fo(addr)n(ess)g FD(argumen)o(ts)f(that)g(represen)o(t)i(an)f(absolute)g(address)g(in)h (the)75 1230 y(calling)16 b(program.)i(The)d(datat)o(yp)q(e)e(of)h(suc) o(h)g(an)g(argumen)o(t)g(is)g(an)g(in)o(teger)g(of)g(the)g(size)h (needed)h(to)d(hold)75 1287 y(an)i(address)g(in)h(the)g(execution)g(en) o(vironmen)o(t.)75 1483 y Fq(2.4)59 b(Language)19 b(Binding)75 1603 y FD(This)k(section)g(de\014nes)h(the)e(rules)i(for)e(MPI)g (language)h(binding)h(in)g(general)f(and)f(for)g(F)l(ortran)g(77)75 1659 y(and)17 b(ANSI)h(C)f(in)h(particular.)26 b(De\014ned)17 b(here)h(are)f(v)m(arious)g(ob)s(ject)g(represen)o(tations,)g(as)f(w)o (ell)i(as)f(the)75 1716 y(naming)k(con)o(v)o(en)o(tions)f(used)g(for)g (expressing)h(this)f(standard.)35 b(The)20 b(actual)g(calling)i (sequences)f(are)75 1772 y(de\014ned)c(elsewhere.)166 1838 y(It)23 b(is)g(exp)q(ected)h(that)e(an)o(y)g(F)l(ortran)g(90)g (and)h(C++)g(implemen)o(tations)h(use)f(the)g(F)l(ortran)f(77)75 1895 y(and)f(ANSI)h(C)e(bindings,)k(resp)q(ectiv)o(ely)l(.)39 b(Although)22 b(w)o(e)e(consider)i(it)f(premature)g(to)f(de\014ne)i (other)75 1951 y(bindings)e(to)d(F)l(ortran)g(90)g(and)h(C++,)h(the)f (curren)o(t)f(bindings)j(are)e(designed)h(to)e(encourage,)i(rather)75 2007 y(than)c(discourage,)g(exp)q(erimen)o(tation)i(with)e(b)q(etter)g (bindings)i(whic)o(h)f(migh)o(t)f(b)q(e)h(adopted)g(later.)166 2073 y(Since)h(the)e(w)o(ord)g(P)l(ARAMETER)h(is)g(a)f(k)o(eyw)o(ord)g (in)h(the)g(F)l(ortran)e(language,)h(w)o(e)g(use)h(the)g(w)o(ord)75 2130 y(\\argumen)o(t")21 b(to)i(denote)g(the)g(argumen)o(ts)e(to)i(a)f (subroutine.)44 b(These)23 b(are)f(normally)i(referred)f(to)75 2186 y(as)f(parameters)f(in)i(C,)e(ho)o(w)o(ev)o(er)g(w)o(e)h(exp)q (ect)h(that)e(C)h(programmers)e(will)k(understand)f(the)f(w)o(ord)75 2243 y(argumen)o(ts)d(\(whic)o(h)i(has)g(no)f(sp)q(eci\014c)i(meaning)f (in)g(C\),)f(th)o(us)g(allo)o(wing)h(us)f(to)g(a)o(v)o(oid)g (unnecessary)75 2299 y(confusion)c(for)f(F)l(ortran)f(programmers.)166 2365 y(There)k(are)g(sev)o(eral)g(imp)q(ortan)o(t)f(language)h(binding) i(issues)f(not)e(addressed)i(b)o(y)e(this)i(standard.)75 2421 y(It)f(is)g(b)q(ey)o(ond)h(the)f(scop)q(e)g(of)g(this)g(standard)g (to)f(discuss)i(the)f(in)o(terop)q(erabilit)o(y)i(of)d(message)h (passing)75 2478 y(b)q(et)o(w)o(een)c(languages.)19 b(Although)c(it)e (w)o(ould)i(b)q(e)f(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)h(to)e (guaran)o(tee)g(that)f(messages)75 2534 y(sen)o(t)20 b(b)q(et)o(w)o(een)h(languages)f(b)q(e)h(prop)q(erly)g(in)o(terpreted,) h(the)e(matc)o(hing)g(of)g(t)o(yp)q(es)g(across)g(languages)75 2591 y(prev)o(en)o(ts)15 b(the)h(inclusion)i(of)d(suc)o(h)h(a)f(guaran) o(tee)g(in)i(this)f(standard.)k(It)c(is)g(fully)h(exp)q(ected)g(that)d (man)o(y)75 2647 y(implemen)o(tations)i(will)g(ha)o(v)o(e)f(suc)o(h)g (features,)f(and)h(that)f(suc)o(h)h(features)g(are)f(a)h(sign)g(of)f (the)h(qualit)o(y)g(of)75 2704 y(the)g(implemen)o(tation.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 10 17 10 16 bop 75 -100 a FD(10)692 b Fp(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Fn(2.4.1)49 b(Data)17 b(T)l(yp)q(es)f(and)h(Naming)e(Conventions)75 134 y FD(Handles)g(are)f(represen)o(ted)h(as)e(in)o(tegers)i(in)g(F)l (ortran)e(and)h(as)g(p)q(oin)o(ters)g(to)g(structures)f(in)i(C.)f(A)g (sp)q(ecial)75 191 y(v)m(alue)g Fv(MPI)p 267 191 13 2 v 14 w(NULL)g FD(is)f(used)g(to)g(represen)o(t)f(n)o(ull)j(handles)e (in)h(F)l(ortran;)e(in)i(C,)e(the)h(constan)o(t)f Fv(NULL)h FD(is)g(used)75 247 y(as)i(usual.)166 305 y(T)o(yp)q(ed)h(comm)o (unication)g(routines)g(supp)q(ort)g Fx(c)o(haracter)p FD(,)g Fx(double)i(precision)p FD(,)e Fx(real)p FD(,)g(and)g Fx(in-)75 362 y(teger)d FD(data)f(in)h(F)l(ortran.)18 b Fx(in)o(t)p FD(,)13 b Fx(double)p FD(,)h Fx(real)p FD(,)f Fx(short)p FD(,)g Fx(c)o(har)p FD(,)f(and)h Fx(long)h FD(are)e(supp)q(orted)h(in)h(C,)e(along)75 418 y(with)k Fx(unsigned)j(in)o(t)p FD(,)d Fx(unsigned)i(long)p FD(,)f Fx(unsigned)h(short)p FD(,)e(and)g Fx(unsigned)i(c)o(har)p FD(.)k(In)16 b(addition,)75 475 y(a)e(sp)q(ecial)j Fx(b)o(yte)d FD(t)o(yp)q(e)g(is)h(supp)q(orted)h(in)f(b)q(oth)g(languages.)k(This)d (t)o(yp)q(e)e(allo)o(ws)h(comm)o(unication)g(to)f(b)q(e)75 531 y(p)q(erformed)f(with)g(no)f(t)o(yp)q(e)h(con)o(v)o(ersion)f(at)g (all.)20 b(In)13 b(C,)f(the)h(de\014nitions)h(of)f(named)f(constan)o (ts,)g(function)75 588 y(protot)o(yp)q(es,)k(and)h(aggregate)f(t)o(yp)q (e)g(information)h(m)o(ust)g(b)q(e)g(supplied)j(in)d(an)g(include)i (\014le)f FB(mpi.h)p FD(.)24 b(In)75 644 y(F)l(ortran,)c(the)h(corresp) q(onding)h(include)g(\014le)g(for)e(named)h(constan)o(ts)e(\()p FB(PARAMETER)p FD(s\))g(will)j(b)q(e)f(called)75 700 y FB(mpif.h)p FD(.)75 832 y Fn(2.4.2)49 b(F)o(o)o(rtran)15 b(77)i(Binding)h(Issues)75 921 y FD(All)13 b(MPI)e(names)h(ha)o(v)o(e)f (an)h FB(MPI)p 618 921 15 2 v 28 w FD(pre\014x,)g(and)g(all)g(c)o (haracters)f(are)g(capitals.)20 b(All)13 b(MPI)e(F)l(ortran)g(subrou-) 75 978 y(tines)16 b(ha)o(v)o(e)g(a)f(return)h(co)q(de)g(in)h(the)f (last)f(argumen)o(t.)21 b(A)16 b(few)g(MPI)f(op)q(erations)h(are)g (functions,)g(whic)o(h)75 1034 y(do)g(not)g(ha)o(v)o(e)f(the)i(return)f (co)q(de)g(argumen)o(t.)22 b(The)16 b(return)g(co)q(de)h(v)m(alue)g (for)f(successful)h(completion)g(is)75 1090 y Fv(MPI)p 152 1090 13 2 v 14 w(SUCCESS)p FD(;)11 b(error)h(co)q(des)i(are)e (implemen)o(tation)j(dep)q(enden)o(t.)20 b(Unless)14 b(explicitly)i(stated,)c(the)h(MPI)75 1147 y(F77)i(binding)k(is)e (consisten)o(t)f(with)h(ANSI)g(standard)f(F)l(ortran)f(77.)23 b(There)16 b(are)g(sev)o(eral)h(p)q(oin)o(ts)g(where)75 1203 y(this)d(standard)f(div)o(erges)g(from)g(the)g(ANSI)h(F)l(ortran)e (77)h(standard.)19 b(These)14 b(exceptions)g(are)f(consisten)o(t)75 1260 y(with)j(common)e(practice)i(in)g(the)g(F)l(ortran)e(comm)o(unit)o (y)l(.)20 b(In)15 b(particular:)143 1360 y FA(\017)23 b FD(MPI)15 b(Iden)o(ti\014ers)h(are)f(limited)i(to)e(thirt)o(y)l(,)g (not)f(six,)i(signi\014can)o(t)g(c)o(haracters.)143 1461 y FA(\017)23 b FD(MPI)15 b(Iden)o(ti\014ers)h(ma)o(y)f(con)o(tain)g (underscores)h(after)e(the)i(\014rst)e(c)o(haracter.)143 1562 y FA(\017)23 b FD(An)c(MPI)f(subroutine)i(with)f(a)f(c)o(hoice)i (argumen)o(t)e(ma)o(y)g(b)q(e)h(called)i(with)e(di\013eren)o(t)g (argumen)o(t)189 1618 y(t)o(yp)q(es.)32 b(An)20 b(example)g(is)g(sho)o (wn)f(in)h(Figure)g(2.1.)32 b(This)20 b(violates)g(the)f(letter)h(of)f (the)g(F)l(ortran)189 1675 y(standard,)14 b(but)h(suc)o(h)g(a)f (violation)i(is)f(common)f(practice.)21 b(The)15 b(alternativ)o(e)g(w)o (ould)g(b)q(e)g(to)f(ha)o(v)o(e)189 1731 y(a)h(separate)f(v)o(ersion)i (of)e Fs(MPI)p 697 1731 14 2 v 16 w(SEND)p FD(,)h(for)g(example,)h(for) e(eac)o(h)h(data)g(t)o(yp)q(e.)143 1832 y FA(\017)23 b FD(Although)12 b(not)f(required,)i(it)f(is)g(strongly)f(suggested)h (that)f(named)h(MPI)f(constan)o(ts)g(b)q(e)h(pro)o(vided)189 1888 y(in)18 b(an)g(include)i(\014le,)g(called)f FB(mpif.h)p FD(.)27 b(On)19 b(systems)e(that)g(do)h(not)f(supp)q(ort)h(include)j (\014les,)e(the)189 1945 y(implemen)o(tation)d(should)g(sp)q(ecify)h (the)e(v)m(alues)h(of)f(named)h(constan)o(ts.)143 2045 y FA(\017)23 b FD(V)l(endors)e(are)f(encouraged)h(to)f(pro)o(vide)h(t)o (yp)q(e)g(declarations)g(in)h(the)f(mpif.h)g(\014le)h(on)e(F)l(ortran) 189 2102 y(systems)h(that)f(supp)q(ort)i(user-de\014ned)h(t)o(yp)q(es.) 39 b(One)22 b(should)h(de\014ne,)h(if)e(p)q(ossible,)i(the)e(t)o(yp)q (e)189 2158 y Fs(MPI)p 274 2158 V 15 w(ADDR)p FD(,)15 b(whic)o(h)g(is)f(an)h Fs(INTEGER)g FD(of)e(the)i(size)g(needed)g(to)f (hold)h(an)f(address)h(in)g(the)f(execu-)189 2215 y(tion)h(en)o (vironmen)o(t.)22 b(On)16 b(systems)f(where)g(t)o(yp)q(e)h (de\014nition)h(is)f(not)f(supp)q(orted,)h(it)g(is)g(up)g(to)f(the)189 2271 y(user)h(to)f(use)i(an)f Fs(INTEGER)g FD(of)g(the)g(righ)o(t)g (kind)h(to)e(represen)o(t)h(addresses)h(\(i.e.,)e FB(INTEGER*4)g FD(on)189 2328 y(a)g(32)f(bit)i(mac)o(hine,)f FB(INTEGER*8)f FD(on)i(a)e(64)h(bit)h(mac)o(hine,)f(etc.\).)75 2502 y Fn(2.4.3)49 b(C)16 b(Binding)i(Issues)75 2591 y FD(W)l(e)23 b(use)g(the)f(ANSI)i(C)e(declaration)h(format.)41 b(All)24 b(MPI)f(names)f(ha)o(v)o(e)g(an)h FB(MPI)p 1548 2591 15 2 v 39 w FD(pre\014x,)i(de\014ned)75 2647 y(constan)o(ts)15 b(are)h(in)h(all)g(capital)g(letters,)f(and)g(de\014ned)i(t)o(yp)q(es)e (and)g(functions)h(ha)o(v)o(e)f(one)g(capital)h(letter)75 2704 y(after)d(the)i(pre\014x.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 11 18 11 17 bop 75 -100 a Fp(2.5.)34 b(PR)o(OCESSES)1381 b FD(11)290 81 y FB(double)23 b(precision)f(a)290 138 y(integer)h(b)290 194 y(call)g(MPI_send\(a,...\))385 251 y(.)385 307 y(.)290 363 y(call)g(MPI_send\(b,...\))75 511 y FD(Figure)e(2.1:)29 b(This)21 b(is)g(an)f(example)h(of)f(calling)i(a)e(routine)h(with)g (mismatc)o(hed)f(formal)g(and)h(actual)75 568 y(argumen)o(ts.)166 710 y(Almost)j(all)h(C)e(functions)i(return)f(an)g(error)f(co)q(de.)47 b(The)24 b(successful)h(return)f(co)q(de)h(will)g(b)q(e)75 767 y FB(MPI)p 150 767 15 2 v 17 w(SUCCESS)p FD(,)16 b(but)i(failure)h(return)f(co)q(des)g(are)f(implemen)o(tation)i(dep)q (enden)o(t.)29 b(A)18 b(few)g(C)f(functions)75 823 y(will)i(return)f(v) m(alues,)h(so)e(that)g(they)g(can)h(b)q(e)g(implemen)o(ted)i(as)d (macros.)26 b(Logical)18 b(\015ags)g(are)f(in)o(tegers)75 880 y(with)c(v)m(alues)h(1)e(and)h(0)g(meaning)g(\\true")f(and)h (\\false")g(resp)q(ectiv)o(ely)l(.)21 b(Choice)13 b(argumen)o(ts)f(are) h(p)q(oin)o(ters)75 936 y(of)i(t)o(yp)q(e)g FB(void*)p FD(.)1608 b Fm(CHANGE)166 997 y FD(Address)18 b(argumen)o(ts)f(are)g (of)h(MPI)f(de\014ned)i(t)o(yp)q(e)f Fv(MPI)p 1146 997 13 2 v 14 w(Aint)p FD(.)28 b(This)18 b(is)g(de\014ned)h(to)e(b)q(e)i (an)e Fv(int)h FD(of)75 1054 y(the)d(size)h(needed)h(to)d(hold)i(an)g (address)f(on)g(the)g(target)f(arc)o(hitecture.)75 1224 y Fq(2.5)59 b(Pro)r(cesses)75 1335 y FD(An)14 b(MPI)g(program)e (consists)i(of)f(autonomous)g(pro)q(cesses,)h(executing)h(their)f(o)o (wn)f(co)q(de,)h(in)h(an)f(MIMD)75 1391 y(st)o(yle.)29 b(The)18 b(co)q(des)h(executed)g(b)o(y)f(eac)o(h)g(pro)q(cess)h(need)g (not)f(b)q(e)h(iden)o(tical.)30 b(The)19 b(pro)q(cesses)f(comm)o(u-)75 1447 y(nicate)g(via)g(calls)g(to)f(MPI)g(comm)o(unication)h(primitiv)o (es.)28 b(T)o(ypically)l(,)19 b(eac)o(h)f(pro)q(cess)g(executes)g(in)g (its)75 1504 y(o)o(wn)h(address)g(space,)h(although)f(shared-memory)g (implemen)o(tations)h(of)f(MPI)g(are)g(p)q(ossible.)33 b(This)75 1560 y(do)q(cumen)o(t)20 b(sp)q(eci\014es)h(the)f(b)q(eha)o (vior)g(of)f(a)g(parallel)i(program)d(assuming)i(that)e(only)i(MPI)g (calls)g(are)75 1617 y(used)c(for)e(comm)o(unication.)21 b(The)15 b(in)o(teraction)h(of)f(an)g(MPI)g(program)f(with)i(other)e(p) q(ossible)j(means)e(of)75 1673 y(comm)o(unication)h(\(e.g.,)e(shared)h (memory\))f(is)i(not)f(sp)q(eci\014ed.)166 1734 y(MPI)c(do)q(es)g(not)g (sp)q(ecify)h(the)f(execution)h(mo)q(del)g(for)e(eac)o(h)h(pro)q(cess.) 19 b(A)11 b(pro)q(cess)g(can)h(b)q(e)f(sequen)o(tial,)75 1791 y(or)f(can)h(b)q(e)h(m)o(ulti-threaded,)g(with)f(threads)g(p)q (ossibly)h(executing)g(concurren)o(tly)l(.)20 b(Care)10 b(has)g(b)q(een)i(tak)o(en)75 1847 y(to)j(mak)o(e)f(MPI)h (\\thread-safe,")f(b)o(y)i(a)o(v)o(oiding)f(the)g(use)h(of)f(implicit)i (state.)166 1909 y(MPI)e(do)q(es)h(not)f(sp)q(ecify)i(either)f(the)f (in)o(teraction)h(of)f(pro)q(cesses)h(with)g(signals,)g(in)g(a)f(UNIX)h (en)o(vi-)75 1965 y(ronmen)o(t,)11 b(or)g(with)h(other)f(ev)o(en)o(ts)g (that)g(do)g(not)h(relate)f(to)g(MPI)g(comm)o(unication.)20 b(That)11 b(is,)h(signals)g(are)75 2021 y(not)h(signi\014can)o(t)h (from)e(the)h(view)h(p)q(oin)o(t)g(of)e(MPI,)h(and)g(implemen)o(tors)h (should)g(attempt)e(to)h(implemen)o(t)75 2078 y(MPI)h(so)g(that)f (signals)i(are)f(transparen)o(t:)19 b(an)14 b(MPI)g(call)h(susp)q (ended)h(b)o(y)e(a)g(signal)h(should)g(resume)g(and)75 2134 y(complete)h(after)e(the)i(signal)g(is)f(handled.)166 2195 y(MPI)j(do)q(es)g(not)f(pro)o(vide)h(mec)o(hanisms)g(to)f(sp)q (ecify)i(the)f(initial)i(allo)q(cation)f(of)e(pro)q(cesses)h(to)f(an)75 2252 y(MPI)j(computation)g(and)h(their)f(binding)j(to)c(ph)o(ysical)j (pro)q(cessors.)34 b(It)21 b(is)f(exp)q(ected)i(that)d(v)o(endors)75 2308 y(will)d(pro)o(vide)f(mec)o(hanisms)h(to)e(do)g(so)h(either)g(at)f (load)h(time)g(or)f(at)g(run)h(time.)20 b(Suc)o(h)15 b(mec)o(hanisms)g(will)75 2365 y(allo)o(w)d(the)f(sp)q(eci\014cation)i (of)e(the)g(initial)j(n)o(um)o(b)q(er)d(of)g(required)i(pro)q(cesses,)f (the)f(co)q(de)h(to)f(b)q(e)h(executed)g(b)o(y)75 2421 y(eac)o(h)g(initial)i(pro)q(cess,)f(and)f(the)g(allo)q(cation)h(of)f (pro)q(cesses)g(to)g(pro)q(cessors.)18 b(Also,)13 b(the)f(curren)o(t)g (prop)q(osal)75 2478 y(do)q(es)19 b(not)f(pro)o(vide)h(for)e(dynamic)j (creation)e(or)g(deletion)i(of)e(pro)q(cesses)g(during)i(program)d (execution)75 2534 y(\(the)f(total)f(n)o(um)o(b)q(er)h(of)f(pro)q (cesses)i(is)f(\014xed\),)g(although)g(it)g(is)g(in)o(tended)i(to)d(b)q (e)h(consisten)o(t)g(with)h(suc)o(h)75 2591 y(extensions.)26 b(Finally)l(,)19 b(the)f(curren)o(t)f(prop)q(osal)g(do)q(es)g(not)g(sp) q(ecify)i(a)d(naming)i(sc)o(heme)f(for)g(pro)q(cesses.)75 2647 y(W)l(e)c(prop)q(ose)h(alw)o(a)o(ys)e(to)h(iden)o(tify)h(pro)q (cesses)g(according)f(to)g(their)h(relativ)o(e)f(rank)g(in)h(a)f (group,)g(so)g(that,)75 2704 y(e\013ectiv)o(ely)l(,)j(pro)q(cesses)g (are)e(iden)o(ti\014ed)k(b)o(y)d(consecutiv)o(e)h(in)o(tegers)f(in)h (the)g(range)e FB(0..groupsize-1)p FD(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 12 19 12 18 bop 75 -100 a FD(12)692 b Fp(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Fq(2.6)59 b(Erro)n(r)21 b(Handling)75 151 y FD(MPI)11 b(pro)o(vides)h(the)g(user) f(with)h(reliable)i(message)d(transmission.)19 b(A)11 b(message)g(sen)o(t)g(is)h(alw)o(a)o(ys)f(receiv)o(ed)75 208 y(correctly)l(,)17 b(and)g(the)g(user)g(do)q(es)g(not)f(need)i(to)e (c)o(hec)o(k)h(for)f(transmission)h(errors,)f(time-outs,)h(or)f(other) 75 264 y(error)11 b(conditions.)20 b(In)13 b(other)f(w)o(ords,)f(MPI)h (do)q(es)g(not)g(pro)o(vide)g(mec)o(hanisms)h(for)e(dealing)j(with)e (failures)75 321 y(in)17 b(the)f(comm)o(unication)h(system.)k(If)c(the) f(MPI)g(implemen)o(tation)h(is)f(built)i(on)e(an)g(unreliable)i(under-) 75 377 y(lying)h(mec)o(hanism,)f(then)g(it)f(is)h(the)g(job)f(of)g(the) g(implemen)o(tor)i(of)e(the)g(MPI)h(subsystem)f(to)g(insulate)75 433 y(the)c(user)h(from)e(this)i(unreliabilit)o(y)l(,)i(or)d(to)f (re\015ect)i(unreco)o(v)o(erable)f(errors)g(as)f(global)i(program)e (failures.)75 490 y(Similarly)l(,)i(MPI)e(itself)h(pro)o(vides)f(no)g (mec)o(hanisms)h(for)e(handling)i(no)q(de)g(failures.)20 b(\(A)11 b(mec)o(hanism)i(that)75 546 y(allo)o(ws)i(the)h(user)f(to)g (restrict)g(the)g(scop)q(e)h(of)e(an)i(unreco)o(v)o(erable)f(error)g (is)h(presen)o(ted)f(in)h(section)g(7.2.\))166 605 y(Of)g(course,)h (MPI)f(programs)f(ma)o(y)h(still)i(b)q(e)f(erroneous.)23 b(A)16 b Fx(program)i(error)d FD(can)i(o)q(ccur)f(when)75 662 y(an)g(MPI)g(call)g(is)h(called)g(with)f(an)g(incorrect)g(argumen)o (t)f(\(non-existing)i(destination)g(in)g(a)e(send)i(op)q(er-)75 718 y(ation,)g(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i(op)q (eration,)f(etc.\))25 b(This)17 b(t)o(yp)q(e)g(of)f(error)g(w)o(ould)h (o)q(ccur)h(in)f(an)o(y)75 775 y(implemen)o(tation.)31 b(In)19 b(addition,)h(a)f Fx(resource)h(error)d FD(ma)o(y)h(o)q(ccur)h (when)g(a)f(program)f(exceeds)j(the)75 831 y(amoun)o(t)d(of)f(a)o(v)m (ailable)j(system)e(resources)h(\(n)o(um)o(b)q(er)f(of)g(p)q(ending)i (messages,)e(system)g(bu\013ers,)g(etc.\).)75 887 y(The)h(o)q (ccurrence)h(of)f(this)g(t)o(yp)q(e)g(of)g(error)f(dep)q(ends)j(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(in)h(the)75 944 y(system)13 b(and)h(the)g(resource)f(allo)q(cation)i(mec)o(hanism)f (used;)h(this)f(ma)o(y)f(di\013er)h(from)f(system)g(to)g(system.)75 1000 y(A)j(high)h(qualit)o(y)g(implemen)o(tation)g(will)h(pro)o(vide)e (generous)h(limits)g(on)f(some)g(of)f(the)i(resources)f(so)f(as)75 1057 y(to)h(alleviate)i(the)f(p)q(ortabilit)o(y)h(problem)f(this)h (represen)o(ts.)24 b(A)17 b(program)f(is)h Fx(safe)g FD(if)g(it)g(is)h(not)e(sub)s(ject)75 1113 y(to)f(resource)g(errors,)f Fx(unsafe)p FD(,)h(otherwise.)166 1172 y(All)k(MPI)f(calls)g(return)g (a)g(co)q(de)g(that)f(indicates)i(successful)g(completion)g(of)e(the)h (op)q(eration,)g(or)75 1229 y(the)g(error)f(condition)i(that)f(o)q (ccurred,)h(otherwise.)28 b(Ho)o(w)o(ev)o(er,)17 b(MPI)h(do)q(es)g(not) g(require)h(the)f(user)g(to)75 1285 y(c)o(hec)o(k)e(for)e(error)h(v)m (alues.)22 b(It)15 b(is)h(recommended)h(that)d(an)i(implemen)o(tation)g (pro)o(vide)g(an)g(en)o(vironmen)o(t)75 1342 y(for)f(MPI)h(execution)g (that)f(can)h(b)q(e)g(set)g(so)f(that)g(an)o(y)g(error)g(encoun)o (tered)h(in)h(an)e(MPI)h(call)g(will)i(cause)75 1398 y(a)d(program)g(to)g(fail)i(with)f(a)f(suitable)i(error)e(message)g (returned)h(to)f(the)h(user.)22 b(The)16 b(en)o(vironmen)o(t)g(for)75 1454 y(an)f(MPI)g(execution)h(can)f(also)f(b)q(e)i(set)f(so)f(that)g (\(non-fatal\))g(errors)h(do)f(not)h(cause)g(a)g(program)e(to)i(fail,) 75 1511 y(and)j(the)f(user)g(can)h(pro)o(vide)g(his)g(or)f(her)g(o)o (wn)g(error)g(reco)o(v)o(ery)f(co)q(de.)27 b(A)18 b(mec)o(hanism)g (that)e(ac)o(hiev)o(es)75 1567 y(this)g(in)g(a)f(POSIX)h(en)o(vironmen) o(t)f(is)h(describ)q(ed)h(in)f(section)g(7.2.)166 1626 y(MPI)d(calls)h(ma)o(y)f(initiate)h(op)q(erations)g(that)e(con)o(tin)o (ue)i(async)o(hronously)g(after)e(the)i(call)g(returned.)75 1683 y(Th)o(us,)22 b(the)g(op)q(eration)f(ma)o(y)g(return)g(with)h(a)f (co)q(de)h(indicating)h(successful)g(completion,)g(y)o(et)e(later)75 1739 y(cause)15 b(an)g(error)g(exception)h(to)e(b)q(e)i(raised.)k(If)15 b(there)g(is)h(a)f(subsequen)o(t)g(call)h(that)f(relates)g(to)f(the)h (same)75 1796 y(op)q(eration)f(\(e.g.,)f(a)g(call)i(that)e(v)o (eri\014es)i(that)e(an)h(async)o(hronous)f(op)q(eration)h(has)g (completed\))h(then)f(the)75 1852 y(error)g(argumen)o(t)g(asso)q (ciated)h(with)g(this)g(call)g(will)i(b)q(e)e(used)g(to)f(indicate)i (the)f(nature)f(of)h(the)f(error.)19 b(In)75 1908 y(a)14 b(few)h(cases,)f(the)g(error)g(ma)o(y)g(o)q(ccur)h(after)f(all)h(calls) h(that)d(relate)i(to)f(the)h(op)q(eration)f(ha)o(v)o(e)g(completed,)75 1965 y(so)e(that)g(no)g(error)g(v)m(alue)i(can)e(b)q(e)h(used)g(to)f (indicate)i(the)f(nature)f(of)g(the)h(error)e(\(e.g.,)h(an)g(error)g (in)h(a)f(send)75 2021 y(with)j(the)g(ready)f(mo)q(de\).)20 b(Suc)o(h)15 b(an)g(error)f(m)o(ust)g(b)q(e)i(treated)e(as)g(fatal,)g (since)i(information)f(cannot)f(b)q(e)75 2078 y(returned)i(for)e(the)h (user)h(to)e(reco)o(v)o(er)h(from)f(it.)166 2137 y(The)21 b(b)q(eha)o(vior)g(of)g(erroneous)f(programs)g(is)h(not)g(de\014ned)h (b)o(y)e(this)i(do)q(cumen)o(t.)37 b(The)21 b(desired)75 2193 y(b)q(eha)o(vior)14 b(is)g(that)e(a)i(relev)m(an)o(t)g(error)e(co) q(de)i(b)q(e)g(returned,)g(if)g(no)f(acceptable)i(b)q(eha)o(vior)e(can) h(b)q(e)g(de\014ned.)75 2250 y(Ho)o(w)o(ev)o(er,)23 b(e\016ciency)g (considerations)g(ma)o(y)f(dictate)g(that)g(the)g(system)g(do)q(es)g (not)g(detect)h(certain)75 2306 y(t)o(yp)q(es)c(of)f(errors.)29 b(Also,)19 b(implemen)o(tations)h(ma)o(y)d(go)h(b)q(ey)o(ond)i(this)e (do)q(cumen)o(t)h(in)h(supp)q(orting)f(in)g(a)75 2362 y(meanningful)e(manners)e(co)q(des)h(that)e(are)h(de\014ned)i(here)e (to)g(b)q(e)h(erroneous.)166 2421 y(MPI,)f(lik)o(e)h(an)o(y)f(other)g (similar)h(message-passing)f(library)l(,)h(allo)o(ws)f(users)h(to)e (write)h(non-determ-)75 2478 y(inistic)20 b(programs,)e(where)h(race)f (conditions)i(exist)f(and)f(di\013eren)o(t)h(timing)g(of)f(comm)o (unication)i(ma)o(y)75 2534 y(lead)14 b(to)f(di\013eren)o(t)h(results.) 19 b(There)14 b(is)g(no)g(requiremen)o(t)g(that)e(rep)q(eated)i (executions)h(of)e(the)g(same)h(suc)o(h)75 2591 y(program)c(will)j (yield)h(the)d(same)g(result,)i(ev)o(en)e(when)h(rep)q(eated)g(with)g (the)g(same)f(ph)o(ysical)i(con\014guration)75 2647 y(|)i(neither)g (user)f(nor)g(MPI)g(can)g(b)q(e)h(exp)q(ected)g(to)f(ha)o(v)o(e)g(full) h(con)o(trol)f(of)g(the)g(execution)h(en)o(vironmen)o(t.)75 2704 y(Ho)o(w)o(ev)o(er,)20 b(rep)q(eatabilit)o(y)i(is)f(a)f(desirable) j(qualit)o(y-of-implemen)o(tation)f(goal.)35 b(In)22 b(particular,)g(it)e(is)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 13 20 13 19 bop 75 -100 a Fp(2.7.)34 b(IMPLEMENT)l(A)l(TION)16 b(ISSUES)1045 b FD(13)75 45 y(exp)q(ected)13 b(that)e(MPI)g(implemen)o (tations)i(will)g(a)o(v)o(oid)e(in)o(tro)q(ducing)i(additional)g (non-determinism.)21 b(F)l(or)75 102 y(example,)e(while)g(MPI)f(can)g (tak)o(e)f(adv)m(an)o(tage)g(of)g(asso)q(ciativit)o(y)h(and)g(comm)o (utativit)o(y)f(in)i(reordering)75 158 y(the)f(execution)g(of)f(a)h (global)g(reduce)g(op)q(eration,)g(it)g(is)g(desired)h(\(but)e(not)g (required\))i(that)e(the)g(same)75 214 y(op)q(erations)e(order)g(b)q(e) h(used)g(in)g(rep)q(eated)g(executions.)75 357 y Fq(2.7)59 b(Implementation)17 b(issues)75 459 y FD(There)c(are)f(a)g(n)o(um)o(b)q (er)h(of)f(areas)f(where)i(an)f(MPI)h(implemen)o(tation)g(ma)o(y)f(in)o (teract)g(with)h(the)g(op)q(erating)75 515 y(en)o(vironmen)o(t)k(and)f (system.)23 b(While)18 b(MPI)e(do)q(es)h(not)f(mandate)g(that)f(an)o(y) h(services)h(\(suc)o(h)g(as)f(I/O)g(or)75 571 y(signal)h(handling\))g (b)q(e)f(pro)o(vided,)g(it)g(do)q(es)g(strongly)g(suggest)f(the)h(b)q (eha)o(vior)g(to)f(b)q(e)h(pro)o(vided)h(if)f(those)75 628 y(services)g(are)g(a)o(v)m(ailable.)23 b(This)16 b(is)g(an)f(imp)q(ortan)o(t)h(p)q(oin)o(t)g(in)g(ac)o(hieving)h(p)q (ortabilit)o(y)g(across)d(platforms)75 684 y(that)g(pro)o(vide)i(the)g (same)e(set)h(of)g(services.)75 806 y Fn(2.7.1)49 b(Indep)q(endence)15 b(of)i(basic)f(runtime)e(routines)75 891 y FD(MPI)f(programs)e(require) j(that)e(library)i(routines)f(that)f(are)h(part)f(of)g(the)h(basic)h (language)f(en)o(vironmen)o(t)75 948 y(\(suc)o(h)i(as)g FB(date)f FD(and)i FB(write)e FD(in)i(F)l(ortran)e(and)h FB(printf)f FD(and)h FB(malloc)g FD(in)h(ANSI)f(C\))g(and)g(are)g (executed)75 1004 y(after)g FB(MPI)p 258 1004 15 2 v 17 w(INIT)g FD(and)i(b)q(efore)f FB(MPI)p 685 1004 V 17 w(FINALIZE)f FD(op)q(erate)g(indep)q(enden)o(tly)k(and)e(that)e (their)h Fo(c)n(ompletion)75 1061 y FD(is)g(indep)q(enden)o(t)h(of)e (the)g(action)h(of)e(other)h(pro)q(cesses)h(in)g(an)f(MPI)g(program.) 166 1117 y(Note)f(that)g(this)h(in)g(no)f(w)o(a)o(y)g(prev)o(en)o(ts)g (the)g(creation)h(of)f(library)h(routines)g(that)e(pro)o(vide)i (parallel)75 1174 y(services)h(whose)f(op)q(eration)h(is)g(collectiv)o (e.)22 b(Ho)o(w)o(ev)o(er,)14 b(the)h(follo)o(wing)h(program)e(is)i (exp)q(ected)g(to)f(com-)75 1230 y(plete)j(in)g(an)f(ANSI)h(C)f(en)o (vironmen)o(t)h(regardless)f(of)g(the)g(size)h(of)f FB(MPI)p 1312 1230 V 16 w(COMM)p 1424 1230 V 17 w(WORLD)g FD(\(assuming)g(that) 75 1287 y(I/O)f(is)f(a)o(v)m(ailable)i(at)e(the)g(executing)h(no)q (des\).)75 1377 y FB(int)23 b(rank;)75 1434 y(MPI_Init\()g(argc,)g (argv)g(\);)75 1490 y(MPI_Comm_rank\()f(MPI_COMM_WORLD,)g(&rank)h(\);) 75 1547 y(if)h(\(rank)f(==)g(0\))h(printf\()f("Starting)f(program\\n")h (\);)75 1603 y(MPI_Finalize\(\);)75 1694 y FD(The)15 b(corresp)q(onding)i(F)l(ortran)d(77)g(program)g(is)i(also)f(exp)q (ected)i(to)d(complete.)166 1750 y(An)k(example)h(of)f(what)f(is)i Fo(not)j FD(required)d(is)g(an)o(y)f(particular)h(ordering)f(of)g(the)g (action)g(of)g(these)75 1807 y(routines)h(when)h(called)g(b)o(y)f(sev)o (eral)h(tasks.)30 b(F)l(or)18 b(example,)j(MPI)e(mak)o(es)f(neither)i (requiremen)o(ts)f(or)75 1863 y(recommendations)f(for)f(the)g(output)g (from)g(the)h(follo)o(wing)g(program)e(\(again)h(assuming)h(that)f(I/O) g(is)75 1920 y(a)o(v)m(ailable)g(at)d(the)i(executing)g(no)q(des\).)75 2011 y FB(MPI_Comm_rank\()22 b(MPI_COMM_WORLD,)g(&rank)h(\);)75 2067 y(printf\()g("Output)g(from)g(task)g(rank)h(\045d\\n",)f(rank)g (\);)166 2158 y FD(In)18 b(addition,)h(calls)f(that)f(fail)h(b)q (ecause)g(of)f(resource)h(exhaustion)g(or)f(other)g(error)g(are)g(not)g (con-)75 2214 y(sidered)h(a)f(violation)h(of)e(the)h(requiremen)o(ts)h (here)f(\(ho)o(w)o(ev)o(er,)f(they)h(are)g(required)h(to)e(complete,)i (just)75 2271 y(not)d(to)f(complete)i(successfully\).)75 2392 y Fn(2.7.2)49 b(Interaction)16 b(with)g(signals)i(in)f(POSIX)75 2478 y FD(The)e(in)o(ten)o(t)g(of)f(MPI)h(is)g(to)f(b)q(e)h(thread)g (and)f(signal)i(safe.)j(This)d(has)e(a)g(n)o(um)o(b)q(er)h(of)g(subtle) g(e\013ects.)20 b(F)l(or)75 2534 y(example,)g(on)e(Unix)h(systems,)g(a) f(catc)o(hable)h(signal)g(suc)o(h)g(as)f(SIGALRM)h(\(an)f(alarm)h (signal\))g(m)o(ust)75 2591 y(not)f(cause)g(an)h(MPI)f(routine)g(to)g (b)q(eha)o(v)o(e)h(di\013eren)o(tly)g(than)f(it)g(w)o(ould)h(ha)o(v)o (e)f(in)h(the)f(absence)h(of)f(the)75 2647 y(signal.)37 b(Of)21 b(course,)h(if)f(the)g(signal)h(handler)f(issues)h(MPI)f(calls) g(or)g(c)o(hanges)f(the)h(en)o(vironmen)o(t)g(in)75 2704 y(whic)o(h)c(the)g(MPI)g(routine)g(is)g(op)q(erating)f(\(for)g (example,)h(consuming)h(all)f(a)o(v)m(ailable)h(memory)e(space\),)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 14 21 14 20 bop 75 -100 a FD(14)692 b Fp(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y FD(the)f(MPI)f(routine)h (should)h(b)q(eha)o(v)o(e)f(as)f(appropriate)h(for)f(that)g(situation)h (\(in)g(particular,)g(in)g(this)g(case,)75 102 y(the)h(b)q(eha)o(vior)h (should)g(b)q(e)g(the)f(same)g(as)g(for)g(a)g(m)o(ultithreaded)h(MPI)f (implemen)o(tation\).)166 158 y(A)20 b(second)g(e\013ect)g(is)g(that)g (a)f(signal)i(handler)g(that)e(p)q(erforms)h(MPI)g(calls)h(m)o(ust)e (not)h(in)o(terfere)75 214 y(with)c(the)h(op)q(eration)f(of)g(MPI.)f(F) l(or)h(example,)g(an)h(MPI)f(receiv)o(e)h(of)e(an)o(y)h(t)o(yp)q(e)g (that)g(o)q(ccurs)g(within)h(a)75 271 y(signal)c(handler)f(m)o(ust)f (not)h(cause)g(erroneous)f(b)q(eha)o(vior)i(b)o(y)e(the)h(MPI)g (implemen)o(tation.)20 b(Note)11 b(that)g(an)75 327 y(implemen)o (tation)16 b(is)g(p)q(ermitted)g(to)e(prohibit)i(the)f(use)h(of)e(MPI)h (calls)h(from)e(within)j(a)d(signal)i(handler,)75 384 y(and)f(is)h(not)f(required)h(to)f(detect)g(suc)o(h)h(use.)166 440 y(It)d(is)h(highly)g(desirable)h(that)d(MPI)h(not)g(use)g FB(SIGALRM)p FD(,)f FB(SIGFPE)p FD(,)g(or)g FB(SIGIO)p FD(.)g(An)i(implemen)o(tation)75 497 y(is)g Fo(r)n(e)n(quir)n(e)n(d)k FD(to)c(clearly)h(do)q(cumen)o(t)f(all)h(of)f(the)g(signals)h(that)e (the)h(MPI)g(implemen)o(tation)h(uses;)f(a)g(go)q(o)q(d)75 553 y(place)i(for)f(this)g(information)h(is)f(a)g(Unix)h(`)p FB(man)p FD(')e(page)h(on)h FB(MPI)p FD(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 15 22 15 21 bop 75 394 a Fr(Chapter)34 b(3)75 640 y Fy(P)m(oin)m(t)41 b(to)f(P)m(oin)m(t)h(Comm)m(unication)75 919 y Fq(3.1)59 b(Intro)r(duction)75 1094 y FD(Sending)13 b(and)f(receiving)h(of)e (messages)f(b)o(y)i(pro)q(cesses)g(is)g(the)f(basic)h(MPI)g(comm)o (unication)g(mec)o(hanism.)75 1151 y(All)20 b(other)f(comm)o(unication) h(functions)g(can)f(b)q(e)h(de\014ned)h(and)e(implemen)o(ted)i(on)e (top)g(of)g(this)g(basic)75 1207 y(comm)o(unication)e(la)o(y)o(er,)f (using)h(few)f(auxiliary)i(data)e(managemen)o(t)f(routines,)i(and)g (using)g(the)f(group)75 1263 y(and)f(comm)o(unicator)g(manipulation)i (functions.)166 1358 y(The)i(basic)g(p)q(oin)o(t)g(to)f(p)q(oin)o(t)h (comm)o(unication)g(op)q(erations)g(are)f Fx(send)g FD(and)h Fx(receiv)o(e)p FD(.)29 b(A)18 b Fx(send)75 1414 y FD(op)q(eration)13 b(creates)f(and)h(sends)g(a)g(message.)18 b(The)13 b(op)q(eration)g(sp) q(eci\014es)i(a)d Fx(send)i(bu\013er)e FD(in)i(the)e(sender)75 1471 y(memory)f(from)g(whic)o(h)h(the)g(message)f(data)g(is)h(tak)o (en.)18 b(In)12 b(addition,)h(the)f(send)g(op)q(eration)g(asso)q (ciates)f(an)75 1527 y Fx(en)o(v)o(elop)q(e)16 b FD(with)g(the)g (message.)22 b(This)16 b(en)o(v)o(elop)q(e)h(sp)q(eci\014es)h(the)e (message)g(destination)h(and)f(con)o(tains)75 1584 y(distinguishing)h (information)d(that)f(can)i(b)q(e)g(used)f(b)o(y)g(the)g Fx(receiv)o(e)g FD(op)q(eration)g(to)g(select)h(a)e(particular)75 1640 y(message.)166 1735 y(A)22 b Fx(receiv)o(e)f FD(op)q(eration)h (consumes)g(a)f(message.)39 b(The)22 b(message)f(to)g(b)q(e)i(receiv)o (ed)f(is)h(selected)75 1791 y(according)13 b(to)f(the)g(v)m(alue)i(on)e (its)g(en)o(v)o(elop)q(e,)i(and)f(the)f(message)g(data)g(is)h(put)f(in) o(to)g(the)h Fx(receiv)o(e)g(bu\013er)p FD(.)166 1886 y(The)f(next)g(sections)h(describ)q(e)g(the)g(basic)f(\(blo)q(c)o (king\))h(send)g(and)f(receiv)o(e)h(op)q(erations.)19 b(W)l(e)12 b(discuss)75 1942 y(send,)21 b(receiv)o(e,)g(basic)f(comm)o (unication)g(seman)o(tics,)h(t)o(yp)q(e)e(matc)o(hing)h(requiremen)o (ts,)h(t)o(yp)q(e)e(con)o(v)o(er-)75 1998 y(sion)14 b(in)h (heterogeneous)f(en)o(vironmen)o(ts,)f(and)h(more)g(general)g(comm)o (unication)g(mo)q(des.)20 b(Non)o(blo)q(c)o(king)75 2055 y(comm)o(unication)c(is)h(addressed)f(next,)f(follo)o(w)o(ed)h(b)o(y)g (c)o(hannel-lik)o(e)i(constructs)e(and)f(send-receiv)o(e)j(op-)75 2111 y(erations.)i(W)l(e)15 b(then)g(consider)h(general)f(datat)o(yp)q (es)f(that)g(allo)o(w)h(one)g(to)g(transfer)f(heterogeneous)h(and)75 2168 y(noncon)o(tiguous)g(data.)75 2528 y Fq(3.2)59 b(Basic)19 b(send)h(op)r(eration)75 2704 y FD(The)15 b(syn)o(tax)g(of)g(the)g (simplest)h(send)g(op)q(eration)f(is)h(giv)o(en)g(b)q(elo)o(w.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 16 23 16 22 bop 75 -100 a FD(16)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(SEND\(buf,)g(count,)g(datat)o(yp)q(e,)g(dest,)g (tag,)f(comm)m(\))117 126 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 209 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts) g(in)g(send)h(bu\013er)h(\(nonnegativ)o(e)e(in)o(te-)905 265 y(ger\))117 348 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 431 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 514 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 596 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))75 725 y FB(int)23 b(MPI)p 245 725 15 2 v 17 w(Send\(void*)g(buf,)g(int)g (count,)h(MPI)p 979 725 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 781 y(int)h(tag,)f(MPI)p 683 781 V 17 w(Comm)g(comm\))75 871 y(MPI)p 150 871 V 17 w(SEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)h(COMM,)f(IERROR\))170 928 y()g(BUF\(*\))170 984 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 1158 y Fn(3.2.1)49 b(Message)16 b(data)75 1251 y FD(The)e(send)g (bu\013er)g(sp)q(eci\014ed)i(b)o(y)e(the)f Fs(MPI)p 803 1251 14 2 v 16 w(SEND)h FD(op)q(eration)g(consists)g(of)f Fs(count)i FD(successiv)o(e)g(en)o(tries)f(of)75 1308 y(the)f(t)o(yp)q(e)h(indicated)h(b)o(y)e Fs(datat)o(yp)q(e)p FD(,)i(starting)e(with)h(the)f(en)o(try)g(at)g(address)h Fs(buf)p FD(.)20 b(Note)13 b(that)g(w)o(e)g(sp)q(ecify)75 1364 y(the)19 b(message)f(length)i(in)g(terms)e(of)g(n)o(um)o(b)q(er)h (of)g Fo(elements)p FD(,)f(not)g(n)o(um)o(b)q(er)i(of)e Fo(bytes)p FD(.)30 b(The)19 b(former)f(is)75 1421 y(mac)o(hine)e(indep) q(enden)o(t)i(and)d(closer)h(to)e(the)h(application)i(lev)o(el.)166 1481 y(The)g(data)f(part)g(of)g(the)g(message)h(consists)f(of)h(a)f (sequence)i(of)e Fs(count)i FD(v)m(alues,)f(eac)o(h)g(of)f(the)h(t)o (yp)q(e)75 1537 y(indicated)j(b)o(y)e Fs(datat)o(yp)q(e)p FD(.)31 b Fs(count)20 b FD(ma)o(y)d(b)q(e)i(zero,)g(in)g(whic)o(h)g (case)g(the)f(data)g(part)f(of)h(the)h(message)e(is)75 1594 y(empt)o(y)l(.)i(The)13 b(basic)h(datat)o(yp)q(es)e(that)h(can)g (b)q(e)h(sp)q(eci\014ed)h(for)d(message)h(data)f(v)m(alues)i(corresp)q (ond)g(to)e(the)75 1650 y(basic)j(datat)o(yp)q(es)f(of)g(the)h(host)f (language.)19 b(The)c(p)q(ossible)h(v)m(alues)g(of)e(this)h(argumen)o (t)f(for)f(F)l(ortran)h(and)75 1707 y(the)h(corresp)q(onding)h(F)l (ortran)e(t)o(yp)q(es)i(are)f(listed)h(b)q(elo)o(w)p 472 1892 1006 2 v 471 1948 2 57 v 497 1931 a(MPI)f(datat)o(yp)q(e)p 1046 1948 V 294 w(F)l(ortran)f(datat)o(yp)q(e)p 1477 1948 V 472 1950 1006 2 v 471 2006 2 57 v 497 1989 a Fs(MPI)p 582 1989 14 2 v 16 w(INTEGER)p 1046 2006 2 57 v 287 w FB(INTEGER)p 1477 2006 V 471 2063 V 497 2046 a Fs(MPI)p 582 2046 14 2 v 16 w(REAL)p 1046 2063 2 57 v 365 w FB(REAL)p 1477 2063 V 471 2119 V 497 2102 a Fs(MPI)p 582 2102 14 2 v 16 w(DOUBLE)p 777 2102 V 17 w(PRECISION)p 1046 2119 2 57 v 50 w FB(DOUBLE)23 b(PRECISION)p 1477 2119 V 471 2176 V 497 2159 a Fs(MPI)p 582 2159 14 2 v 16 w(COMPLEX)p 1046 2176 2 57 v 263 w FB(COMPLEX)p 1477 2176 V 471 2232 V 497 2215 a Fs(MPI)p 582 2215 14 2 v 16 w(LOGICAL)p 1046 2232 2 57 v 291 w FB(LOGICAL)p 1477 2232 V 471 2289 V 497 2272 a Fs(MPI)p 582 2272 14 2 v 16 w(CHARA)o(CTER)p 1046 2289 2 57 v 211 w FB(CHARACTER\(1\))p 1477 2289 V 471 2345 V 497 2328 a Fs(MPI)p 582 2328 14 2 v 16 w(BYTE)p 1046 2345 2 57 v 1477 2345 V 472 2347 1006 2 v 75 2549 a FD(The)13 b(p)q(ossible)i(v)m(alues)g(for)d(this)i(argumen)o(t)f(for) f(C)h(and)h(the)f(corresp)q(onding)h(C)f(t)o(yp)q(es)g(are)g(listed)i (b)q(elo)o(w.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 24 17 23 bop 75 -100 a Fp(3.2.)34 b(BASIC)16 b(SEND)f(OPERA)l(TION)1075 b FD(17)p 466 5 1019 2 v 465 62 2 57 v 491 45 a(MPI)15 b(datat)o(yp)q(e)p 1004 62 V 259 w(C)g(datat)o(yp)q(e)p 1483 62 V 466 63 1019 2 v 465 120 2 57 v 491 103 a Fs(MPI)p 576 103 14 2 v 16 w(CHAR)p 1004 120 2 57 v 321 w FB(signed)23 b(char)p 1483 120 V 465 176 V 491 159 a Fs(MPI)p 576 159 14 2 v 16 w(SHORT)p 1004 176 2 57 v 291 w FB(signed)g(short)p 1483 176 V 465 233 V 491 216 a Fs(MPI)p 576 216 14 2 v 16 w(INT)p 1004 233 2 57 v 365 w FB(signed)g(int)p 1483 233 V 465 289 V 491 272 a Fs(MPI)p 576 272 14 2 v 16 w(LONG)p 1004 289 2 57 v 321 w FB(signed)g(long)g(int)p 1483 289 V 465 346 V 491 329 a Fs(MPI)p 576 329 14 2 v 16 w(UNSIGNED)p 815 329 V 17 w(CHAR)p 1004 346 2 57 v 81 w FB(unsigned)f(char)p 1483 346 V 465 402 V 491 385 a Fs(MPI)p 576 385 14 2 v 16 w(UNSIGNED)p 815 385 V 17 w(SHORT)p 1004 402 2 57 v 51 w FB(unsigned)g(short)i(int)p 1483 402 V 465 459 V 491 442 a Fs(MPI)p 576 442 14 2 v 16 w(UNSIGNED)p 1004 459 2 57 v 218 w FB(unsigned)e(int)p 1483 459 V 465 515 V 491 498 a Fs(MPI)p 576 498 14 2 v 16 w(UNSIGNED)p 815 498 V 17 w(LONG)p 1004 515 2 57 v 81 w FB(unsigned)g(long)i(int)p 1483 515 V 465 571 V 491 555 a Fs(MPI)p 576 555 14 2 v 16 w(FLO)o(A)l(T)p 1004 571 2 57 v 301 w FB(float)p 1483 571 V 465 628 V 491 611 a Fs(MPI)p 576 611 14 2 v 16 w(DOUBLE)p 1004 628 2 57 v 262 w FB(double)p 1483 628 V 465 684 V 491 667 a Fs(MPI)p 576 667 14 2 v 16 w(LONG)p 712 667 V 16 w(DOUBLE)p 1004 684 2 57 v 126 w FB(long)f(double)p 1483 684 V 465 741 V 491 724 a Fs(MPI)p 576 724 14 2 v 16 w(BYTE)p 1004 741 2 57 v 1483 741 V 466 742 1019 2 v 166 912 a FD(The)16 b(datat)o(yp)q(e)f Fs(MPI)p 536 912 14 2 v 15 w(BYTE)i FD(do)q(es)e(not)g(corresp)q(ond)h(to)f(a)g(F)l (ortran)g(or)g(C)g(datat)o(yp)q(e.)20 b(A)c(v)m(alue)g(of)75 968 y(t)o(yp)q(e)d Fs(MPI)p 260 968 V 16 w(BYTE)h FD(consists)f(of)g(a) g(b)o(yte)g(\(8)g(binary)h(digits\).)19 b(A)13 b(b)o(yte)h(is)f(unin)o (terpreted)i(and)e(is)h(di\013eren)o(t)75 1025 y(from)i(a)g(c)o (haracter.)23 b(Di\013eren)o(t)16 b(mac)o(hines)h(ma)o(y)f(ha)o(v)o(e)g (di\013eren)o(t)h(represen)o(tations)f(for)g(c)o(haracters,)f(or)75 1081 y(ma)o(y)i(use)i(more)e(than)h(one)g(b)o(yte)g(to)f(represen)o(t)h (c)o(haracters.)28 b(On)18 b(the)g(other)g(hand,)h(a)e(b)o(yte)h(has)g (the)75 1138 y(same)d(binary)h(v)m(alue)g(on)f(all)h(mac)o(hines.)166 1194 y(MPI)11 b(supp)q(orts)g(the)f(basic)i(datat)o(yp)q(es)e(of)g(F)l (ortran)g(77)g(and)h(ANSI)h(C.)e(Additional)j(MPI)e(datat)o(yp)q(es)75 1251 y(should)32 b(b)q(e)g(pro)o(vided)f(if)h(the)f(host)f(language)h (has)g(additional)i(data)d(t)o(yp)q(es:)51 b(F)l(or)30 b(example,)75 1307 y Fs(MPI)p 160 1307 V 16 w(LONG)p 296 1307 V 17 w(LONG)p 433 1307 V 17 w(INT)p FD(,)13 b(for)h(64)g(bit)h(in)o(tegers)g(in)g(C;)f Fs(MPI)p 1113 1307 V 16 w(DOUBLE)p 1308 1307 V 16 w(COMPLEX)h FD(for)f(double)i (preci-)75 1364 y(sion)h(complex)g(in)g(F)l(ortran;)e Fs(MPI)p 667 1364 V 16 w(REAL2)p FD(,)h Fs(MPI)p 928 1364 V 16 w(REAL4)g FD(and)h Fs(MPI)p 1266 1364 V 15 w(REAL8)g FD(for)e(2,4)g(or)h(8)g(b)o(yte)g(reals,)75 1420 y(in)g(F)l(ortran;)e Fs(MPI)p 391 1420 V 15 w(INTEGER1)i(MPI)p 716 1420 V 16 w(INTEGER2)f FD(and)g Fs(MPI)p 1129 1420 V 16 w(INTEGER4)h FD(for)e(1,)h(2)g(or)f(4)h(b)o(yte)g(in)o(tegers)75 1476 y(in)h(F)l(ortran;)e(etc.)189 1581 y Fo(R)n(ationale.)77 b FD(One)23 b(goal)e(of)g(the)h(design)g(is)g(to)f(allo)o(w)h(for)e (MPI)i(to)f(b)q(e)h(implemen)o(ted)h(as)e(a)189 1637 y(library)l(,)d(with)f(no)g(need)h(for)e(additional)j(prepro)q(cessing) f(or)e(compilation.)27 b(Th)o(us,)17 b(one)g(cannot)189 1693 y(assume)c(that)g(a)g(comm)o(unication)h(call)g(has)g(information) f(on)h(the)f(datat)o(yp)q(e)g(of)g(v)m(ariables)i(in)f(the)189 1750 y(comm)o(unication)20 b(bu\013er;)i(this)f(information)f(m)o(ust)f (b)q(e)i(supplied)h(b)o(y)e(an)g(explicit)i(argumen)o(t.)189 1806 y(The)e(need)h(for)f(suc)o(h)g(datat)o(yp)q(e)g(information)g (will)i(b)q(ecome)f(clear)f(in)h(section)g(3.6.)34 b(\()p Fo(End)20 b(of)189 1863 y(r)n(ationale.)p FD(\))75 1984 y Fn(3.2.2)49 b(Message)16 b(envelop)q(e)75 2070 y FD(In)g(addition)g (to)e(the)h(data)g(part,)f(messages)g(con)o(tain)h(information)h(that)e (can)h(b)q(e)h(used)f(to)g(distinguish)75 2127 y(messages)h(and)g (selectiv)o(ely)i(receiv)o(e)f(them.)23 b(This)17 b(information)f(is)h (con)o(tained)g(in)g(a)f(\014xed)h(n)o(um)o(b)q(er)f(of)75 2183 y(\014xed-format)f(\014elds,)h(whic)o(h)g(w)o(e)f(collectiv)o(ely) i(call)g(the)e Fx(message)i(en)o(v)o(elop)q(e)p FD(.)i(These)d (\014elds)g(are)75 2275 y Fx(source)75 2368 y(destination)75 2462 y(tag)75 2555 y(con)o(text)166 2647 y FD(The)c(message)g(source)g (is)h(implicitly)i(determined)f(b)o(y)e(the)g(iden)o(tit)o(y)h(of)f (the)g(message)g(sender.)20 b(The)75 2704 y(other)15 b(\014elds)h(are)f(sp)q(eci\014ed)i(b)o(y)f(argumen)o(ts)e(in)i(the)f (send)h(op)q(eration.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 18 25 18 24 bop 75 -100 a FD(18)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FD(The)h(in)o (teger-v)m(alued)h(message)e(tag)f(is)i(sp)q(eci\014ed)i(b)o(y)d(the)g Fs(tag)h FD(argumen)o(t.)22 b(This)17 b(in)o(teger)g(can)f(b)q(e)75 102 y(used)c(b)o(y)g(the)f(program)g(to)g(distinguish)i(di\013eren)o(t) f(t)o(yp)q(es)g(of)f(messages.)18 b(The)12 b(range)f(of)g(v)m(alid)i (tag)e(v)m(alues)75 158 y(is)18 b(implemen)o(tation)h(dep)q(enden)o(t)g (and)f(can)f(b)q(e)h(queried)h(using)f(the)g Fs(MPI)p 1362 158 14 2 v 16 w(GET)p 1466 158 V 17 w(V)l(ALID)p 1610 158 V 16 w(T)l(A)o(G)p 1712 158 V 17 w(RANGE)75 214 y FD(en)o(vironmen)o(tal)e(inquiry)h(function,)e(as)g(describ)q(ed) i(in)f(Chapter)f(7.)166 382 y Fl(Discussion:)34 b Ft(Alternativ)o(e)14 b(design)g(\(on)g(the)g(\\Europ)q(ean)h(wish)e(list"\):)166 474 y(The)22 b(range)g(of)g(v)n(alid)e(tag)h(v)n(alues)h(is)g(0)p Fu(:::U)5 b(B)r Ft(,)22 b(where)h(the)f(v)n(alue)f(of)h Fu(U)5 b(B)24 b Ft(can)e(b)q(e)g(found)g(using)f(the)75 531 y Fv(MPI)p 152 531 13 2 v 14 w(GET)p 247 531 V 15 w(V)m(ALID)p 379 531 V 14 w(T)m(A)o(G)p 473 531 V 14 w(RANGE)16 b Ft(en)o(vironmen)o(tal)e(inquiry)i(function,)h(as)f (describ)q(ed)j(in)d(Chapter)h(7.)26 b Fl(\(Need)75 587 y(to)14 b(c)o(hange)f(the)g(de\014nition)d(of)k(this)e(inquiry)g (function.\))j Ft(The)e(v)n(alue)f(of)g Fu(U)5 b(B)15 b Ft(shall)c(b)q(e)i(no)g(less)g(than)f(127.)75 644 y(W)m(e)h(exp)q (ect)j(go)q(o)q(d)d(qualit)o(y)g(MPI)h(implemen)o(tations)d(to)j(pro)o (vide)f(a)h(m)o(uc)o(h)f(larger)g(tagt)h(range.)166 818 y FD(The)21 b(con)o(text)f(of)g(the)g(message)g(sen)o(t)g(is)h(sp)q (eci\014ed)i(b)o(y)e(the)f Fs(comm)15 b FD(argumen)o(t.)35 b(The)20 b(message)75 874 y(carries)11 b(the)g(con)o(text)f(asso)q (ciated)h(with)g(this)g(comm)o(unicator)g(\(see)f(Chapter)h(5\).)18 b(Eac)o(h)10 b(comm)o(unication)75 931 y(con)o(text)h(pro)o(vides)i(a)e (separate)h(\\comm)o(unication)g(univ)o(erse":)19 b(messages)12 b(are)f(alw)o(a)o(ys)h(receiv)o(ed)h(within)75 987 y(the)k(con)o(text)f (they)h(w)o(ere)g(sen)o(t,)g(and)g(messages)f(sen)o(t)h(in)g (di\013eren)o(t)g(con)o(texts)g(do)g(not)f(in)o(terfere.)25 b(This)75 1044 y(pro)o(vides)16 b(a)f(p)q(o)o(w)o(erful)g (encapsulation)i(mec)o(hanism)e(for)g(libraries)i(and)e(mo)q(dules.)166 1136 y(The)f(message)f(destination)h(is)h(sp)q(eci\014ed)g(b)o(y)f(the) f Fs(dest)j FD(argumen)o(t)c(as)i(a)f(rank)g(within)i(the)f(pro)q(cess) 75 1192 y(group)i(asso)q(ciated)h(with)g(that)f(same)h(comm)o(unicator) f(\(remote)g(pro)q(cess)h(group,)f(for)g(in)o(tercomm)o(uni-)75 1249 y(cators\).)32 b(The)19 b(range)h(of)f(v)m(alid)i(v)m(alues)g(is)f Fs(0,)g(...)32 b(,)20 b(n-1)p FD(,)h(where)e Fs(n)h FD(is)h(the)e(n)o (um)o(b)q(er)h(of)f(pro)q(cesses)h(in)75 1305 y(this)e(group.)25 b(Th)o(us,)17 b(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unications)g(do)f (not)g(use)g(absolute)h(addresses,)f(but)g(only)75 1361 y(relativ)o(e)f(ranks)f(within)h(a)f(group.)k(This)d(pro)o(vides)g(imp) q(ortan)o(t)f(mo)q(dularit)o(y)l(.)166 1453 y(The)j(message)f(en)o(v)o (elop)q(e)i(w)o(ould)f(normally)g(b)q(e)h(enco)q(ded)g(b)o(y)e(a)g (\014xed-length)j(message)d(header.)75 1510 y(Ho)o(w)o(ev)o(er,)c(the)i (actual)f(mec)o(hanism)h(used)g(to)f(asso)q(ciate)g(an)g(en)o(v)o(elop) q(e)i(with)e(a)g(message)g(is)h(implemen-)75 1566 y(tation)d(dep)q (enden)o(t:)20 b(some)12 b(of)g(the)g(information)g(\(e.g.,)g(source)g (or)g(destination\))g(ma)o(y)g(b)q(e)h(implicit,)i(and)75 1623 y(need)f(not)g(b)q(e)g(explicitly)i(carried)e(b)o(y)g(messages;)f (pro)q(cesses)h(ma)o(y)f(b)q(e)h(iden)o(ti\014ed)i(b)o(y)d(relativ)o(e) h(ranks,)f(or)75 1679 y(absolute)j(ids;)f(etc.)189 1962 y Fo(A)n(dvic)n(e)j(to)i(users.)60 b FD(Users)18 b(that)g(are)g (comfortable)g(with)h(the)g(notion)g(of)f(a)g(\015at)g(name)h(space)189 2019 y(for)14 b(pro)q(cesses,)h(and)h(a)f(unique)h(comm)o(unication)g (con)o(text,)f(as)f(o\013ered)h(b)o(y)g(most)g(existing)h(com-)189 2075 y(m)o(unication)f(libraries,)h(need)f(only)h(use)e(the)h (prede\014ned)h(v)m(ariable)g Fv(MPI)p 1437 2075 V 15 w(COMM)p 1582 2075 V 14 w(W)o(ORLD)f FD(as)f(the)189 2132 y Fs(comm)d FD(argumen)o(t.)25 b(This)17 b(will)i(allo)o(w)f(to)e (comm)o(unicate)i(with)f(all)h(the)f(pro)q(cesses)h(a)o(v)m(ailable)h (at)189 2188 y(initialization)f(time.)i(\()p Fo(End)c(of)g(advic)n(e)g (to)h(users.)p FD(\))75 2533 y Fq(3.3)59 b(Basic)19 b(receive)g(op)r (eration)75 2704 y FD(The)c(syn)o(tax)g(of)g(the)g(simplest)h(receiv)o (e)g(op)q(eration)g(is)f(giv)o(en)h(b)q(elo)o(w.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 19 26 19 25 bop 75 -100 a Fp(3.3.)34 b(BASIC)16 b(RECEIVE)g(OPERA)l(TION)990 b FD(19)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(RECV)16 b(\(buf,)f(count,)h (datat)o(yp)q(e,)h(source,)e(tag,)g(comm)m(,)d(status\))117 122 y Ft(OUT)108 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(receiv)o (e)h(bu\013er)g(\(c)o(hoice\))117 197 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 271 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(handle\))117 345 y(IN)155 b Fs(source)468 b Ft(rank)14 b(of)f(source)j(\(in)o(teger\))117 420 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 494 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 568 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\)) 75 693 y FB(int)23 b(MPI)p 245 693 15 2 v 17 w(Recv\(void*)g(buf,)g (int)g(count,)h(MPI)p 979 693 V 16 w(Datatype)f(datatype,)g(int)g (source,)393 749 y(int)h(tag,)f(MPI)p 683 749 V 17 w(Comm)g(comm,)g (MPI)p 1034 749 V 17 w(Status)g(*status\))75 836 y(MPI)p 150 836 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g (COMM,)g(STATUS,)g(IERROR\))170 892 y()g(BUF\(*\))170 949 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 949 V 16 w(STATUS)p 1623 949 V 17 w(SIZE\),)170 1005 y(IERROR)166 1092 y FD(The)15 b(receiv)o(e)h (bu\013er)f(consists)h(of)e(the)h(storage)f(con)o(taining)i Fs(count)g FD(consecutiv)o(e)g(elemen)o(ts)g(of)f(the)75 1148 y(t)o(yp)q(e)e(sp)q(eci\014ed)i(b)o(y)e Fs(datat)o(yp)q(e)p FD(,)h(starting)e(at)h(address)f Fs(buf)p FD(.)20 b(The)13 b(length)h(of)e(the)h(receiv)o(ed)h(message)f(m)o(ust)75 1204 y(b)q(e)i(less)g(than)f(or)g(equal)h(to)e(the)i(length)g(of)e(the) i(receiv)o(e)g(bu\013er.)20 b(I.e.,)13 b(an)i(error)e(o)q(ccurs)i(if)g (all)g(incoming)75 1261 y(data)j(do)q(es)h(not)g(\014t,)g(without)g (truncation,)g(in)o(to)g(the)g(receiv)o(e)h(bu\013er.)30 b(The)19 b Fs(MPI)p 1531 1261 14 2 v 16 w(PROBE)h FD(function)75 1317 y(describ)q(ed)d(in)f(section)g(3.9)e(can)i(b)q(e)f(used)h(to)f (receiv)o(e)h(messages)f(of)f(unkno)o(wn)i(length.)166 1450 y Fl(Discussion:)32 b Ft(It)12 b(w)o(as)h(suggested)g(that)f(MPI)h (should)f(sp)q(ecify)h(the)f(b)q(eha)o(vior)g(of)g(receiv)o(e)i(when)e (incoming)93 b Fm(CHANGE)75 1500 y Ft(message)18 b(is)h(to)q(o)f(long.) 32 b(\(An)19 b(error)g(indication)f(is)g(returned)i(b)o(y)f(the)g (receiv)o(e)h(call,)f(and)f(the)h(error)h(handler)75 1549 y(is)e(called;)j(one)e(migh)o(t)d(still)i(w)o(an)o(t)g(the)h(con)o (ten)o(t)g(of)f(the)i(receiv)o(e)g(bu\013er)f(and)g(of)f(the)h(status)g (v)n(ariable)f(to)g(b)q(e)75 1599 y(w)o(ell-de\014ned.\))166 1649 y(The)c(w)o(eak)o(est)h(requiremen)o(ts)f(are)g(that:)145 1733 y Fj(\017)23 b Ft(No)13 b(area)h(outside)h(the)f(receiv)o(e)h (bu\013er)g(is)f(o)o(v)o(erwritten.)145 1806 y Fj(\017)23 b Fv(MPI)p 266 1806 13 2 v 14 w(GET)p 361 1806 V 15 w(TYPE)13 b Ft(and)g Fv(MPI)p 652 1806 V 15 w(GET)p 748 1806 V 14 w(SOURCE)f Ft(return)k(the)e(righ)o(t)f(answ)o(er.)145 1879 y Fj(\017)23 b Ft(The)12 b(v)n(alues)h(returned)h(b)o(y)e Fv(MPI)p 695 1879 V 14 w(GET)p 790 1879 V 15 w(COUNT)g Ft(and)g Fv(MPI)p 1115 1879 V 14 w(GET)p 1210 1879 V 15 w(ELEMENT)h Ft(indicate)f(ho)o(w)g(man)o(y)f(items)189 1929 y(\(resp.)19 b(elemen)o(ts\))14 b(w)o(ere)h(actually)e(receiv)o (ed)i(in)e(the)i(receiv)o(e)g(bu\013er.)166 2013 y(A)j(stronger)i (requiremen)o(t)e(is)h(that)f(the)h(receiv)o(e)h(op)q(eration)e(\014ts) h(as)g(m)o(uc)o(h)e(data)h(as)h(can)g(b)q(e)g(\014t)f(in)g(the)75 2063 y(receiv)o(e)d(bu\013er.)k(\(One)c(can)f(statisfy)f(the)i (previous)f(requiremen)o(ts)g(b)o(y)f(receiving)h(no)g(data)f(and)h (returning)g(zero)75 2113 y(coun)o(ts,)g(in)f(case)i(of)f(o)o(v)o (er\015o)o(w.\))166 2169 y(Do)20 b(w)o(e)h(w)o(an)o(t)f(to)h(sp)q (ecify)g(suc)o(h)g(b)q(eha)o(vior?)38 b(\(So)21 b(far,)g(w)o(e)g(ha)o (v)o(e)g(a)o(v)o(oided)e(de\014ning)i(the)g(b)q(eha)o(vior)g(of)75 2226 y(erroneous)15 b(programs.\))166 2365 y FD(The)k(selection)h(of)f (a)f(message)h(b)o(y)f(a)h(receiv)o(e)h(op)q(eration)f(is)g(done)g (uniquely)i(according)f(to)e(the)75 2421 y(v)m(alue)i(of)f(the)g (message)g(en)o(v)o(elop)q(e.)32 b(The)19 b(receiv)o(e)h(op)q(eration)g (sp)q(eci\014es)h(an)d Fx(en)o(v)o(elop)q(e)k(pattern)p FD(;)f(a)75 2478 y(message)13 b(can)g(b)q(e)g(receiv)o(ed)h(b)o(y)f (that)g(receiv)o(e)h(op)q(eration)f(only)g(if)h(its)f(en)o(v)o(elop)q (e)h(matc)o(hes)f(that)f(pattern.)75 2534 y(A)19 b(pattern)g(sp)q (eci\014es)i(v)m(alues)g(for)e(the)g Fs(source)p FD(,)i Fs(tag)e FD(and)h Fs(context)g FD(\014elds)h(of)e(the)g(message)g(en)o (v)o(elop)q(e.)75 2591 y(The)d(receiv)o(er)h(ma)o(y)e(sp)q(ecify)j(a)e (wildcard)h Fv(MPI)p 889 2591 V 14 w(ANY)p 988 2591 V 15 w(SOURCE)d FD(v)m(alue)k(for)d Fs(source)p FD(,)h(and/or)g(a)g (wildcard)75 2647 y Fv(MPI)p 152 2647 V 14 w(ANY)p 251 2647 V 15 w(T)m(A)o(G)c FD(v)m(alue)j(for)d Fs(tag)p FD(,)i(indicating)h(that)d(an)o(y)h(source)h(and/or)e(tag)h(are)g (acceptable.)20 b(It)13 b(cannot)75 2704 y(sp)q(ecify)h(a)f(wildcard)h (v)m(alue)g(for)f Fs(context)p FD(.)20 b(Th)o(us,)13 b(a)g(message)f(can)i(b)q(e)f(receiv)o(ed)h(b)o(y)f(a)g(receiv)o(e)h (op)q(eration)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 27 20 26 bop 75 -100 a FD(20)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FD(only)23 b(if)g(it)g(is)g(addressed)h(to)e(the)g(receiving)j(pro)q(cess,)f(has)f (a)f(matc)o(hing)h(con)o(text,)g(has)g(matc)o(hing)75 102 y(source)14 b(unless)g(source=)p Fv(MPI)p 581 102 13 2 v 15 w(ANY)p 681 102 V 15 w(SOURCE)d FD(in)k(the)e(pattern,)g(and) h(has)g(a)f(matc)o(hing)g(tag)g(unless)i(tag=)75 158 y Fv(MPI)p 152 158 V 14 w(ANY)p 251 158 V 15 w(T)m(A)o(G)f FD(in)i(the)f(pattern.)166 231 y(The)d(message)g(tag)f(is)i(sp)q (eci\014ed)h(b)o(y)e(the)g Fs(tag)h FD(argumen)o(t)e(of)h(the)g(receiv) o(e)h(op)q(eration.)19 b(The)12 b(message)75 288 y Fs(context)20 b FD(is)f(the)f(con)o(text)g(attac)o(hed)g(with)h(the)f(comm)o (unicator)g(sp)q(eci\014ed)j(b)o(y)d(the)h(argumen)o(t)e Fs(comm)m FD(.)75 344 y(The)g(message)g(source,)h(if)f(di\013eren)o(t)h (from)32 b Fv(MPI)p 929 344 V 14 w(ANY)p 1028 344 V 15 w(SOURCE)p FD(,)15 b(is)j(sp)q(eci\014ed)h(as)e(a)g(rank)f(within)j (the)75 401 y(pro)q(cess)g(group)f(asso)q(ciated)h(with)g(that)e(same)h (comm)o(unicator)h(\(remote)e(pro)q(cess)i(group,)g(for)f(in)o(ter-)75 457 y(comm)o(unicators\).)27 b(Th)o(us,)18 b(the)g(range)g(of)f(v)m (alid)j(v)m(alues)f(for)e(the)h Fs(source)g FD(argumen)o(t)g(is)g FA(f)g Fs(0,)f(...)28 b(,)18 b(n-1)75 514 y FA(g[f)d Fs(MPI)p 251 514 14 2 v 16 w(ANY)p 359 514 V 17 w(SOURCE)i FA(g)p FD(,)d(where)h Fs(n)h FD(is)g(the)f(n)o(um)o(b)q(er)h(of)e(pro)q (cesses)i(in)g(this)g(group.)166 587 y(Note)i(the)i(asymmetry)d(b)q(et) o(w)o(een)j(send)f(and)g(receiv)o(e)h(op)q(erations:)28 b(A)19 b(receiv)o(e)h(op)q(eration)f(ma)o(y)75 644 y(accept)13 b(messages)f(from)g(an)h(arbitrary)f(sender;)i(on)f(the)g(other)f (hand,)i(a)e(send)h(op)q(eration)g(m)o(ust)g(sp)q(ecify)75 700 y(a)i(unique)j(receiv)o(er.)k(This)16 b(matc)o(hes)f(a)h(\\push")f (comm)o(unication)i(mec)o(hanism,)f(where)g(data)f(transfer)75 757 y(is)i(e\013ected)h(b)o(y)f(the)g(sender)g(\(rather)f(than)h(a)g (\\pull")h(mec)o(hanism,)g(where)f(data)f(transfer)g(is)i(e\013ected)75 813 y(b)o(y)d(the)g(receiv)o(er\).)166 886 y(Source)f(=)h(destination)f (is)h(allo)o(w)o(ed:)k(a)14 b(pro)q(cess)g(can)g(send)h(a)f(message)f (to)g(itself.)21 b(\(Ho)o(w)o(ev)o(er,)12 b(it)i(is)75 943 y(unsafe)h(to)g(do)g(so)g(with)h(the)f(basic)h(send)g(and)f(receiv) o(e)h(op)q(erations)g(describ)q(ed)h(ab)q(o)o(v)o(e,)e(since)h(this)g (ma)o(y)75 999 y(lead)g(to)f(deadlo)q(c)o(k;)g(see)h(Section)g(3.4\))75 1218 y Fn(3.3.1)49 b(Return)16 b(status)75 1337 y FD(The)e(source)g(or) f(the)h(tag)f(of)g(a)h(receiv)o(ed)h(message)e(ma)o(y)g(not)h(b)q(e)g (kno)o(wn)g(if)g(wildcard)h(v)m(alues)g(w)o(ere)e(used)75 1393 y(in)h(the)f(receiv)o(e)h(op)q(eration.)19 b(Also,)13 b(the)g(actual)g(length)h(of)f(the)g(message)f(receiv)o(ed)i(ma)o(y)e (not)h(b)q(e)h(kno)o(wn.)75 1450 y(Th)o(us,)j(this)h(information)f (needs)h(to)e(b)q(e)i(returned)g(b)o(y)f(the)g(receiv)o(e)h(op)q (eration.)26 b(The)17 b(information)g(is)75 1506 y(returned)12 b(b)o(y)g(the)f Fs(status)j FD(argumen)o(t)d(of)g(the)h Fs(MPI)p 923 1506 V 16 w(RECV)g FD(function.)20 b(This)12 b(is)g(an)g(argumen)o(t)f(of)g(a)g(sp)q(ecial)75 1563 y(MPI-de\014ned)19 b(t)o(yp)q(e.)27 b(Status)17 b(v)m(ariables)i(need)f (to)f(b)q(e)h(explicitly)i(allo)q(cated)f(b)o(y)e(the)h(user)g({)f (they)g(are)75 1619 y(not)e(system)g(ob)s(jects.)166 1693 y(The)f(status)f(v)m(ariable)i(can)f(b)q(e)g(\\deco)q(ded")h(to)e (retriev)o(e)h(the)g Fs(count)p FD(,)g Fs(source)h FD(and)f Fs(tag)f FD(\014elds,)i(using)75 1749 y(the)g(query)g(functions)h (listed)g(b)q(elo)o(w.)k(The)15 b(information)g(returned)h(b)o(y)f (these)g(query)g(functions)g(is)h(the)75 1805 y(information)f(stored)g (in)i(the)e Fs(status)i FD(v)m(ariable)g(b)o(y)e(the)h(last)f(receiv)o (e)h(\(or)f(prob)q(e)g(-)h(-)f({)g(see)h(Section)g(3.9\))75 1862 y(call)g(that)f(w)o(as)f(passed)h Fs(status)i FD(as)e(argumen)o (t.)k(It)c(is)h(erroneous)f(to)f(call)j(these)e(query)g(functions)h(if) g(the)75 1918 y Fs(status)h FD(v)m(ariable)g(w)o(as)d(nev)o(er)h(set)g (b)o(y)h(a)e(receiv)o(e)j(\(or)d(prob)q(e\).)75 2039 y Fs(MPI)p 160 2039 V 16 w(GET)p 264 2039 V 17 w(SOURCE\(status,)k (source\))117 2133 y Ft(IN)155 b Fs(status)476 b Ft(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 2242 y(OUT)108 b Fs(source)468 b Ft(source)15 b(rank)f(\(in)o(teger\)) 75 2384 y FB(int)23 b(MPI)p 245 2384 15 2 v 17 w(Get)p 334 2384 V 17 w(source\(MPI)p 591 2384 V 16 w(Status)g(status,)g(int)g (*source\))75 2487 y(MPI)p 150 2487 V 17 w(GET)p 239 2487 V 17 w(SOURCE\(STATUS,)e(SOURCE,)i(IERROR\))170 2544 y(INTEGER)g(STATUS\(MPI)p 604 2544 V 16 w(STATUS)p 764 2544 V 17 w(SIZE\),)g(SOURCE,)f(IERROR)166 2647 y FD(Returns)13 b(the)g(rank)g(of)f(the)h(message)f(source)h(in)h(the)f (group)f(asso)q(ciated)h(with)g(the)g(comm)o(unicator)75 2704 y(that)h(w)o(as)h(used)h(to)e(receiv)o(e)i(\(remote)f(group,)f (for)h(in)o(tercomm)o(unicators\).)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 28 21 27 bop 75 -100 a Fp(3.3.)34 b(BASIC)16 b(RECEIVE)g(OPERA)l(TION)990 b FD(21)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(GET)p 264 45 V 17 w(T)l(A)o(G\(status,)17 b(tag\))117 126 y Ft(IN)155 b Fs(status)476 b Ft(return)15 b(status)g(of)e(receiv)o(e)i(or)f(prob)q (e)h(op)q(eration)f(\(Status\))117 209 y(OUT)108 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))75 337 y FB(int)23 b(MPI)p 245 337 15 2 v 17 w(Get)p 334 337 V 17 w(tag\(MPI)p 519 337 V 16 w(Status)g(status,)g(int)h(*tag\))75 427 y(MPI)p 150 427 V 17 w(GET)p 239 427 V 17 w(TAG\(STATUS,)e(TAG,)h (IERROR\))170 484 y(INTEGER)g(STATUS\(MPI)p 604 484 V 16 w(STATUS)p 764 484 V 17 w(SIZE\),)g(TAG,)g(IERROR)166 574 y FD(Returns)16 b(the)f(tag)f(of)h(receiv)o(ed)h(message.)75 682 y Fs(MPI)p 160 682 14 2 v 16 w(GET)p 264 682 V 17 w(COUNT\(status,)h(datat)o(yp)q(e,)f(count\))117 763 y Ft(IN)155 b Fs(status)476 b Ft(return)15 b(status)g(of)e(receiv)o(e)i (op)q(eration)f(\(Status\))117 846 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(handle\))117 928 y(OUT)108 b Fs(count)482 b Ft(n)o(um)o(b)q(er) 13 b(of)h(receiv)o(ed)h(elemen)o(ts)f(\(in)o(teger\))75 1057 y FB(int)23 b(MPI)p 245 1057 15 2 v 17 w(Get)p 334 1057 V 17 w(count\(MPI)p 567 1057 V 16 w(Status)g(status,)g(MPI)p 1013 1057 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 1147 y(MPI)p 150 1147 V 17 w(GET)p 239 1147 V 17 w(COUNT\(STATUS,)e (DATATYPE,)g(COUNT,)h(IERROR\))170 1203 y(INTEGER)g(STATUS\(MPI)p 604 1203 V 16 w(STATUS)p 764 1203 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 1294 y FD(Returns)15 b(the)f(n)o(um)o(b)q(er)h(of)f(elemen) o(ts)h(receiv)o(ed.)21 b(\(Here,)14 b(again,)h(w)o(e)f(coun)o(t)g Fo(elements)p FD(,)f(not)h Fo(bytes)p FD(.\))75 1350 y(The)f Fs(datat)o(yp)q(e)h FD(argumen)o(t)e(should)i(matc)o(h)e(the)h (argumen)o(t)f(pro)o(vided)i(b)o(y)f(the)g(receiv)o(e)g(call)h(that)e (set)h(the)75 1407 y Fs(status)19 b FD(v)m(ariable.)26 b(\(W)l(e)17 b(shall)h(later)f(see,)g(in)h(Section)g(3.13.5,)d(that)h Fs(MPI)p 1347 1407 14 2 v 16 w(GET)p 1451 1407 V 16 w(COUNT)i FD(ma)o(y)e(return,)75 1463 y(in)g(certain)g(situations,)f(the)g(v)m (alue)i Fv(MPI)p 773 1463 13 2 v 14 w(UNDEFINED)p FD(.\))166 1523 y(Note)e(that)g(it)h(is)g(not)f(mandatory)g(to)g(query)g(the)h (return)g(status)e(after)h(a)g(receiv)o(e.)22 b(The)16 b(receiv)o(er)75 1580 y(will)h(use)e(these)h(calls,)g(or)e(some)h(of)g (them,)g(only)h(when)f(the)h(information)f(they)g(return)g(is)h (needed.)189 1705 y Fo(R)n(ationale.)43 b FD(The)16 b(use)g(of)f(a)h (separate)f(status)g(argumen)o(t)g(prev)o(en)o(ts)h(errors)e(that)i (are)f(often)g(at-)189 1762 y(tac)o(hed)f(with)h Ft(INOUT)g FD(argumen)o(t)f(\(e.g.,)f(passing)i(the)28 b Fv(MPI)p 1233 1762 V 14 w(ANY)p 1332 1762 V 15 w(T)m(A)o(G)13 b FD(constan)o(t)h(as)g(the)h(actual)189 1818 y(argumen)o(t)k(for)g Fs(tag)p FD(\).)32 b(The)20 b(use)g(of)g(an)f(explicit)j(user)e(ob)s (ject)f(b)q(ecomes)h(imp)q(ortan)o(t)g(for)f(non-)189 1875 y(blo)q(c)o(king)d(comm)o(unication)g(as)f(it)g(allo)o(ws)g(the)h (receiv)o(e)g(op)q(eration)f(to)g(b)q(e)g(stateless)g(and,)g(hence,)189 1931 y(reen)o(tran)o(t.)j(This)d(prev)o(en)o(ts)f(confusions)h(in)g (the)f(case)g(where)h(m)o(ultiple)h(receiv)o(es)f(can)f(b)q(e)h(p)q (osted)189 1987 y(b)o(y)g(a)g(pro)q(cess.)189 2070 y(The)j Fs(datat)o(yp)q(e)i FD(argumen)o(t)d(is)i(passed)g(to)e(the)i Fs(MPI)p 1096 2070 14 2 v 16 w(GET)p 1200 2070 V 16 w(COUNT)g FD(so)f(as)g(to)g(impro)o(v)o(e)g(p)q(erfor-)189 2127 y(mance)k(\(a)f(message)h(migh)o(t)g(b)q(e)g(receiv)o(ed)i(without)e (coun)o(ting)g(the)g(n)o(um)o(b)q(er)g(of)g(elemen)o(ts)h(it)189 2183 y(con)o(tains\),)14 b(and)i(to)e(allo)o(w)i(its)f(use)h(after)e(a) h(call)h(to)f Fs(MPI)p 1170 2183 V 16 w(PROBE)p FD(.)h(\()p Fo(End)f(of)i(r)n(ationale.)p FD(\))166 2391 y Fl(Implemen)o(tati)o(on) d(note:)44 b Ft(One)16 b(exp)q(ects)i(deco)q(de)f(functions)f(to)g(b)q (e)g(in-lined)f(in)g(man)o(y)f(implemen)o(ta-)75 2448 y(tions.)166 2591 y FD(All)19 b(send)g(and)g(receiv)o(e)g(op)q (erations)f(use)h(the)f Fs(buf)p FD(,)h Fs(count)p FD(,)h Fs(datat)o(yp)q(e)p FD(,)g Fs(source)p FD(,)f Fs(dest)p FD(,)h Fs(tag)p FD(,)f Fs(comm)75 2647 y FD(and)e Fs(status)i FD(argumen)o(ts)d(in)i(the)f(same)g(w)o(a)o(y)f(as)g(the)h(basic)h Fs(MPI)p 1201 2647 V 16 w(SEND)f FD(and)g Fs(MPI)p 1523 2647 V 16 w(RECV)h FD(op)q(erations)75 2704 y(describ)q(ed)f(in)f(this) g(section.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 29 22 28 bop 75 -100 a FD(22)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(Alternative)g (Prop)q(osal)75 139 y FD(The)e(source)g(or)f(the)h(tag)f(of)g(a)h (receiv)o(ed)h(message)e(ma)o(y)g(not)h(b)q(e)g(kno)o(wn)g(if)g (wildcard)h(v)m(alues)g(w)o(ere)e(used)-2119 b Fm(CHANGE)75 196 y FD(in)14 b(the)f(receiv)o(e)h(op)q(eration.)19 b(Also,)13 b(the)g(actual)g(length)h(of)f(the)g(message)f(receiv)o(ed)i (ma)o(y)e(not)h(b)q(e)h(kno)o(wn.)75 252 y(Th)o(us,)j(this)h (information)f(needs)h(to)e(b)q(e)i(returned)g(b)o(y)f(the)g(receiv)o (e)h(op)q(eration.)26 b(The)17 b(information)g(is)75 309 y(returned)12 b(b)o(y)g(the)f Fs(status)j FD(argumen)o(t)d(of)g (the)h Fs(MPI)p 923 309 14 2 v 16 w(RECV)g FD(function.)20 b(This)12 b(is)g(an)g(argumen)o(t)f(of)g(a)g(sp)q(ecial)75 365 y(MPI-de\014ned)19 b(t)o(yp)q(e.)27 b(Status)17 b(v)m(ariables)i (need)f(to)f(b)q(e)h(explicitly)i(allo)q(cated)f(b)o(y)e(the)h(user)g ({)f(they)g(are)75 422 y(not)e(system)g(ob)s(jects.)166 483 y(The)f(status)f(v)m(ariable)i(can)f(b)q(e)g(\\deco)q(ded")h(to)e (retriev)o(e)h(the)g Fs(count)p FD(,)g Fs(source)h FD(and)f Fs(tag)f FD(\014elds,)i(using)75 539 y(the)g(query)g(functions)h (listed)g(b)q(elo)o(w.)k(The)15 b(information)g(returned)h(b)o(y)f (these)g(query)g(functions)g(is)h(the)75 596 y(information)f(stored)g (in)i(the)e Fs(status)i FD(v)m(ariable)g(b)o(y)e(the)h(last)f(receiv)o (e)h(\(or)f(prob)q(e)g(-)h(-)f({)g(see)h(Section)g(3.9\))75 652 y(call)g(that)f(w)o(as)f(passed)h Fs(status)i FD(as)e(argumen)o(t.) k(It)c(is)h(erroneous)f(to)f(call)j(these)e(query)g(functions)h(if)g (the)75 708 y Fs(status)h FD(v)m(ariable)g(w)o(as)d(nev)o(er)h(set)g(b) o(y)h(a)e(receiv)o(e)j(\(or)d(prob)q(e\).)166 769 y(The)i Fs(status)h FD(argumen)o(t)e(in)h(the)f Fs(MPI)p 808 769 V 16 w(RECV)h FD(function)h(is)f(a)f(structure)g(\(of)f (prede\014ned)j(datat)o(yp)q(e)75 826 y Fv(MPI)p 152 826 13 2 v 14 w(Status)p FD(\))d(in)h(C,)f(an)g(arra)o(y)f(of)g(in)o (tegers)i(in)g(F)l(ortran.)j(In)d(C,)e(the)h Fs(source)h FD(and)f Fs(tag)g FD(\014eld)i(are)d(accessed)75 882 y(as)h(comp)q(onen)o(ts)h(of)f(this)h(structure:)k Fs(status.source)e FD(and)e Fs(status.tag)p FD(,)h(resp)q(ectiv)o(ely)l(.)21 b(In)16 b(F)l(ortran,)d(they)75 939 y(are)j(accessed)h(as)g(en)o(tries) g(in)g(an)f(arra)o(y:)22 b Fs(ST)l(A)l(TUS\(MPI)p 1064 939 14 2 v 17 w(SOURCE\))17 b FD(AND)g Fs(ST)l(A)l(TUS\(MPI)p 1675 939 V 17 w(T)l(A)o(G\))p FD(,)f(re-)75 995 y(sp)q(ectiv)o(ely;)f Fv(MPI)p 373 995 13 2 v 15 w(SOURCE)c FD(and)j Fv(MPI)p 723 995 V 14 w(ST)m(A)m(TUS)e FD(are)h(prede\014ned)i(constan)o(ts)e (set)g(to)g(the)g(correct)g(indices)75 1052 y(withing)19 b Fs(ST)l(A)l(TUS)p FD(.)g(The)g Fs(status)h FD(v)m(ariable)g(ma)o(y)d (con)o(tain)i(additional)h(\014elds)f(with)g(additional)h(infor-)75 1108 y(mation)15 b(that)g(is)g(not)g(directly)i(accessible)g(to)d(the)h (user.)166 1169 y(Information)i(on)g(the)f(length)i(of)e(the)h (incoming)i(message)d(is)h(not)g(directly)h(a)o(v)m(ailable)h(as)d(a)h (\014led)75 1225 y(of)d Fs(status)p FD(.)22 b(Rather,)15 b(an)g(inquiry)h(function)g(need)f(to)g(b)q(e)g(applied)i(to)d Fs(status)p FD(,)j(in)f(order)e(to)g(retriev)o(e)i(this)75 1282 y(information.)75 1390 y Fs(MPI)p 160 1390 14 2 v 16 w(GET)p 264 1390 V 17 w(COUNT\(status,)h(datat)o(yp)q(e,)f (count\))117 1471 y Ft(IN)155 b Fs(status)476 b Ft(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 1555 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1639 y(OUT)108 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(elemen)o(ts)f(\(in)o(teger\))75 1768 y FB(int)23 b(MPI)p 245 1768 15 2 v 17 w(Get)p 334 1768 V 17 w(count\(MPI)p 567 1768 V 16 w(Status)g(status,)g(MPI)p 1013 1768 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 1859 y(MPI)p 150 1859 V 17 w(GET)p 239 1859 V 17 w(COUNT\(STATUS,)e (DATATYPE,)g(COUNT,)h(IERROR\))170 1916 y(INTEGER)g(STATUS\(MPI)p 604 1916 V 16 w(STATUS)p 764 1916 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 2006 y FD(Returns)15 b(the)f(n)o(um)o(b)q(er)h(of)f(elemen) o(ts)h(receiv)o(ed.)21 b(\(Here,)14 b(again,)h(w)o(e)f(coun)o(t)g Fo(elements)p FD(,)f(not)h Fo(bytes)p FD(.\))75 2063 y(The)f Fs(datat)o(yp)q(e)h FD(argumen)o(t)e(should)i(matc)o(h)e(the)h (argumen)o(t)f(pro)o(vided)i(b)o(y)f(the)g(receiv)o(e)g(call)h(that)e (set)h(the)75 2119 y Fs(status)19 b FD(v)m(ariable.)26 b(\(W)l(e)17 b(shall)h(later)f(see,)g(in)h(Section)g(3.13.5,)d(that)h Fs(MPI)p 1347 2119 14 2 v 16 w(GET)p 1451 2119 V 16 w(COUNT)i FD(ma)o(y)e(return,)75 2176 y(in)g(certain)g(situations,)f(the)g(v)m (alue)i Fv(MPI)p 773 2176 13 2 v 14 w(UNDEFINED)p FD(.\))166 2237 y(Note)e(that)g(it)h(is)g(not)f(mandatory)g(to)g(query)g(the)h (return)g(status)e(after)h(a)g(receiv)o(e.)22 b(The)16 b(receiv)o(er)75 2293 y(will)h(use)e(these)h(calls,)g(or)e(some)h(of)g (them,)g(only)h(when)f(the)h(information)f(they)g(return)g(is)h (needed.)189 2421 y Fo(R)n(ationale.)43 b FD(The)16 b(use)g(of)f(a)h (separate)f(status)g(argumen)o(t)g(prev)o(en)o(ts)h(errors)e(that)i (are)f(often)g(at-)189 2478 y(tac)o(hed)f(with)h Ft(INOUT)g FD(argumen)o(t)f(\(e.g.,)f(passing)i(the)28 b Fv(MPI)p 1233 2478 V 14 w(ANY)p 1332 2478 V 15 w(T)m(A)o(G)13 b FD(constan)o(t)h(as)g(the)h(actual)189 2534 y(argumen)o(t)k(for)g Fs(tag)p FD(\).)32 b(The)20 b(use)g(of)g(an)f(explicit)j(user)e(ob)s (ject)f(b)q(ecomes)h(imp)q(ortan)o(t)g(for)f(non-)189 2591 y(blo)q(c)o(king)d(comm)o(unication)g(as)f(it)g(allo)o(ws)g(the)h (receiv)o(e)g(op)q(eration)f(to)g(b)q(e)g(stateless)g(and,)g(hence,)189 2647 y(reen)o(tran)o(t.)j(This)d(prev)o(en)o(ts)f(confusions)h(in)g (the)f(case)g(where)h(m)o(ultiple)h(receiv)o(es)f(can)f(b)q(e)h(p)q (osted)189 2704 y(b)o(y)g(a)g(pro)q(cess.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 30 23 29 bop 75 -100 a Fp(3.4.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b FD(23)189 45 y(The)14 b Fs(count)i FD(v)m(alue)g(is)f(retriev)o(ed)g(b)o(y)g(a)f(function)h(call)h(with)f (a)f Fs(datat)o(yp)q(e)j FD(argumen)o(t,)c(rather)h(than)189 102 y(b)o(y)e(a)h(direct)h(access)f(to)f(a)h Fs(status)i FD(\014eld,)f(so)e(as)h(to)f(a)o(v)o(oid)h(the)g(need)h(for)e (computing)h Fs(count)i FD(at)d(eac)o(h)189 158 y(receiv)o(e,)j(and)h (to)e(allo)o(w)i(its)f(use)h(after)e(a)h(call)h(to)f Fs(MPI)p 1123 158 14 2 v 16 w(PROBE)p FD(.)h(\()p Fo(End)f(of)i(r)n (ationale.)p FD(\))166 257 y(All)i(send)g(and)g(receiv)o(e)g(op)q (erations)f(use)h(the)f Fs(buf)p FD(,)h Fs(count)p FD(,)h Fs(datat)o(yp)q(e)p FD(,)g Fs(source)p FD(,)f Fs(dest)p FD(,)h Fs(tag)p FD(,)f Fs(comm)75 313 y FD(and)e Fs(status)i FD(argumen)o(ts)d(in)i(the)f(same)g(w)o(a)o(y)f(as)g(the)h(basic)h Fs(MPI)p 1201 313 V 16 w(SEND)f FD(and)g Fs(MPI)p 1523 313 V 16 w(RECV)h FD(op)q(erations)75 370 y(describ)q(ed)f(in)f(this)g (section.)166 426 y Fo(A)n(dditional)g(pr)n(op)n(osal)166 482 y FD(The)k Fs(status)i FD(v)m(ariable)g(also)e(con)o(tain)h(a)e Fs(size)i FD(\014eld)g(\()p Fs(status.size)h FD(in)f(C,)f Fs(ST)l(A)l(TUS\(MPI)p 1693 482 V 16 w(SIZE\))g FD(in)75 539 y(F)l(ortran\))12 b(that)i(enco)q(des)g(the)g(n)o(um)o(b)q(er)g(of) g(b)o(ytes)g(of)f(data)g(in)i(the)f(incoming)h(message.)k(Note)13 b(that)h(this)75 595 y(n)o(um)o(b)q(er)g(of)g(b)o(ytes)f(ma)o(y)g(b)q (e)i(di\013eren)o(t)f(than)g(the)g(n)o(um)o(b)q(er)g(of)f(b)o(ytes)h (actually)h(stored)e(in)i(the)f(receiv)o(er's)75 652 y(memory)l(.)166 784 y Fl(Discussion:)166 841 y Ft(The)h(alternativ)o (e)g(design)g(ma)o(y)e(a)o(v)o(oid)g(the)j(need)g(for)e(a)h(function)f (call)g(to)h(get)g Fv(tag)g Ft(and)f Fv(source)i Ft(\(standard)75 897 y(F)m(ortran)j(do)q(es)i(not)e(ha)o(v)o(e)g(macro)g(expansion\).)35 b(The)20 b(coun)o(terargumen)o(t)f(is)h(that)f(man)o(y)f(\(most?\))34 b(F)m(ortran)75 954 y(compilers)12 b(do)h(supp)q(ort)g(inlining,)e (although,)h(p)q(erhaps)i(di\013eren)o(tly)f(on)g(eac)o(h)h(system,)e (and)h(that)g(it)g(is)f(inelegan)o(t)75 1010 y(to)i(ha)o(v)o(e)f(a)h (di\013eren)o(t)h(access)h(metho)q(d)d(for)g(di\013eren)o(t)i(status)g (\014elds.)75 1235 y Fq(3.4)59 b(Semantics)18 b(of)i(p)r(oint-to-p)r (oint)e(communication)75 1336 y FD(A)g(v)m(alid)h(MPI)e(implemen)o (tation)i(guaran)o(tees)e(certain)h(general)g(prop)q(erties)g(of)f(p)q (oin)o(t-to-p)q(oin)o(t)h(com-)75 1393 y(m)o(unication,)e(whic)o(h)g (are)f(describ)q(ed)i(in)f(this)f(section.)75 1512 y Fs(Order)46 b FD(Messages)14 b(are)h Fo(non-overtaking)p FD(,)g(within)h(eac)o(h)f(con)o(text:)20 b(if)c(t)o(w)o(o)d(messages)i (are)g(successiv)o(ely)75 1568 y(sen)o(t)10 b(from)g(the)g(same)g (source,)h(to)e(the)i(same)f(destination,)h(with)g(the)f(same)g(con)o (text,)h(then)f(the)g(messages)75 1625 y(are)17 b(receiv)o(ed)i(in)f (the)f(order)g(they)h(w)o(ere)f(sen)o(t.)26 b(I.e.,)17 b(if)h(the)f(receiv)o(er)i(p)q(osts)e(t)o(w)o(o)f(successiv)o(e)i (receiv)o(es)75 1681 y(that)g(b)q(oth)g(matc)o(h)g(either)i(message,)e (then)h(the)f(\014rst)h(receiv)o(e)g(will)h(receiv)o(e)g(the)e(\014rst) g(message,)h(and)75 1738 y(the)e(second)h(receiv)o(e)h(will)g(receiv)o (e)f(the)f(second)h(message.)26 b(This)18 b(requiremen)o(t)g (facilitates)g(matc)o(hing)75 1794 y(of)e(sends)h(to)f(receiv)o(es.)25 b(It)16 b(guaran)o(tees)g(that)g(message-passing)g(co)q(de)h(is)g (deterministic,)i(if)e(pro)q(cesses)75 1851 y(are)h(single-threaded)j (and)e(wildcard)h Fv(MPI)p 831 1851 13 2 v 14 w(ANY)p 930 1851 V 15 w(SOURCE)d FD(is)i(not)f(used)h(in)h(receiv)o(es.)31 b(\(Some)19 b(of)f(the)75 1907 y(calls)k(describ)q(ed)h(later,)g(suc)o (h)e(as)g Fs(MPI)p 771 1907 14 2 v 16 w(CANCEL)g FD(or)g Fs(MPI)p 1124 1907 V 16 w(W)l(AIT)l(ANY)p FD(,)h(are)f(additional)h (sources)f(of)75 1964 y(nondeterminism.\),)166 2020 y(If)c(a)g(pro)q (cess)h(has)f(a)g(single)h(thread)f(of)g(execution,)i(then)e(an)o(y)g (t)o(w)o(o)f(comm)o(unications)i(executed)75 2076 y(b)o(y)e(this)h(pro) q(cess)g(are)f(ordered.)23 b(On)17 b(the)f(other)g(hand,)h(if)g(the)f (pro)q(cess)h(is)g(m)o(ulti-threaded,)g(then)g(the)75 2133 y(seman)o(tics)e(of)f(thread)h(execution)h(ma)o(y)e(not)g (de\014ne)i(a)e(relativ)o(e)i(order)e(b)q(et)o(w)o(een)h(t)o(w)o(o)e (send)j(op)q(erations)75 2189 y(executed)g(b)o(y)f(t)o(w)o(o)f (distinct)j(threads:)i(the)d(op)q(erations)f(are)g(logically)i (concurren)o(t,)e(ev)o(en)h(though)f(one)75 2246 y(ph)o(ysically)22 b(precedes)e(the)g(other.)33 b(In)20 b(suc)o(h)g(a)f(case,)i(the)e(t)o (w)o(o)g(messages)g(sen)o(t)g(can)h(b)q(e)g(receiv)o(ed)h(in)75 2302 y(an)o(y)f(order.)37 b(Similarly)l(,)24 b(if)d(t)o(w)o(o)e(receiv) o(e)j(op)q(erations)f(that)f(are)g(logically)j(concurren)o(t)e(receiv)o (e)g(t)o(w)o(o)75 2359 y(successiv)o(ely)i(sen)o(t)d(messages,)h(then)g (the)g(t)o(w)o(o)f(messages)g(can)h(matc)o(h)f(the)h(t)o(w)o(o)e (receiv)o(es)j(in)f(either)75 2415 y(order.)31 b(Note)19 b(that)f(the)h(receiv)o(er)h(is)g(not)e(required)i(to)f(receiv)o(e)h (messages)e(in)i(the)f(order)g(they)g(w)o(ere)75 2472 y(sen)o(t,)c(if)g(they)h(ha)o(v)o(e)e(distinct)j(en)o(v)o(elop)q(es.)75 2591 y Fs(Progress)45 b FD(If)13 b(a)g(pair)g(of)f(matc)o(hing)h(send)h (and)f(receiv)o(es)g(ha)o(v)o(e)g(b)q(een)h(initiated)g(on)f(t)o(w)o(o) e(pro)q(cesses,)j(then)75 2647 y(at)k(least)h(one)h(of)e(these)h(t)o(w) o(o)f(op)q(erations)h(will)i(complete,)f(indep)q(enden)o(tly)i(of)c (other)h(actions)g(in)h(the)75 2704 y(system:)e(the)13 b(send)h(op)q(eration)f(will)h(complete,)g(unless)g(the)f(receiv)o(e)h (is)f(satis\014ed)h(b)o(y)e(another)h(message,)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 31 24 30 bop 75 -100 a FD(24)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FD(and)e(completes;) g(the)g(receiv)o(e)h(op)q(eration)f(will)h(complete,)g(unless)g(the)f (message)f(sen)o(t)g(is)i(consumed)f(b)o(y)75 102 y(another)h(matc)o (hing)g(receiv)o(e)h(that)f(w)o(as)f(p)q(osted)h(at)g(the)g(same)g (destination)h(pro)q(cess.)75 224 y Fs(F)o(airness)46 b FD(MPI)15 b(mak)o(es)g(no)g(guaran)o(tee)g(of)g Fo(fairness)f FD(in)i(the)g(handling)h(of)e(comm)o(unication.)21 b(Supp)q(ose)75 281 y(that)c(a)f(send)i(w)o(as)f(p)q(osted.)26 b(Then)18 b(it)f(is)h(p)q(ossible)h(that)e(the)g(destination)h(pro)q(cess)g(rep)q (eatedly)g(p)q(osts)75 337 y(a)d(receiv)o(e)h(that)f(matc)o(hes)f(this) i(send,)g(y)o(et)e(the)i(message)f(is)g(nev)o(er)h(receiv)o(ed,)g(b)q (ecause)g(it)g(is)f(eac)o(h)h(time)75 394 y(o)o(v)o(ertak)o(en)e(b)o(y) i(another)f(message,)g(sen)o(t)h(from)f(another)g(source.)21 b(Similarly)l(,)d(supp)q(ose)f(that)e(a)g(receiv)o(e)75 450 y(w)o(as)f(p)q(osted)i(b)o(y)f(a)g(m)o(ulti-threaded)h(pro)q(cess.) k(Then)c(it)f(is)h(p)q(ossible)h(that)d(messages)h(that)f(matc)o(h)h (this)75 507 y(receiv)o(e)20 b(are)g(rep)q(eatedly)g(receiv)o(ed,)i(y)o (et)d(the)g(receiv)o(e)i(is)f(nev)o(er)f(satis\014ed,)i(b)q(ecause)f (it)g(is)g(o)o(v)o(ertak)o(en)75 563 y(b)o(y)15 b(other)h(receiv)o(es)g (p)q(osted)g(at)e(this)i(no)q(de)h(\(b)o(y)e(other)g(executing)h (threads\).)21 b(It)15 b(is)h(the)g(programmer's)75 620 y(resp)q(onsibilit)o(y)i(to)c(prev)o(en)o(t)h(starv)m(ation)g(in)h(suc) o(h)g(situations.)75 742 y Fs(Resource)h(lim)o(itations)44 b FD(The)16 b(receiv)o(e)g(op)q(eration)g Fs(MPI)p 1035 742 14 2 v 15 w(RECV)h FD(clearly)f(is)g(non-lo)q(cal:)22 b(it)16 b(cannot)f(com-)-2120 b Fm(CHANGE)75 799 y FD(plete)16 b(unless)h(a)e(matc)o(hing)g(send)h(is)g(p)q(osted)f(b)o(y)g(another)g (pro)q(cess.)21 b(The)15 b(send)h(op)q(eration)g Fs(MPI)p 1745 799 V 16 w(SEND)75 855 y FD(also)c(is)h(non-lo)q(cal:)20 b(The)13 b(send)f(op)q(eration)h(ma)o(y)f(not)f(b)q(e)i(able)g(to)f (complete)h(b)q(efore)g(a)f(matc)o(hing)g(receiv)o(e)75 912 y(is)17 b(p)q(osted.)23 b(Messages)15 b(sen)o(t)h(b)o(y)h(an)f Fs(MPI)p 795 912 V 15 w(SEND)h FD(op)q(eration)g(can)f(b)q(e)h (bu\013ered)f(b)o(y)h(the)f(system,)g(so)f(as)75 968 y(to)g(allo)o(w)h(the)g(blo)q(c)o(king)h(send)f(op)q(eration)g(to)f (complete)h(ahead)g(of)f(the)h(matc)o(hing)f(receiv)o(e.)22 b(Ho)o(w)o(ev)o(er,)75 1025 y(if)15 b(no)g(suc)o(h)g(bu\013er)f(space)h (is)g(a)o(v)m(ailable,)h(then)f(the)g(send)g(op)q(eration)g(blo)q(c)o (ks)g(un)o(til)h(a)e(matc)o(hing)h(receiv)o(e)75 1081 y(is)h(executed,)f(or)g(su\016cien)o(t)h(bu\013er)f(space)h(b)q(ecomes) g(a)o(v)m(ailable.)166 1138 y(In)g(some)g(situations,)g(lac)o(k)g(of)g (bu\013er)g(space)g(ma)o(y)f(lead)i(to)e(deadlo)q(c)o(k)i(situations.) 23 b(This)16 b(is)h(illus-)75 1195 y(trated)d(b)o(y)i(the)f(examples)h (b)q(elo)o(w)g(\(in)o(v)o(olving)g(t)o(w)o(o)e(pro)q(cesses)h(with)h (ranks)f(0)f(and)i(1\))166 1252 y(The)21 b(follo)o(wing)i(program)d(is) i(safe,)g(and)f(can)h(succeed)g(ev)o(en)g(if)g(no)f(bu\013er)g(space)h (for)f(data)f(is)75 1308 y(a)o(v)m(ailable.)75 1417 y FB(CALL)j(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1474 y(IF)h(\(rank.EQ.0\))123 1530 y(THEN)170 1587 y(CALL)g (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 1643 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))123 1699 y(ELSE)95 b(!)23 b(rank.EQ.1)170 1756 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(status,)g(ierr\))170 1812 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 1869 y(END)g(IF)166 1978 y FD(The)15 b(follo)o(wing)h(program)e(is)i (erroneous,)f(and)g(will)i(alw)o(a)o(ys)d(fail.)75 2087 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2143 y(IF)h(\(rank.EQ.0\))123 2200 y(THEN)170 2256 y(CALL)g (MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(status,) g(ierr\))170 2313 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g (1,)h(tag,)f(comm,)g(ierr\))123 2369 y(ELSE)95 b(!)23 b(rank.EQ.1)170 2426 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g (0,)h(tag,)f(comm,)g(status,)g(ierr\))170 2482 y(CALL)h (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\)) 75 2538 y(END)g(IF)166 2647 y FD(The)15 b(receiv)o(e)g(op)q(eration)g (of)f(the)h(\014rst)f(pro)q(cess)h(m)o(ust)f(complete)i(b)q(efore)f (its)f(send,)h(and)g(can)g(com-)75 2704 y(plete)j(only)g(if)g(the)f (matc)o(hing)g(send)h(of)f(the)g(second)h(pro)q(cessor)f(is)h (executed;)g(the)g(receiv)o(e)g(op)q(eration)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 32 25 31 bop 75 -100 a Fp(3.4.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b FD(25)75 45 y(of)14 b(the)h(second)g(pro)q (cess)g(m)o(ust)f(complete)h(b)q(efore)g(its)f(send)i(and)e(can)h (complete)g(only)g(if)g(the)g(matc)o(hing)75 102 y(send)h(of)f(the)g (\014rst)g(pro)q(cess)g(is)h(executed.)21 b(This)15 b(program)f(will)j (deadlo)q(c)o(k.)166 158 y(The)c(follo)o(wing)h(program)f(is)g(unsafe,) h(and)f(will)i(deadlo)q(c)o(k)g(if)e(there)h(is)g(no)f(su\016cien)o(t)h (bu\013er)f(space.)75 248 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 305 y(IF)h(\(rank.EQ.0\))123 361 y(THEN)170 418 y(CALL)g(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(ierr\))170 474 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(1,)h(tag,)f(comm,)g(status,)g(ierr\))123 531 y(ELSE)95 b(!)23 b(rank.EQ.1)170 587 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))170 644 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(status,)g(ierr\))75 700 y(END)g(IF)166 790 y FD(The)f(message)g(sen)o(t)f(b)o(y)h(eac)o(h)g(pro)q(cess)g(has)g(to)g (b)q(e)g(copied)h(out)f(b)q(efore)g(the)g(send)h(op)q(eration)75 847 y(returns)15 b(and)h(the)f(receiv)o(e)h(op)q(eration)g(starts.)i(F) l(or)d(the)g(program)g(to)f(complete,)i(it)f(is)h(necessary)g(that)75 903 y(at)h(least)g(one)h(of)f(the)g(t)o(w)o(o)f(messages)h(sen)o(t)g (is)h(bu\013ered.)27 b(Th)o(us,)17 b(this)h(program)e(can)i(succeed)h (only)e(if)75 960 y(the)f(comm)o(unication)h(system)f(can)g(bu\013er)g (at)g(least)g Fs(count)i FD(w)o(ords)d(of)h(data.)22 b(Otherwise,)17 b(a)f(deadlo)q(c)o(k)75 1016 y(o)q(ccurs.)166 1073 y(It)24 b(is)h(exp)q(ected)g(that)e(MPI)h(implemen)o(tations)h (will)h(follo)o(w)e(the)h(practice)f(of)g(most)f(curren)o(t)75 1129 y(message-passing)18 b(systems)f(b)o(y)g(bu\013ering)h(outgoing)g (messages)f(whenev)o(er)h(feasible.)28 b(Ho)o(w)o(ev)o(er,)17 b(the)75 1185 y(amoun)o(t)12 b(of)h(bu\013er)h(space)f(a)o(v)m(ailable) i(and)e(the)h(bu\013er)f(allo)q(cation)h(p)q(olicy)h(ma)o(y)e(di\013er) g(from)g(implemen-)75 1242 y(tation)19 b(to)f(implemen)o(tation.)33 b(In)20 b(general,)g(the)g(programmer)d(can)j(mak)o(e)e(no)h (assumptions)h(on)f(the)75 1298 y(amoun)o(t)g(of)g(a)o(v)m(ailable)i (bu\013er)e(space,)i(and)e(ho)o(w)g(this)h(space)g(is)g(allo)q(cated.) 33 b(The)20 b(only)g(totally)f(safe)75 1355 y(assumption)d(is)h(that)e (an)h(arbitrary)g(subset)g(of)g(the)g(send)g(op)q(erations)h(are)e (going)h(to)g(return)g(b)q(efore)g(a)75 1411 y(matc)o(hing)f(receiv)o (e)h(is)g(p)q(osted,)f(and)h(the)f(rest)g(will)h(blo)q(c)o(k)g(un)o (til)h(a)e(matc)o(hing)g(receiv)o(e)h(is)g(p)q(osted.)166 1468 y(Noblo)q(c)o(king)21 b(message)e(passing)i(op)q(erations,)g(as)e (describ)q(ed)j(in)f(Section)g(3.8,)e(can)h(b)q(e)h(used)g(to)75 1524 y(a)o(v)o(oid)f(the)g(need)h(for)f(bu\013ering)h(outgoing)f (messages.)34 b(This)21 b(prev)o(en)o(ts)f(deadlo)q(c)o(ks)h(due)g(to)f (lac)o(k)g(of)75 1581 y(bu\013er)14 b(space,)f(and)h(impro)o(v)o(es)g (p)q(erformance,)f(b)o(y)h(allo)o(wing)g(o)o(v)o(erlap)g(of)f (computation)g(and)h(comm)o(uni-)75 1637 y(cation,)h(and)h(a)o(v)o (oiding)f(the)h(o)o(v)o(erheads)f(of)f(allo)q(cating)j(bu\013ers)e(and) h(cop)o(ying)f(messages)g(in)o(to)g(bu\013ers.)75 1694 y(F)l(urthermore,)c(an)g(MPI)g(implemen)o(tation)i(will)g(pro)o(vide)f (information)f(and)g(con)o(trol)g(on)h(the)f(amoun)o(t)f(of)75 1750 y(a)o(v)m(ailable)k(bu\013er)f(space)g(and)g(on)f(the)h (bu\013ering)g(p)q(olicy)h(via)f(the)g(calls)g Fs(MPI)p 1378 1750 14 2 v 16 w(GET)p 1482 1750 V 17 w(BUFFER)p 1668 1750 V 17 w(P)l(ARAMS)p FD(,)75 1806 y Fs(MPI)p 160 1806 V 16 w(GET)p 264 1806 V 17 w(BUFFER)p 450 1806 V 17 w(P)l(ARAMS)j FD(and)g Fs(MPI)p 833 1806 V 15 w(USER)p 960 1806 V 18 w(SPECIFIES)p 1192 1806 V 16 w(BUFFER)g FD(describ)q(ed)h(in)f(Chapter)f(7.)189 1897 y Fo(R)n(ationale.)189 1970 y FD(MPI)d(could)i(mandate)f(bu\013ering)g(of)g(outgoing)f (messages)h(b)o(y)g(sp)q(ecifying)h(that)f(a)f(blo)q(c)o(king)i(send) 189 2026 y(op)q(eration)20 b(is)g(lo)q(cal:)30 b(its)20 b(completion)g(dep)q(ends)i(only)e(on)f(the)h(state)f(of)g(the)h(lo)q (cal)h(executing)189 2083 y(pro)q(cess)16 b(and,)f(in)i(particular,)f (do)q(es)g(not)g(dep)q(end)h(on)f(the)g(o)q(ccurrence)h(of)e(a)h(matc)o (hing)f(receiv)o(e.)189 2139 y(The)h(implication)j(of)c(suc)o(h)i (de\014nition)h(is)f(that)e(a)h(blo)q(c)o(king)i(send)f(op)q(eration)f (will)i(fail)f(if)g(insuf-)189 2195 y(\014cien)o(t)j(bu\013er)g(space)f (is)h(a)o(v)m(ailable)i(when)e(the)f(send)h(executes)g(and)g(no)f(matc) o(hing)h(receiv)o(e)g(is)189 2252 y(p)q(osted.)f(This)c(is)g(the)g (implicit)i(seman)o(tics)d(of)g(blo)q(c)o(king)i(send)f(op)q(erations)f (on)h(systems)f(suc)o(h)g(as)189 2308 y(PVM)g([10)o(,)f(4,)h(3])g(and)g (P)l(ARMA)o(CS)g([5,)g(7)o(])g(where,)h(in)g(fact,)e(the)h(\014rst)g (condition)i(is)f(su\016cien)o(t)g(to)189 2365 y(cause)h(failure.)22 b(W)l(e)16 b(ha)o(v)o(e)g(decided)h(on)f(a)g(more)f(lenien)o(t)i (de\014nition:)23 b(lac)o(k)16 b(of)f(bu\013er)h(space)g(will)189 2421 y(not)d(cause)i(a)f(send)g(to)g(fail;)h(instead)f(the)g(send)h(op) q(eration)g(blo)q(c)o(ks)f(un)o(til)i(a)d(matc)o(hing)h(receiv)o(e)h (or)189 2478 y(su\016cien)o(t)e(bu\013er)f(space)h(b)q(ecame)g(a)o(v)m (ailable.)20 b Fx(This)15 b(allo)o(ws)f(more)g(programs)f(to)h (complete)189 2534 y(successfully)p FD(.)26 b(On)18 b(the)f(do)o(wn)g (side,)i(erroneous)e(programs)f(will)j(fail)f(in)h(a)e(harder)g(to)g (detect)189 2591 y(deadlo)q(c)o(k)e(mo)q(de,)f(rather)g(than)g(b)o(y)g (a)g(simple)h(failure)h(of)d(a)h(send)h(op)q(eration.)20 b(Our)14 b(design)h(p)q(oin)o(t)189 2647 y(matc)o(hes)c(the)h(implicit) i(practice)e(of)f(comm)o(unication)h(libraries)i(suc)o(h)d(as)h (Express)f([19)o(],)h(NX)g([20)o(],)189 2704 y(CMMD)i([)p Fx(?)p FD(])19 b(and)c(EUI)h([15)o(].)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 33 26 32 bop 75 -100 a FD(26)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)189 45 y FD(Man)o(y)g(users) i(w)o(ould)f(lik)o(e)i(to)e(ha)o(v)o(e)g(a)g(b)q(etter)g(guaran)o(tee)g (on)g(the)g(bu\013ering)h(of)f(messages)g(sen)o(t)189 102 y(b)o(y)f(blo)q(c)o(king)j(op)q(erations;)e(i.e.,)g(a)f(guaran)o (tee)g(that)g(some)h(programs)e(will)k(succeed)f(on)f(all)g(im-)189 158 y(plemen)o(tations)g(of)f(MPI)g(ev)o(en)h(if)g(they)g(are)f (written)g(so)h(as)f(require)h(message)f(bu\013ering.)24 b(This)189 214 y(issue)16 b(arises)f(whether)g(w)o(e)g(mandate)g (bu\013ering)h(of)f(outgoing)g(messages)f(or)h(not.)189 288 y(The)i(function)h Fs(MPI)p 549 288 14 2 v 15 w(GET)p 652 288 V 17 w(BUFFER)p 838 288 V 17 w(P)l(ARAMS)h FD(describ)q(ed)f (in)g(Chapter)f(7)g(partially)g(pro)o(vides)189 344 y(a)10 b(measure)h(of)f(a)o(v)m(ailalb)q(e)j(bu\013er)e(space;)h(ho)o(w)o(ev)o (er,)f(some)f(of)h(the)g(bu\013ering)g(parameters)f(ma)o(y)g(not)189 401 y(b)q(e)g(signi\014can)o(t)i(on)e(some)f(MPI)i(implemen)o(tations.) 19 b(The)10 b(function)h Fs(MPI)p 1427 401 V 16 w(SUGGEST)p 1642 401 V 18 w(BUFFER)p 1829 401 V 17 w(P)l(ARAMS)189 457 y FD(pro)o(vides)k(user)h(con)o(trol)f(of)g(bu\013ering.)20 b(Ho)o(w)o(ev)o(er,)14 b(MPI)i(implemen)o(tations)g(are)f(not)g (mandated)189 514 y(to)f(act)h(on)g(the)g(user)h(suggestion.)189 587 y(W)l(e)i(ha)o(v)o(e)h(not)f(found)h(a)f(w)o(a)o(y)g(of)g(pro)o (viding)i(b)q(etter)f(measure)f(or)g(more)h(con)o(trol)f(of)g (bu\013ering)189 643 y(without)i(o)o(v)o(ersp)q(ecifying)i(MPI)e (implemen)o(tation.)37 b(In)21 b(general,)g(bu\013er)g(space)g(for)e (outgoing)189 700 y(messages)f(ma)o(y)h(b)q(e)h(allo)q(cated)g(at)f (the)g(sender)h(side,)h(the)e(receiv)o(er)h(side,)h(or)e(b)q(oth.)32 b(A)19 b(bu\013er)189 756 y(ma)o(y)13 b(b)q(e)i(dedicated)g(to)f(a)g (pair)g(of)g(comm)o(unicating)h(pro)q(cesses,)f(or)g(shared)g(b)o(y)g (all)h(outgoing)f(\(or)189 813 y(all)g(incoming\))h(comm)o(unication)g (to)e(a)h(pro)q(cess,)g(or)f(shared)h(b)o(y)g(all)h(outgoing)e(\(or)g (all)i(incoming\))189 869 y(comm)o(unication)f(to)g(a)g(pro)q(cessor.) 19 b(Bu\013ers)14 b(ma)o(y)f(b)q(e)i(allo)q(cated)g(in)g(sp)q(ecial)g (device)h(memory)l(,)d(or)189 926 y(in)k(real)h(memory)e(\(user)h(or)f (k)o(ernel\),)i(or)e(in)i(virtual)f(memory)g(\(user)f(or)h(k)o (ernel\).)25 b(A)17 b(de\014nition)189 982 y(that)10 b(precludes)i(some)f(of)f(these)i(c)o(hoices)f(\(as)f(w)o(ell)i(as)f (more)f(designs)i(w)o(e)f(ha)o(v)o(e)f(not)h(y)o(et)f(imagined\))189 1039 y(will)16 b(signi\014can)o(tly)h(hamp)q(er)f(p)q(erformance)f(on)g (some)g(systems.)20 b(\()p Fo(End)15 b(of)i(r)n(ationale.)p FD(\))75 1180 y Fq(3.5)59 b(Data)19 b(T)-5 b(yp)r(e)19 b(Matching)75 1281 y FD(One)d(can)f(think)h(of)f(message)g (transmission)g(as)g(consisting)h(of)f(three)g(phases:)131 1366 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i(send)f (bu\013er)h(and)f(a)g(message)g(is)g(assem)o(bled)131 1456 y(2.)22 b(A)15 b(message)g(is)g(transferred)g(from)f(sender)i(to)f (receiv)o(er)131 1546 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h (incoming)i(message)e(and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o (e)h(bu\013er)166 1631 y(T)o(yp)q(e)k(matc)o(hing)f(has)h(to)e(b)q(e)j (observ)o(ed)e(at)g(eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f(eac)o(h)75 1687 y(v)m(ariable)i(in)f(the)g (sender)g(bu\013er)g(has)g(to)f(matc)o(h)g(the)g(t)o(yp)q(e)h(sp)q (eci\014ed)i(for)d(that)g(en)o(try)g(b)o(y)h(the)f(send)75 1744 y(op)q(eration;)d(the)f(t)o(yp)q(e)h(sp)q(eci\014ed)i(b)o(y)e(the) f(send)i(op)q(eration)e(has)h(to)f(matc)o(h)g(the)h(t)o(yp)q(e)f(sp)q (eci\014ed)j(b)o(y)e(the)75 1800 y(receiv)o(e)g(op)q(eration;)e(and)h (the)g(t)o(yp)q(e)f(of)h(eac)o(h)f(v)m(ariable)i(in)g(the)e(receiv)o(e) i(bu\013er)f(has)f(to)g(matc)o(h)g(the)h(t)o(yp)q(e)75 1857 y(sp)q(eci\014ed)f(for)e(that)g(en)o(try)g(b)o(y)g(the)g(receiv)o (e)h(op)q(eration.)20 b(A)12 b(program)f(that)h(fails)h(to)e(observ)o (e)h(these)h(three)75 1913 y(rules)j(is)g(erroneous.)166 1970 y(T)l(o)f(de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l (,)j(w)o(e)d(need)i(to)e(deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 2026 y(t)o(yp)q(es)e(of)g(the)h(host)f (language)g(with)h(t)o(yp)q(es)g(sp)q(eci\014ed)h(in)f(comm)o (unication)h(op)q(erations;)e(and)h(matc)o(hing)75 2083 y(of)h(t)o(yp)q(es)g(at)g(sender)g(and)h(receiv)o(er.)166 2139 y(A)h(t)o(yp)q(e)f(sp)q(eci\014ed)j(for)d(an)h(en)o(try)f(b)o(y)h (a)f(send)h(op)q(eration)g(matc)o(hes)f(the)h(t)o(yp)q(e)g(sp)q (eci\014ed)i(for)d(that)75 2195 y(en)o(try)11 b(b)o(y)h(a)f(receiv)o(e) h(op)q(eration)g(if)g(b)q(oth)f(op)q(erations)h(use)g(iden)o(tical)h (names:)36 b Fs(MPI)p 1500 2195 V 16 w(INTEGER)12 b FD(matc)o(hes)75 2252 y Fs(MPI)p 160 2252 V 16 w(INTEGER)p FD(,)j Fs(MPI)p 475 2252 V 16 w(REAL)h FD(matc)o(hes)30 b Fs(MPI)p 891 2252 V 15 w(REAL)p FD(,)16 b(and)f(so)g(on.)166 2308 y(The)k(t)o(yp)q(e)g(of)f(a)g(v)m(ariable)i(in)g(a)e(host)h(program)e (matc)o(hes)h(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)i(in)e(the)g(comm)o(u-) 75 2365 y(nication)g(op)q(eration)g(if)f(the)h(datat)o(yp)q(e)f(name)g (used)h(b)o(y)f(that)f(op)q(eration)i(corresp)q(onds)f(to)g(the)g (basic)75 2421 y(t)o(yp)q(e)i(of)g(the)g(host)g(program)f(v)m(ariable:) 31 b(an)20 b(en)o(try)f(with)i(t)o(yp)q(e)f(name)40 b Fs(MPI)p 1449 2421 V 16 w(INTEGER)21 b FD(matc)o(hes)e(a)75 2478 y(F)l(ortran)c(v)m(ariable)j(of)f(t)o(yp)q(e)f FB(INTEGER)p FD(,)g(an)g(en)o(try)g(with)h(t)o(yp)q(e)g(name)g Fs(MPI)p 1368 2478 V 16 w(REAL)g FD(matc)o(hes)f(a)g(F)l(ortran)75 2534 y(v)m(ariable)j(of)e(t)o(yp)q(e)g FB(REAL)p FD(,)f(and)i(so)f(on.) 26 b(There)18 b(is)f(one)h(exception)g(to)f(this)h(last)f(rule:)25 b(An)18 b(en)o(try)f(with)75 2591 y(t)o(yp)q(e)h(name)37 b Fs(MPI)p 408 2591 V 15 w(BYTE)19 b FD(can)f(b)q(e)h(used)g(to)e(matc) o(h)h(an)o(y)g(b)o(yte)g(of)g(storage)e(\(on)i(a)g(b)o(yte-addressable) 75 2647 y(mac)o(hine\),)d(irresp)q(ectiv)o(e)i(of)d(the)h(datat)o(yp)q (e)g(of)f(the)h(v)m(ariable)i(that)d(con)o(tains)h(this)g(b)o(yte.)20 b(The)15 b(v)m(alue)h(of)75 2704 y(the)f(message)g(en)o(try)g(will)i(b) q(e)f(the)f(binary)h(v)m(alue)g(of)f(the)g(corresp)q(onding)h(b)o(yte)f (in)h(memory)l(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 34 27 33 bop 75 -100 a Fp(3.5.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) 1105 b FD(27)166 45 y(W)l(e)15 b(th)o(us)g(ha)o(v)o(e)g(t)o(w)o(o)f (cases:)143 140 y FA(\017)23 b FD(Comm)o(unication)11 b(of)f(t)o(yp)q(ed)i(v)m(alues)g(\(e.g.,)e(with)h(datat)o(yp)q(e)g (di\013eren)o(t)g(from)21 b Fs(MPI)p 1586 140 14 2 v 16 w(BYTE)p FD(\),)11 b(where)189 197 y(the)k(datat)o(yp)q(es)f(of)g (the)h(corresp)q(onding)h(en)o(tries)f(in)h(the)f(sender)g(program,)e (in)j(the)f(send)g(call,)h(in)189 253 y(the)f(receiv)o(e)h(call)g(and)g (in)g(the)f(receiv)o(er)h(program)e(should)i(all)g(matc)o(h.)143 348 y FA(\017)23 b FD(Comm)o(unication)16 b(of)g(un)o(t)o(yp)q(ed)h(v)m (alues)g(\(e.g.,)e(of)g(datat)o(yp)q(e)h Fs(MPI)p 1330 348 V 16 w(BYTE)p FD(\),)g(where)g(b)q(oth)g(sender)189 405 y(and)g(receiv)o(er)h(use)f(the)g(datat)o(yp)q(e)g Fs(MPI)p 881 405 V 15 w(BYTE)p FD(.)h(In)f(this)h(case,)f(there)g(are)g (no)f(requiremen)o(ts)i(on)189 461 y(the)d(t)o(yp)q(es)h(of)f(the)g (corresp)q(onding)h(en)o(tries)g(in)h(the)e(sender)h(and)g(the)f (receiv)o(er)h(programs,)e(nor)h(is)189 517 y(it)h(required)h(that)f (they)g(b)q(e)h(the)f(same.)166 612 y(The)g(follo)o(wing)h(examples)g (illustrate)h(these)e(t)o(w)o(o)f(cases.)166 669 y(First)h(program:)75 765 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 821 y(IF\(rank.EQ.0\))123 877 y(THEN)170 934 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 990 y(ELSE)170 1047 y(CALL)h(MPI_RECV\(a\(1\),)e(15,)h(MPI_REAL,)g(0,)g (tag,)h(comm,)f(status,)g(ierr\))75 1103 y(END)g(IF)166 1198 y FD(This)c(co)q(de)f(is)h(correct)e(if)i(b)q(oth)f(sender)h(and)f (receiv)o(er)h(programs)d(ha)o(v)o(e)i(allo)q(cated)h(consecutiv)o(e)75 1255 y(storage)14 b(for)h(ten)g(real)g(n)o(um)o(b)q(ers,)g(starting)g (from)f Fs(a\(1\))p FD(.)166 1312 y(Second)i(program:)75 1407 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1463 y(IF\(rank.EQ.0\))123 1520 y(THEN)170 1576 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 1633 y(ELSE)170 1689 y(CALL)h(MPI_RECV\(a\(1\),)e(40,)h(MPI_BYTE,)g(0,) g(tag,)h(comm,)f(status,)g(ierr\))75 1746 y(END)g(IF)166 1841 y FD(This)d(co)q(de)g(is)f(erroneous,)h(since)g(sender)g(and)g (receiv)o(er)g(do)f(not)f(pro)o(vide)i(matc)o(hing)f(datat)o(yp)q(e)75 1897 y(argumen)o(ts.)166 1954 y(Third)d(program:)75 2049 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2106 y(IF\(rank.EQ.0\))123 2162 y(THEN)170 2219 y(CALL)h (MPI_SEND\(a\(1\),)e(40,)h(MPI_BYTE,)g(1,)g(tag,)h(comm,)f(ierr\))123 2275 y(ELSE)170 2331 y(CALL)h(MPI_RECV\(a\(1\),)e(60,)h(MPI_BYTE,)g(0,) g(tag,)h(comm,)f(status,)g(ierr\))75 2388 y(END)g(IF)166 2483 y FD(This)16 b(co)q(de)f(is)h(correct,)f(irresp)q(ectiv)o(e)h(of)f (the)g(t)o(yp)q(e)h(of)e Fs(a\(1\))g FD(and)i(follo)o(wing)g(v)m (ariables)g(in)g(store.)189 2591 y Fo(A)n(dvic)n(e)c(to)i(users.)38 b FD(If)12 b(a)g(bu\013er)g(of)f(t)o(yp)q(e)h Fv(MPI)p 965 2591 13 2 v 15 w(BYTE)f FD(is)i(passed)f(as)g(an)g(argumen)o(t)f (to)g Fs(MPI)p 1732 2591 14 2 v 16 w(SEND)p FD(,)95 b Fm(CHANGE)189 2647 y FD(then)14 b(MPI)g(will)h(send)g(the)f(data)f (stored)g(at)g(con)o(tiguous)h(lo)q(cations,)h(starting)e(from)g(the)h (address)189 2704 y(indicated)22 b(b)o(y)e(the)h Fs(buf)g FD(argumen)o(t.)35 b(This)21 b(ma)o(y)f(ha)o(v)o(e)g(unexp)q(ected)i (results)f(when)g(the)g(data)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 35 28 34 bop 75 -100 a FD(28)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)189 45 y FD(la)o(y)o(out)c (is)i(not)e(as)h(a)f(casual)i(user)f(w)o(ould)g(exp)q(ect)h(it)f(to)g (b)q(e:)19 b(F)l(or)12 b(example,)i(some)f(compilers)h(pass)189 102 y(a)e(F)l(ortran)f Fv(CHARA)o(CTER)g FD(argumen)o(t)h(as)g(a)g (structure)g(that)g(con)o(tains)g(the)h(c)o(haracter)f(length)h(and)189 158 y(a)k(p)q(oin)o(ter)i(to)e(the)h(actual)g(string.)29 b(In)18 b(suc)o(h)h(en)o(vironmen)o(t,)f(sending)h(or)f(receiving)h(a)f (F)l(ortran)189 214 y Fv(CHARA)o(CTER)f FD(v)m(ariable)j(using)g(the)f Fv(MPI)p 907 214 13 2 v 15 w(BYTE)f FD(will)j(not)e(ha)o(v)o(e)f(the)i (an)o(ticipated)g(result.)32 b(F)l(or)189 271 y(this)15 b(reason,)f(the)h(user)g(is)g(advised)h(to)e(use)h(t)o(yp)q(ed)g(comm)o (unication)h(whenev)o(er)f(p)q(ossible.)22 b(\()p Fo(End)189 327 y(of)16 b(advic)n(e)g(to)h(users.)p FD(\))75 451 y Fn(3.5.1)49 b(T)l(yp)q(e)17 b Fv(MPI)p 417 451 V 14 w(CHARA)o(CTER)75 537 y FD(The)12 b(t)o(yp)q(e)h Fv(MPI)p 342 537 V 14 w(CHARA)o(CTER)d FD(matc)o(hes)i(one)g(c)o(haracter)g(of)g (a)g(F)l(ortran)f(v)m(ariable)i(of)f(t)o(yp)q(e)g Fv(CHARA)o(CTER)p FD(.)-2122 b Fm(CHANGE)75 594 y FD(F)l(ortran)12 b(v)m(ariables)j(of)e (t)o(yp)q(e)h Fv(CHARA)o(CTER)e FD(or)h(substrings)h(are)f(transferred) g(as)g(if)h(they)g(w)o(ere)f(arra)o(ys)f(of)75 650 y(c)o(haracters.)18 b(The)c Fs(buf)f FD(argumen)o(t)g(should)h(b)q(e)f(a)g(reference)h(to)e (a)h(string)g(or)f(substring)i(that)e(starts)g(with)75 707 y(the)j(\014rst)g(c)o(haracter)g(of)f(the)i(comm)o(unication)g (bu\013er.)j(This)d(is)g(illustrated)g(in)g(the)g(example)g(b)q(elo)o (w.)75 815 y FB(CHARACTER*10)22 b(a)75 871 y(CHARACTER*10)g(b)75 984 y(CALL)h(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1041 y(IF\(rank.EQ.0\))123 1097 y(THEN)170 1154 y(CALL)h(MPI_SEND\(a,)e (5,)i(MPI_CHARACTER,)e(1,)h(tag,)h(comm,)f(ierr\))123 1210 y(ELSE)170 1267 y(CALL)h(MPI_RECV\(b\(6:10\),)d(5,)j (MPI_CHARACTER,)e(0,)h(tag,)h(comm,)f(status,)g(ierr\))75 1323 y(END)g(IF)166 1431 y FD(The)14 b(last)h(\014v)o(e)f(c)o (haracters)g(of)f(string)i Fv(b)f FD(at)g(pro)q(cess)g(1)g(are)g (replaced)i(b)o(y)e(the)g(\014rst)g(\014v)o(e)h(c)o(haracters)75 1487 y(of)g(string)g Fv(a)g FD(at)g(pro)q(cess)g(0.)166 1627 y Fl(Implemen)o(tati)o(on)10 b(note:)33 b Ft(Some)10 b(compilers)h(pass)h(F)m(ortran)f Fv(CHARA)o(CTER)f Ft(argumen)o(ts)h (as)h(a)f(structure)75 1683 y(with)16 b(a)f(\014eld)h(length)g(and)g(a) g(p)q(oin)o(ter)g(to)g(the)g(actual)g(string.)24 b(In)16 b(suc)o(h)h(en)o(vironmen)o(t,)e(the)i(MPI)f(call)f(need)i(to)75 1740 y(dereference)g(the)d(p)q(oin)o(ter)g(in)g(order)g(to)g(reac)o(h)h (the)f(string.)75 1944 y Fs(Alternative)i(De\014nition)75 2031 y FD(The)h(t)o(yp)q(e)g Fv(MPI)p 351 2031 V 14 w(CHARA)o(CTER)e FD(matc)o(hes)h(a)h(F)l(ortran)f(v)m(ariable)i(of)e(t)o(yp)q(e)h Fv(CHARA)o(CTER)e FD(or)h(a)h(substring)75 2087 y(of)e(arbitrary)f (length.)75 2195 y FB(CHARACTER*10)22 b(a)75 2252 y(CHARACTER*10)g(b)75 2365 y(CALL)h(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2421 y(IF\(rank.EQ.0\))123 2478 y(THEN)170 2534 y(CALL)h (MPI_SEND\(a\(1:5\),)d(1,)j(MPI_CHARACTER,)e(1,)i(tag,)f(comm,)g (ierr\))123 2591 y(ELSE)170 2647 y(CALL)h(MPI_RECV\(b\(6:10\),)d(1,)j (MPI_CHARACTER,)e(0,)h(tag,)h(comm,)f(status,)g(ierr\))75 2704 y(END)g(IF)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 36 29 35 bop 75 -100 a Fp(3.6.)34 b(D)o(A)l(T)l(A)15 b(CONVERSION)1197 b FD(29)166 45 y(The)14 b(last)h(\014v)o(e)f(c)o(haracters)g(of)f (string)i Fv(b)f FD(at)g(pro)q(cess)g(1)g(are)g(replaced)i(b)o(y)e(the) g(\014rst)g(\014v)o(e)h(c)o(haracters)75 102 y(of)g(string)g Fv(a)g FD(at)g(pro)q(cess)g(0.)166 159 y(Mismatc)o(hes)f(b)q(et)o(w)o (een)g(the)g(length)g(of)f(an)h(incoming)h(c)o(haracter)e(and)h(the)g (length)h(of)e(a)h(c)o(haracter)75 215 y(in)23 b(the)g(receiv)o(e)g (bu\013er)f(are)g(handled)i(as)e(in)h(F)l(ortran)f(c)o(haracter)f (assignmen)o(ts:)34 b(if)23 b(the)g(incoming)75 271 y(c)o(haracter)16 b(is)h(longer)f(than)h(the)f(target)f(c)o(haracter)h(then)h(it)f(is)h (truncated;)g(if)g(it)f(is)h(shorter)f(then)h(it)f(is)75 328 y(padded)g(with)g(blanks.)166 461 y Fl(Discussion:)33 b Ft(The)14 b(later)g(de\014nition)f(is)g(more)g(elegan)o(t)g(than)h (the)g(former)e(one.)19 b(Unfortunately)m(,)12 b(it)h(do)q(es)75 511 y(not)h(sit)g(w)o(ell)f(with)g(the)i(use)g(of)e(deriv)o(ed)h(datat) o(yp)q(es.)166 561 y(Consider)c(a)f(system)g(where)i(the)f(length)g(of) f(a)g(c)o(haracter)i(argumen)o(t)d(is)h(passed)i(as)f(an)f(additional)f (argumen)o(t.)75 611 y(Assume)16 b(that)h(the)g(program)e(pic)o(ks)h (the)h(address)h(of)e(a)g(string)g(with)g(a)g(call)g(to)g Fv(MPI)p 1448 611 13 2 v 14 w(ADRRESS)p Ft(,)f(next)i(builds)75 661 y(the)c(deriv)o(ed)g(datat)o(yp)q(e)g(for)f(a)g(bu\013er)h(that)g (includes)g(the)g(string)f(with)g(a)h(call)e(to)i Fv(MPI)p 1435 661 V 14 w(STRUCT)p Ft(.)e(The)i(length)g(of)75 711 y(the)i(string)g(w)o(as)g(passed)h(as)f(an)g(extra)g(argumen)o(t)f (to)g(the)i(function)e Fv(MPI)p 1250 711 V 15 w(ADDRESS)p Ft(,)f(whic)o(h)i(ignored)g(it.)21 b(The)75 760 y(information)8 b(that)j(a)g(particular)g(v)n(ariable)f(is)h(of)g(c)o(haracter)h(t)o (yp)q(e)g(is)f(passed)h(as)f(an)g(argumen)o(t)f(to)h Fv(MPI)p 1690 760 V 14 w(STRUCT)p Ft(,)75 810 y(whic)o(h)g(has)g(no)g (w)o(a)o(y)f(to)g(reco)o(v)o(er)j(the)e(information)d(on)j(the)g(c)o (haracter)i(length)d(\(unless)i(one)f(w)o(an)o(ts)g Fv(MPI)p 1701 810 V 15 w(STRUCT)75 860 y Ft(to)k(access)i(a)e(sym)o(b)q(ol)f (table\).)22 b(Th)o(us,)15 b(w)o(e)h(w)o(ould)e(need)i Fv(MPI)p 1049 860 V 15 w(ADDRESS)e Ft(to)h(alw)o(a)o(ys)g(c)o(hec)o(k)h (whether)g(an)f(extra)75 910 y(argumen)o(t)e(\(c)o(haracter)i(length\)) f(w)o(as)g(passed)h(and,)e(if)g(so,)h(sa)o(v)o(e)g(the)g(v)n(alue)f(of) h(this)g(extra)g(argumen)o(t.)166 960 y Fv(MPI)p 243 960 V 14 w(SEND)g Ft(also)f(needs)i(sp)q(ecial)f(co)q(de)g(to)g(handle) f(the)i(case)f(where)h Fv(t)o(yp)q(e)g(=)f(MPI)p 1476 960 V 14 w(CHARA)o(CTER)p Ft(.)d(In)j(this)75 1010 y(case,)e(the)f(v)n (alue)f(of)g(the)h(extra)g(length)f(argumen)o(t)f(needs)j(b)q(e)f(used) h(to)e(\014nd)h(the)g(length)f(of)g(the)h(outgoing)f(message.)166 1067 y(The)18 b(basic)g(problem)e(is)i(that)f(w)o(e)h(assumed)f(that)h (a)g(deriv)o(ed)g(datat)o(yp)q(e)g(is)f(a)g(static)h(pattern)h(\(a)e (list)h(of)75 1124 y(displacemen)o(ts)11 b(and)h(lengths\))g(that,)g (together)g(with)g(a)f(base)h(address,)h(iden)o(ti\014es)f(the)h(set)f (of)f(memory)e(lo)q(cations)75 1180 y(that)i(form)f(the)i(comm)o (unicati)o(on)d(bu\013er.)18 b(With)10 b(the)i(second)h(de\014nition)d (for)h Fv(MPI)p 1372 1180 V 15 w(CHARA)o(CTER)p Ft(,)e(w)o(e)i(no)o(w)g (ha)o(v)o(e)75 1236 y(a)17 b(dynamic)f(deriv)o(ed)i(datat)o(yp)q(e)g (that)f(assumes)g(v)n(alue)g(\(length\))h(only)e(when)i(b)q(ound)g(to)f (a)g(sp)q(eci\014c)i(lo)q(cation.)75 1293 y(Mo)o(ving)14 b(to)o(w)o(ard)h(suc)o(h)g(dynamic,)f(parameterized)h(datat)o(yp)q(es)h (is)f(tempting:)k(the)c(need)h(for)f(them)f(is)h(lik)o(ely)f(to)75 1349 y(arise)h(in)f(other)i(places)f(in)g(F)m(ortran)f(90)g(or)h(C++,)g (whenev)o(er)h(language)e(t)o(yping)g(is)h(dynamic,)e(and)i(ob)r(jects) h(are)75 1406 y(self-t)o(yp)q(ed.)28 b(\(An)17 b(example:)23 b(an)17 b(arra)o(y)g(section)h(in)e(F)m(ortran)h(90,)g(with)g(a)f (compiler)g(that)h(uses)h(dop)q(e)g(v)o(ectors)75 1462 y(to)d(main)o(tain)d(the)k(dynamic)d(information)f(on)j(data)g(la)o(y)o (out.)20 b(The)15 b(righ)o(t)g(buzzw)o(ord)h(is)e(\\p)q(olymorphic".\)) 20 b(The)75 1519 y(question)15 b(is)f(whether)i(w)o(e)e(w)o(an)o(t)g (to)h(handle)f(this)h(at)f(the)h(curren)o(t)h(MPI)f(la)o(y)o(er,)e(or)i (as)f(a)g(metastructure)i(on)e(top)75 1575 y(of)f(curren)o(t)j(MPI.)75 1804 y Fq(3.6)59 b(Data)19 b(conversion)75 1907 y FD(One)d(of)f(the)g (goals)g(of)g(MPI)g(is)h(to)e(supp)q(ort)i(parallel)g(computations)f (across)g(heterogeneous)g(en)o(viron-)75 1963 y(men)o(ts.)20 b(Comm)o(unication)15 b(in)h(a)f(heterogeneous)g(en)o(vironmen)o(t)h (ma)o(y)e(require)i(data)f(con)o(v)o(ersions.)166 2020 y(W)l(e)g(use)h(the)f(follo)o(wing)h(terminology:)75 2116 y Fx(t)o(yp)q(e)h(con)o(v)o(ersion)22 b FD(c)o(hanges)16 b(the)h(datat)o(yp)q(e)f(of)g(a)g(v)m(alue,)h(e.g.)23 b(b)o(y)17 b(rounding)g(a)f Fs(REAL)h FD(to)f(an)g Fs(INTE-)189 2173 y(GER)p FD(.)75 2269 y Fx(represen)o(tation)h(con)o(v)o(ersion)22 b FD(c)o(hanges)16 b(the)h(binary)g(represen)o(tation)f(of)g(a)g(v)m (alue,)h(e.g.)23 b(from)16 b(Hex)189 2325 y(\015oating)f(p)q(oin)o(t)g (to)g(IEEE)g(\015oating)h(p)q(oin)o(t.)166 2421 y(The)d(t)o(yp)q(e)g (matc)o(hing)g(rules)g(imply)h(that)e(MPI)h(comm)o(unication)h(nev)o (er)f(en)o(tails)g(t)o(yp)q(e)g(con)o(v)o(ersion.)75 2478 y(On)20 b(the)f(other)f(hand,)i(MPI)f(requires)h(that)e(a)h (represen)o(tation)g(con)o(v)o(ersion)g(b)q(e)h(p)q(erformed)f(when)g (a)75 2534 y(t)o(yp)q(ed)i(v)m(alue)h(is)f(transferred)f(across)g(en)o (vironmen)o(ts)h(that)f(use)h(di\013eren)o(t)f(represen)o(tations)h (for)f(the)75 2591 y(datat)o(yp)q(e)g(of)g(this)h(v)m(alue.)38 b(MPI)20 b(do)q(es)h(not)f(sp)q(ecify)i(rules)g(for)d(represen)o (tation)i(con)o(v)o(ersion.)36 b(Suc)o(h)75 2647 y(con)o(v)o(ersion)21 b(is)g(exp)q(ected)i(to)d(preserv)o(e)h(in)o(teger,)h(logical)g(or)e(c) o(haracter)h(v)m(alues,)i(and)e(to)f(con)o(v)o(ert)g(a)75 2704 y(\015oating)15 b(p)q(oin)o(t)h(v)m(alue)g(to)f(the)g(nearest)g(v) m(alue)h(that)f(can)g(represen)o(ted)h(on)f(the)g(target)f(system.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 37 30 36 bop 75 -100 a FD(30)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FD(Ov)o(er\015o)o (w)10 b(and)h(under\015o)o(w)g(exceptions)h(ma)o(y)e(o)q(ccur)i(during) f(\015oating)g(p)q(oin)o(t)g(con)o(v)o(ersions;)h(con)o(v)o(er-)75 102 y(sion)i(of)e(in)o(tegers)i(or)e(c)o(haracters)h(ma)o(y)f(also)h (lead)h(to)f(exceptions)h(when)g(a)e(v)m(alue)j(that)d(can)h(represen)o (ted)75 158 y(in)18 b(one)g(system)f(cannot)g(b)q(e)h(represen)o(ted)g (in)h(the)e(other)g(system.)26 b(An)18 b(exception)h(o)q(ccurring)f (during)75 214 y(represen)o(tation)f(con)o(v)o(ersion)f(results)h(in)h (a)e(failure)i(of)e(the)g(comm)o(unication;)i(an)e(error)g(o)q(ccurs)h (either)75 271 y(in)f(the)f(send)h(op)q(eration,)f(or)g(the)g(receiv)o (e)h(op)q(eration,)f(or)g(b)q(oth.)166 333 y(If)22 b(a)f(v)m(alue)i (sen)o(t)e(in)h(a)f(message)g(is)h(un)o(t)o(yp)q(ed)h(\(i.e.,)f(of)f(t) o(yp)q(e)h Fs(MPI)p 1370 333 14 2 v 15 w(BYTE)p FD(\),)g(then)g(the)f (binary)75 389 y(represen)o(tation)15 b(of)f(the)h(b)o(yte)g(stored)f (at)g(the)h(receiv)o(er)h(is)f(iden)o(tical)i(to)d(the)h(binary)h (represen)o(tation)e(of)75 446 y(the)f(b)o(yte)g(loaded)h(at)e(the)h (sender.)20 b(This)14 b(holds)f(true,)g(whether)g(sender)h(and)f (receiv)o(er)h(run)f(in)h(the)f(same)75 502 y(or)i(in)h(distinct)g(en)o (vironmen)o(ts;)f(no)g(represen)o(tation)h(con)o(v)o(ersion)f(is)h (required.)166 564 y(Note)d(that)g(no)g(con)o(v)o(ersion)h(ev)o(er)f(o) q(ccurs)h(when)g(an)f(MPI)g(program)g(executes)h(in)g(a)f(homogeneous) 75 621 y(system,)i(where)h(all)g(pro)q(cesses)g(run)g(in)g(the)g(same)f (en)o(vironmen)o(t.)22 b(Also)16 b(note)f(the)h(di\013eren)o(t)f(b)q (eha)o(vior)75 677 y(of)e Fs(MPI)p 210 677 V 15 w(BYTE)h FD(and)f(of)g Fs(MPI)p 575 677 V 15 w(CHAR)p FD(.)h(A)f(bu\013er)g (descriptor)g(en)o(try)g(with)g(datat)o(yp)q(e)g(of)f Fs(MPI)p 1660 677 V 16 w(CHAR)i FD(can)75 734 y(only)f(matc)o(h)f(a)g (C)g(v)m(ariable)i(of)e(t)o(yp)q(e)h FB(CHAR)p FD(;)e(and)i(represen)o (tation)g(con)o(v)o(ersion)f(ma)o(y)g(o)q(ccur)h(when)g(v)m(alues)75 790 y(of)g(t)o(yp)q(e)h Fs(MPI)p 311 790 V 15 w(CHAR)h FD(are)e(transferred.,)g(e.g.,)f(from)h(an)g(EBCDIC)h(enco)q(ding)h(to) e(an)g(ASCI)q(I)i(enco)q(ding.)166 852 y(Consider)h(the)f(previous)h (three)f(examples)h(on)f(page)g(27.)166 914 y(The)c(\014rst)f(program)f (is)i(correct,)g(assuming)g(b)q(oth)f(sender)i(and)e(receiv)o(er)i (declared)f(ten)g(consecutiv)o(e)75 971 y(real)20 b(v)m(ariables)h(in)g (storage)e(starting)g(at)g Fs(a\(1\))p FD(.)33 b(If)20 b(the)g(sender)g(and)g(receiv)o(er)h(execute)f(in)h(di\013eren)o(t)75 1027 y(en)o(vironmen)o(ts,)12 b(then)f(the)g(ten)f(real)i(v)m(alues)f (that)g(are)f(fetc)o(hed)h(from)f(the)h(send)h(bu\013er)e(will)j(b)q(e) e(con)o(v)o(erted)75 1084 y(to)g(the)h(represen)o(tation)f(for)g(reals) h(on)f(the)h(receiv)o(er)g(site)g(b)q(efore)g(they)g(are)f(stored)g(in) h(the)g(receiv)o(e)g(bu\013er.)75 1140 y(While)19 b(the)e(n)o(um)o(b)q (er)h(of)f(real)g(elemen)o(ts)h(fetc)o(hed)g(from)f(the)g(send)h (bu\013er)f(equal)i(the)e(n)o(um)o(b)q(er)h(of)f(real)75 1196 y(elemen)o(ts)d(stored)e(in)i(the)f(receiv)o(e)h(bu\013er,)f(the)g (n)o(um)o(b)q(er)g(of)g(b)o(ytes)g(stored)f(need)i(not)f(equal)h(the)f (n)o(um)o(b)q(er)75 1253 y(of)f(b)o(ytes)g(loaded:)19 b(e.g.)g(the)12 b(sender)h(ma)o(y)e(use)i(a)f(four)g(b)o(yte)g (represen)o(tation)h(and)f(the)g(receiv)o(er)i(an)e(eigh)o(t)75 1309 y(b)o(yte)17 b(represen)o(tation)h(for)f(reals.)27 b(If)18 b(the)g(send)g(or)f(receiv)o(e)h(bu\013er)g(do)f(not)h(con)o (tain)f(ten)h(consecutiv)o(e)75 1366 y(real)e(v)m(ariables,)g(then)f (the)g(program)g(is)g(erroneous,)g(and)g(its)h(b)q(eha)o(vior)f(is)h (unde\014ned.)166 1428 y(The)f(second)h(program)e(is)i(erroneous,)f (and)g(its)g(b)q(eha)o(vior)h(is)g(unde\014ned.)166 1490 y(The)e(third)g(program)e(is)i(correct.)19 b(The)13 b(exact)g(same)g (sequence)i(of)e(fort)o(y)f(b)o(ytes)h(that)g(w)o(ere)g(loaded)75 1546 y(from)18 b(the)h(send)h(bu\013er)f(will)i(b)q(e)e(stored)g(in)h (the)f(receiv)o(e)h(bu\013er,)f(ev)o(en)h(if)f(sender)h(and)f(receiv)o (er)h(run)75 1603 y(in)e(a)g(di\013eren)o(t)f(en)o(vironmen)o(t.)28 b(The)18 b(message)f(sen)o(t)g(has)h(exactly)f(the)h(same)f(length)i (\(in)f(b)o(ytes\))f(and)75 1659 y(the)f(same)f(binary)h(represen)o (tation)f(as)g(the)h(message)f(receiv)o(ed.)22 b(If)16 b(the)f(v)m(ariables)i(in)f(the)g(send)g(bu\013er)75 1716 y(are)f(of)h(di\013eren)o(t)g(t)o(yp)q(es)g(from)f(the)h(v)m (ariables)h(in)f(the)g(receiv)o(e)h(bu\013er,)e(or)h(they)g(are)f(of)h (the)f(same)h(t)o(yp)q(e)75 1772 y(but)h(di\013eren)o(t)g(data)g (represen)o(tations)f(are)h(used,)h(then)f(the)g(bits)h(stored)e(in)i (the)f(receiv)o(e)h(bu\013er)f(ma)o(y)75 1829 y(enco)q(de)f(v)m(alues)h (that)d(are)h(di\013eren)o(t)g(from)g(the)g(v)m(alues)h(they)g(enco)q (ded)g(in)g(the)g(send)f(bu\013er.)166 1891 y(Data)f(represen)o(tation) i(con)o(v)o(ersion)f(also)h(applies)h(to)e(the)g(en)o(v)o(elop)q(e)i (of)e(a)g(message:)20 b(source,)15 b(des-)75 1947 y(tination)h(and)f (tag)f(are)h(all)h(in)o(tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i (con)o(v)o(erted.)166 2085 y Fl(Implemen)o(tati)o(on)11 b(note:)36 b Ft(The)13 b(curren)o(t)i(de\014nition)e(do)q(es)h(not)f (require)h(messages)g(to)f(carry)h(data)e(t)o(yp)q(e)75 2135 y(information.)j(A)e(message)g(can)g(b)q(e)g(comp)q(osed)g(and)g (sen)o(t)g(using)g(only)f(the)i(information)c(pro)o(vided)j(b)o(y)f (the)i(send)75 2185 y(call,)k(and)g(can)h(b)q(e)g(receiv)o(ed)h(and)e (stored)h(using)f(only)g(the)h(information)d(pro)o(vided)i(b)o(y)g(the) h(receiv)o(e)h(call.)30 b(If)75 2235 y(messages)16 b(are)f(sen)o(t)i(b) q(et)o(w)o(een)g(di\013eren)o(t)f(mac)o(hines)f(then)h(one)f(can)h (either)g(use)h(a)e(\\univ)o(ersal")g(data)g(enco)q(ding)75 2284 y(for)g(messages,)f(use)i(kno)o(wledge)f(of)f(the)i(receiv)o(er)g (en)o(vironmen)o(t)e(in)h(order)g(to)g(con)o(v)o(ert)h(data)e(at)h(the) h(sender,)g(or)75 2334 y(use)e(kno)o(wledge)f(of)f(the)i(sender)g(en)o (vironmen)o(t)e(in)h(order)g(to)g(con)o(v)o(ert)h(data)f(at)g(the)g (receiv)o(er.)19 b(In)13 b(either)h(case)g(the)75 2384 y(datat)o(yp)q(e)g(argumen)o(t)f(in)g(the)i(lo)q(cal)e(call)g(can)h(b)q (e)h(used)f(to)g(deriv)o(e)g(the)h(t)o(yp)q(es)g(of)e(the)h(v)n(alues)g (transferred.)166 2446 y(Additional)e(t)o(yp)q(e)j(information)c(can)j (b)q(e)g(carried)h(in)e(messages)h(in)f(order)i(to)f(pro)o(vide)f(b)q (etter)j(error)f(detec-)75 2502 y(tion.)166 2647 y FD(MPI)k(do)q(es)g (not)g(require)h(supp)q(ort)f(for)f(in)o(ter-language)i(comm)o (unication:)28 b(The)19 b(b)q(eha)o(vior)h(of)e(a)-2120 b Fm(CHANGE)75 2704 y FD(program)12 b(is)i(unde\014ned)h(if)e(messages) g(are)g(sen)o(t)g(b)o(y)g(a)g(C)g(pro)q(cess)g(and)h(receiv)o(ed)g(b)o (y)f(a)g(F)l(ortran)f(pro)q(cess,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 38 31 37 bop 75 -100 a Fp(3.7.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b FD(31)75 45 y(or)15 b(vice-v)o(ersa.)166 102 y(MPI)f(implemen)o(tors)g(ma)o(y)f(pro)o(vide)h(supp)q(ort)g(for)f (in)o(ter-language)h(comm)o(unication)h(in)f(a)g(v)m(ariet)o(y)75 158 y(of)h(w)o(a)o(ys:)143 250 y FA(\017)23 b FD(One)16 b(ma)o(y)f(in)o(v)o(ok)o(e)g(C)g(comm)o(unication)h(routines)g(from)f (a)g(F)l(ortran)f(program)h(or)g(F)l(ortran)f(com-)189 306 y(m)o(unication)g(routines)g(from)f(a)h(C)f(program,)g(on)g (systems)h(that)e(supp)q(ort)i(in)o(v)o(ok)m(ation)h(of)e(foreign)189 363 y(language)i(routines.)143 456 y FA(\017)23 b FD(One)e(ma)o(y)e (allo)o(w)i(the)f(use)h(of)f(\\C)g(MPI)g(t)o(yp)q(es",)h(suc)o(h)f(as)g Fv(MPI)p 1332 456 13 2 v 15 w(INT,)e(MPI)p 1521 456 V 14 w(CHAR)p FD(,)h(etc.)36 b(from)189 512 y(F)l(ortran)14 b(programs,)f(and)j(vice-v)o(ersa.)143 605 y FA(\017)23 b FD(One)16 b(ma)o(y)e(\014x)h(the)h(corresp)q(ondance)g(b)q(et)o(w)o (een)f(C)g(datat)o(yp)q(es)g(and)g(F)l(ortran)f(datat)o(yp)q(es.)189 709 y Fo(R)n(ationale.)46 b FD(MPI)17 b(do)q(es)f(not)g(handle)i(in)o (ter-language)f(comm)o(unication)g(b)q(ecause)g(there)g(seem)189 766 y(to)d(b)q(e)i(no)f(agreed)g(standards)g(for)g(in)o(ter-language)g (t)o(yp)q(e)h(con)o(v)o(ersions.)k(\()p Fo(End)15 b(of)i(r)n(ationale.) p FD(\))75 908 y Fq(3.7)59 b(Communication)18 b(Mo)r(des)75 1010 y FD(The)13 b(basic)h(send)f(op)q(eration)g(describ)q(ed)i(in)f (section)f(3.2)f(used)h(the)g Fx(standard)g FD(comm)o(unication)h(mo)q (de.)75 1066 y(In)g(suc)o(h)g(a)f(comm)o(unication)h(mo)q(de,)g(a)f (send)h(op)q(eration)g(can)f(b)q(e)h(started)f(whether)h(or)f(not)g(a)g (matc)o(hing)75 1123 y(receiv)o(e)18 b(w)o(as)e(p)q(osted.)24 b(The)17 b(completion)h(of)e(the)h(send)h(op)q(eration)e(indicates)j (that)d(the)g(message)h(and)75 1179 y(its)f(en)o(v)o(elop)q(e)h(ha)o(v) o(e)e(b)q(een)i(safely)f(stored)f(a)o(w)o(a)o(y)f(and)i(that)f(the)h (sender)g(is)g(free)g(to)f(access)h(and)f(mo)q(dify)75 1236 y(the)j(sender)h(bu\013er.)30 b(Th)o(us,)18 b(the)h(op)q(eration)f (is)h Fx(lo)q(cally-blo)r(c)o(king)p FD(:)29 b(it)19 b(do)q(es)f(not)g(return)g(un)o(til)i(the)75 1292 y(send)c(op)q (eration)f(has)g(completed)h Fx(lo)q(cally)p FD(,)i(on)d(the)g(sender)h (side.)166 1349 y(The)f(completion)i(of)e(a)g(send)h(op)q(eration)f (giv)o(es)h(no)f(indication)i(that)e(the)g(message)g(w)o(as)g(receiv)o (ed)75 1405 y(on)i(the)g(receiv)o(er)h(side.)26 b(A)17 b(blo)q(c)o(king)i(send)e(ma)o(y)g(b)q(e)g(implemen)o(ted)i(so)e(that)f (it)h(returns)g(only)h(after)e(a)75 1462 y(matc)o(hing)j(receiv)o(e)h (has)f(b)q(een)h(executed)g(on)f(the)h(receiv)o(er)f(side.)33 b(This)20 b(a)o(v)o(oids)e(the)i(need)g(to)e(bu\013er)75 1518 y(message)e(data)f(out)h(of)f(sender)i(or)e(receiv)o(er)i(memory)l (.)22 b(In)17 b(this)f(case)g(the)g(send)h(op)q(eration)f(completes)75 1574 y(only)e(after)e(the)i(matc)o(hing)f(receiv)o(e)h(has)g(started)e (executing.)21 b(On)13 b(the)h(other)f(hand,)g(it)h(is)g(also)f(p)q (ossible)75 1631 y(for)j(MPI)g(to)f(bu\013er)h(messages,)g(so)g(as)g (to)f(allo)o(w)i(the)f(sender)h(to)e(pro)q(ceed)i(ahead)g(of)e(the)i (receiv)o(er.)23 b(In)75 1687 y(suc)o(h)15 b(a)g(case)g(the)g(send)g (op)q(eration)g(ma)o(y)f(complete)i(successfully)h(b)q(efore)e(the)g (message)f(w)o(as)g(receiv)o(ed.)166 1744 y(There)h(are)g(t)o(w)o(o)f (additional)j(comm)o(unication)f(mo)q(des:)166 1800 y(A)e(send)h(that)e (uses)h(the)g Fx(ready)g FD(comm)o(unication)g(mo)q(de)h(ma)o(y)e(b)q (e)i(started)e(only)h(if)h(the)f(matc)o(hing)75 1857 y(receiv)o(e)g(is)f(already)g(p)q(osted;)g(otherwise)g(the)g(op)q (eration)g(is)g(erroneous)f(and)h(its)g(outcome)f(is)h(unde\014ned.)75 1913 y(In)19 b(some)f(systems,)h(this)f(allo)o(ws)h(the)f(remo)o(v)m (al)h(of)f(a)g(hand-shak)o(e)h(op)q(eration)g(that)e(is)i(otherwise)g (re-)75 1970 y(quired,)g(and)f(results)g(in)g(impro)o(v)o(ed)g(p)q (erformance.)27 b(The)18 b(completion)h(of)e(the)h(send)g(op)q(eration) g(do)q(es)75 2026 y(not)e(dep)q(end)i(on)e(the)g(status)f(of)h(a)g (matc)o(hing)g(receiv)o(e,)h(and)f(merely)h(indicates)h(that)d(the)h (send)h(bu\013er)75 2083 y(can)i(b)q(e)g(reused.)30 b(A)19 b(send)g(op)q(eration)g(that)f(uses)g(the)h(ready)f(mo)q(de)h(has)g (the)f(same)g(seman)o(tics)h(as)f(a)75 2139 y(standard)h(send)h(op)q (eration;)h(it)f(is)g(merely)g(that)f(it)g(can)h(b)q(e)g(used)g(only)g (in)g(some)f(of)g(the)h(situations)75 2195 y(where)15 b(a)g(regular)h(send)f(can)h(b)q(e)g(used.)166 2252 y(A)i(send)g(that)f (uses)g(the)h Fx(sync)o(hronous)e FD(mo)q(de)i(can)g(b)q(e)g(started)f (whether)g(or)g(not)h(a)f(matc)o(hing)75 2308 y(receiv)o(e)12 b(w)o(as)e(p)q(osted.)19 b(Ho)o(w)o(ev)o(er,)10 b(the)h(send)h(will)g (complete)g(successfully)h(only)f(if)f(a)g(matc)o(hing)g(receiv)o(e)g (is)75 2365 y(p)q(osted,)h(and)g(the)g(receiv)o(e)g(op)q(eration)g(has) f(started)g(to)g(receiv)o(e)i(the)e(message)g(sen)o(t)h(b)o(y)f(the)h (sync)o(hronous)75 2421 y(send.)29 b(\(I.e.,)19 b(the)f(receiv)o(e)h (has)f(b)q(een)h(p)q(osted,)g(and)g(the)f(incoming)h(message)f(has)g(b) q(een)i(matc)o(hed)e(to)75 2478 y(this)g(p)q(osted)g(receiv)o(e.\))28 b(Th)o(us,)18 b(the)g(completion)g(of)g(a)f(sync)o(hronous)h(send)g (not)f(only)i(indicates)g(that)75 2534 y(the)g(send)h(bu\013er)f(can)g (b)q(e)h(reused,)g(but)f(also)g(indicates)i(that)d(the)i(receiv)o(er)f (has)g(reac)o(hed)h(a)f(certain)75 2591 y(p)q(oin)o(t)e(in)h(its)f (execution,)h(namely)g(that)e(it)h(has)g(started)f(executing)i(the)f (matc)o(hing)g(receiv)o(e.)26 b(If)17 b(b)q(oth)75 2647 y(sends)f(and)h(receiv)o(es)f(are)g(blo)q(c)o(king)h(op)q(erations)f (then)h(the)f(use)g(of)f(the)h(sync)o(hronous)g(mo)q(de)g(pro)o(vides) 75 2704 y(sync)o(hronous)h(comm)o(unication)g(seman)o(tics:)23 b(a)16 b(comm)o(unication)h(do)q(es)g(not)g(complete)g(at)f(either)h (end)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 39 32 38 bop 75 -100 a FD(32)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FD(b)q(efore)c(b)q (oth)g(pro)q(cesses)g(\\attend")f(to)g(the)h(comm)o(unication;)h(the)f (completion)h(of)e(a)h(sync)o(hronous)f(send)75 102 y(is)16 b(a)f Fx(global)i FD(ev)o(en)o(t,)d(and)h(the)h(op)q(eration)f(is)h Fx(globally-blo)q(cking)p FD(.)166 158 y(Tw)o(o)22 b(additional)j(send) e(functions)h(are)f(pro)o(vided)h(for)f(the)g(t)o(w)o(o)f(additional)i (comm)o(unication)75 214 y(mo)q(des.)38 b(The)22 b(comm)o(unication)g (mo)q(de)f(is)h(indicated)h(b)o(y)e(a)g(one)h(letter)f(pre\014x:)32 b Fs(R)22 b FD(for)f(ready)g(and)75 271 y Fs(S)15 b FD(for)g(sync)o (hronous.)166 327 y(Send)h(in)g(ready)f(mo)q(de)75 431 y Fs(MPI)p 160 431 14 2 v 16 w(RSEND)h(\(buf,)f(count,)h(datat)o(yp)q (e,)h(dest,)f(tag,)f(comm)m(\))117 508 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 583 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 657 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 731 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 805 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 879 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))75 1004 y FB(int)23 b(MPI)p 245 1004 15 2 v 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1004 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1060 y(int)h(tag,)f(MPI)p 683 1060 V 17 w(Comm)g(comm\))75 1147 y(MPI)p 150 1147 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 1203 y()f(BUF\(*\))170 1260 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 1346 y FD(Send)16 b(in)g(sync)o(hronous)f(mo)q(de)75 1450 y Fs(MPI)p 160 1450 14 2 v 16 w(SSEND)h(\(buf,)f(count,)h(datat)o (yp)q(e,)g(dest,)h(tag,)e(comm)m(\))117 1527 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 1601 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1675 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1750 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1824 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 1898 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))75 2023 y FB(int)23 b(MPI)p 245 2023 15 2 v 17 w(Ssend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2023 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2079 y(int)h(tag,)f(MPI)p 683 2079 V 17 w(Comm)g(comm\))75 2165 y(MPI)p 150 2165 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 2222 y()f(BUF\(*\))170 2278 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 2365 y FD(There)16 b(is)g(only)g(one)g(receiv)o(e)h(mo)q(de,)e(whic)o (h)i(can)f(matc)o(h)f(an)o(y)g(of)h(the)f(send)i(mo)q(des.)k(The)16 b(receiv)o(e)75 2421 y(op)q(eration)g(describ)q(ed)h(in)g(the)e(last)h (section)g(is)g Fx(blo)q(c)o(king)p FD(:)22 b(it)16 b(returns)f(only)h (after)f(the)h(receiv)o(e)g(bu\013er)75 2478 y(con)o(tains)h(the)h (newly)g(receiv)o(ed)g(message.)26 b(It)17 b(is)h Fx(async)o(hronous)p FD(:)23 b(the)17 b(completion)i(of)d(a)h(receiv)o(e)h(is)75 2534 y(a)e(lo)q(cal)i(op)q(eration,)f(and)f(a)g(receiv)o(e)i(can)f (complete)g(b)q(efore)g(the)f(matc)o(hing)h(send)g(has)f(completed)i (\(of)75 2591 y(course,)d(it)g(can)h(complete)g(only)f(after)g(the)g (matc)o(hing)g(send)h(has)f(started\).)166 2647 y(Comm)o(unication)e (imp)q(oses)g(an)f(order)h(on)f(the)h(ev)o(en)o(ts)f(o)q(ccurring)h(at) f(the)h(comm)o(unicating)g(no)q(des.)75 2704 y(It)i(is)g(alw)o(a)o(ys)e (the)i(case)g(that)e(the)i(completion)h(of)e(a)g(receiv)o(e)i(o)q (ccurs)e(after)g(the)h(start)e(of)h(the)h(matc)o(hing)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 40 33 39 bop 75 -100 a Fp(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FD(33)75 45 y(send.)29 b(If)19 b(the)f(sync)o(hronous)g(send)h(mo)q (de)f(is)h(used,)g(then)g(it)f(is)h(also)f(the)g(case)g(that)g(the)g (completion)75 102 y(of)i(a)h(send)g(o)q(ccurs)g(after)f(the)g(start)g (of)g(the)h(matc)o(hing)f(receiv)o(e.)38 b(Of)20 b(course,)i(on)f(eac)o (h)f(pro)q(cess,)i(a)75 158 y(comm)o(unication)14 b(completes)h(after)d (it)i(is)g(started.)19 b(No)13 b(other)h(order)f(is)h(imp)q(osed)h(b)o (y)e(MPI.)g(E.g.,)g(if)h(the)75 214 y(standard)h(send)g(mo)q(de)h(is)f (used,)h(the)f(send)h(op)q(eration)f(ma)o(y)f(complete)i(b)q(efore)g (the)f(matc)o(hing)g(receiv)o(e)75 271 y(w)o(as)f(started.)166 329 y(In)h(a)f(m)o(ulti-threaded)i(implemen)o(tation)g(of)e(MPI,)h(the) f(system)g(ma)o(y)g(desc)o(hedule)j(a)d(thread)h(that)75 385 y(is)21 b(blo)q(c)o(k)o(ed)g(on)g(a)f(send)h(or)e(receiv)o(e)j(op)q (eration,)f(and)g(sc)o(hedule)h(another)e(thread)g(for)g(execution)h (in)75 442 y(the)d(same)g(address)g(space.)28 b(In)19 b(suc)o(h)f(a)f(case)h(it)h(the)f(user)g(resp)q(onsibilit)o(y)i(not)e (to)f(access)h(or)f(mo)q(dify)75 498 y(a)f(comm)o(unication)h(bu\013er) f(un)o(til)i(the)e(comm)o(unication)h(completes;)g(otherwise)g(the)f (outcome)g(of)g(the)75 555 y(computation)f(is)h(unde\014ned.)189 668 y Fo(R)n(ationale.)61 b FD(W)l(e)19 b(prohibit)h(read)f(accesses)g (to)f(a)g(send)i(bu\013er)e(while)j(it)e(is)g(b)q(een)h(used,)g(ev)o (en)189 724 y(though)e(the)g(send)g(op)q(eration)h(is)f(not)g(supp)q (osed)h(to)e(alter)h(the)g(con)o(ten)o(t)g(of)f(this)i(bu\013er.)28 b(This)189 781 y(ma)o(y)17 b(seem)h(more)f(stringen)o(t)g(than)h (necessary)l(,)g(but)g(the)g(additional)h(restriction)g(causes)f (little)189 837 y(loss)g(of)g(functionalit)o(y)i(and)e(allo)o(ws)h(b)q (etter)f(p)q(erformance)h(on)f(some)g(systems)g({)h(consider)g(the)189 893 y(case)d(where)g(data)g(transfer)f(is)i(done)f(b)o(y)h(a)e(DMA)h (engine)h(whic)o(h)g(is)g(not)f(cac)o(he-coheren)o(t)h(with)189 950 y(the)e(main)h(pro)q(cessor.)j(\()p Fo(End)d(of)g(r)n(ationale.)p FD(\))166 1139 y Fl(Implemen)o(tati)o(on)10 b(note:)34 b Ft(A)13 b(ready)f(send)h(can)f(b)q(e)h(implemen)o(ted)d(as)i(a)g (standard)h(send;)g(in)e(suc)o(h)i(a)f(case)75 1189 y(there)j(will)e(b) q(e)h(no)g(p)q(erformance)g(adv)n(an)o(tage)f(\(or)h(disadv)n(an)o (tage\))f(for)g(the)i(use)f(of)g(ready)g(send.)166 1240 y(A)f(standard)h(send)h(can)e(b)q(e)h(implemen)o(ted)e(as)h(a)g(sync)o (hronous)i(send.)j(In)c(suc)o(h)g(a)f(case,)h(no)f(data)h(bu\013ering) 75 1290 y(is)g(needed.)166 1341 y(A)g(p)q(ossible)g(comm)o(unication)c (proto)q(col)k(for)g(the)g(v)n(arious)f(comm)o(unication)e(mo)q(des)i (is)h(outlined)f(b)q(elo)o(w:)166 1392 y Fv(ready)i(send)p Ft(:)k(The)14 b(message)g(is)g(sen)o(t)h(as)e(so)q(on)h(as)g(p)q (ossible.)166 1443 y Fv(synchronous)f(send:)18 b Ft(The)12 b(sender)h(sends)f(a)f(request-to-send)i(message.)k(The)12 b(receiv)o(er)h(stores)f(this)g(request.)75 1493 y(When)j(a)f(matc)o (hing)f(receiv)o(e)j(is)f(p)q(osted,)g(the)h(receiv)o(er)g(sends)g(bac) o(k)f(a)f(p)q(ermission-to-send)h(message,)f(and)g(the)75 1543 y(sender)h(no)o(w)f(sends)h(the)g(message.)166 1594 y Fv(standa)o(rd)k(send:)27 b Ft(First)18 b(proto)q(col)g(ma)o(y)e(b)q (e)i(used)h(for)e(short)i(messages,)f(and)g(second)h(proto)q(col)e(for) h(long)75 1644 y(messages.)166 1695 y(Additional)d(con)o(trol)h (messages)h(migh)o(t)e(b)q(e)i(needed)h(for)e(\015o)o(w)g(con)o(trol)h (and)f(error)i(reco)o(v)o(ery)m(.)27 b(Of)16 b(course,)75 1745 y(there)f(are)f(man)o(y)f(other)h(p)q(ossible)g(proto)q(cols.)75 1978 y Fq(3.8)59 b(Nonblo)r(cking)20 b(communication)75 2083 y FD(One)e(can)g(impro)o(v)o(e)f(p)q(erformance)g(on)h(man)o(y)e (systems)h(b)o(y)h(o)o(v)o(erlapping)f(comm)o(unication)i(and)e(com-)75 2139 y(putation.)35 b(This)21 b(is)g(esp)q(ecially)i(true)d(on)g (systems)g(where)g(comm)o(unication)h(can)g(b)q(e)g(executed)g(au-)75 2195 y(tonomously)12 b(b)o(y)h(an)g(in)o(telligen)o(t)h(comm)o (unication)g(con)o(troller.)19 b(Ligh)o(t-w)o(eigh)o(t)13 b(threads)f(are)h(one)g(mec)o(h-)75 2252 y(anism)21 b(for)g(ac)o (hieving)h(suc)o(h)g(o)o(v)o(erlap.)37 b(An)21 b(alternativ)o(e)h(mec)o (hanism)g(that)e(often)h(leads)g(to)g(b)q(etter)75 2308 y(p)q(erformance)e(is)g(to)f(use)h Fx(non)o(blo)q(c)o(king)j(comm)o (unication)p FD(.)32 b(A)19 b(non)o(blo)q(c)o(king)h(send)f(call)h (initiates)75 2365 y(the)15 b(send)h(op)q(eration,)f(but)h(do)q(es)f (not)g(complete)h(it.)k(The)15 b(send)h(call)h(will)f(return)g(b)q (efore)f(the)g(message)75 2421 y(w)o(as)d(copied)h(out)f(of)g(the)g (send)h(bu\013er.)19 b(A)13 b(separate)e(call)j(is)f(needed)g(to)f (complete)h(the)g(comm)o(unication,)75 2478 y(i.e.)20 b(to)13 b(v)o(erify)i(that)e(the)h(data)g(has)g(b)q(een)h(copied)g(out) f(of)g(the)g(send)h(bu\013er.)k(With)c(suitable)g(hardw)o(are,)75 2534 y(the)k(transfer)f(of)h(data)f(out)h(of)g(the)g(sender)g(memory)g (ma)o(y)f(pro)q(ceed)i(concurren)o(tly)g(with)f(computa-)75 2591 y(tions)g(done)g(at)f(the)g(sender)h(after)f(the)h(send)g(w)o(as)f (initiated)i(and)e(b)q(efore)h(it)g(completed.)31 b(Similarly)l(,)75 2647 y(a)19 b(non)o(blo)q(c)o(king)h(receiv)o(e)g(initiates)h(the)e (receiv)o(e)h(op)q(eration,)g(but)f(do)q(es)g(not)g(complete)h(it.)32 b(The)19 b(call)75 2704 y(will)f(return)f(b)q(efore)g(a)f(message)g(is) i(stored)e(in)o(to)g(the)h(receiv)o(e)h(bu\013er.)24 b(A)17 b(separate)f(call)h(is)h(needed)g(to)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 41 34 40 bop 75 -100 a FD(34)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FD(complete)h(the)f (receiv)o(e)i(op)q(eration)e(and)g(v)o(erify)h(that)e(the)i(data)e(has) i(b)q(een)g(receiv)o(ed)g(in)o(to)g(the)f(receiv)o(e)75 102 y(bu\013er.)k(With)14 b(suitable)h(hardw)o(are,)f(the)g(transfer)f (of)h(data)g(in)o(to)g(the)g(receiv)o(er)h(memory)f(ma)o(y)f(pro)q (ceed)75 158 y(concurren)o(tly)f(with)g(computations)f(done)g(after)g (the)g(receiv)o(e)h(w)o(as)f(initiated)i(and)e(b)q(efore)h(it)f (completed.)75 214 y(The)17 b(use)h(of)f(non)o(blo)q(c)o(king)h(receiv) o(es)g(ma)o(y)f(also)g(a)o(v)o(oid)g(system)f(bu\013ering)i(and)f (memory)g(to)g(memory)75 271 y(cop)o(ying,)e(as)g(information)g(is)h (pro)o(vided)g(early)g(on)f(the)g(lo)q(cation)h(of)f(the)g(receiv)o(e)h (bu\013er.)166 362 y(Non)o(blo)q(c)o(king)d(sends)f(can)g(use)h(the)f (same)f(three)h(mo)q(des)g(as)g(blo)q(c)o(king)h(sends:)19 b Fs(standa)o(rd)p FD(,)13 b Fs(ready)f FD(and)75 418 y Fs(synchronous)p FD(.)26 b(These)16 b(carry)g(the)h(same)f(meaning.) 24 b(The)17 b(initiation)h(and)f(completion)g(of)f(a)g(standard)75 475 y(send)k(do)f(not)g(dep)q(end)i(on)e(the)h(status)e(of)h(a)g(matc)o (hing)g(receiv)o(e.)33 b(A)20 b(ready)f(send)h(can)f(b)q(e)h(initiated) 75 531 y(only)f(if)h(a)e(matc)o(hing)h(receiv)o(e)h(has)f(already)g(b)q (een)h(initiated,)h(otherwise)e(the)g(call)h(is)g(erroneous;)g(its)75 588 y(completion)d(do)q(es)g(not)f(dep)q(end)i(on)e(the)g(status)g(of)f (a)h(matc)o(hing)h(receiv)o(e.)23 b(A)17 b(sync)o(hronous)f(send)h(can) 75 644 y(b)q(e)j(initiated)h(b)q(efore)f(a)f(matc)o(hing)h(receiv)o(e)g (has)f(b)q(een)i(initiated,)h(but)d(will)i(complete)g(only)f(after)e(a) 75 700 y(matc)o(hing)13 b(receiv)o(e)g(has)g(b)q(een)h(initiated,)g (and)f(has)g(started)f(receiving)i(the)f(message)f(generated)h(b)o(y)g (the)75 757 y(send)j(op)q(eration.)166 848 y(In)f(all)h(cases)e(the)h (op)q(eration)g(that)f(initiates)i(the)f(comm)o(unication)g(\(send)g (or)f(receiv)o(e\))i(is)f(a)f(lo)q(cal,)75 904 y(non)o(blo)q(c)o(king) 22 b(call.)39 b(If)21 b(the)g(call)h(causes)g(some)e(system)h(resource) g(to)f(b)q(e)i(exhausted,)h(then)e(it)g(will)75 961 y(fail)f(and)f (return)f(an)h(error)f(co)q(de.)31 b(Go)q(o)q(d)19 b(qualit)o(y)g (implemen)o(tations)h(of)f(MPI)g(should)g(ensure)h(that)75 1017 y(this)d(happ)q(ens)h(only)f(in)h(\\pathological")f(cases.)25 b(I.e.,)16 b(an)h(MPI)g(implemen)o(tation)h(should)g(b)q(e)f(able)h(to) 75 1073 y(supp)q(ort)d(a)g(large)g(n)o(um)o(b)q(er)h(of)f(p)q(ending)i (non)o(blo)q(c)o(king)f(op)q(erations.)166 1164 y(Non)o(blo)q(c)o(king) g(sends)g(can)f(b)q(e)h(matc)o(hed)f(with)h(blo)q(c)o(king)g(receiv)o (es,)g(and)f(vice-v)o(ersa.)75 1482 y Fn(3.8.1)49 b(Comm)o(unication)14 b(Objects)75 1634 y FD(Non)o(blo)q(c)o(king)k(comm)o(unications)g(use)f (opaque)g(request)g(ob)s(jects)f(to)h(iden)o(tify)h(comm)o(unication)g (op)q(er-)75 1691 y(ations)f(and)g(matc)o(h)f(the)i(op)q(eration)f (that)f(initiates)i(the)f(comm)o(unication)h(with)f(the)g(op)q(eration) h(that)75 1747 y(terminates)f(it.)27 b(These)17 b(are)g(system)g(ob)s (jects)g(that)f(are)h(accessed)h(via)g(a)f(handle.)27 b(A)17 b(request)h(ob)s(ject)75 1804 y(iden)o(ti\014es)j(v)m(arious)f (prop)q(erties)g(of)f(a)g(comm)o(unication)h(op)q(eration,)h(suc)o(h)e (as)g(the)h(\(send)f(or)g(receiv)o(e\))75 1860 y(bu\013er)c(that)g(is)h (asso)q(ciated)f(with)h(it,)f(its)h(con)o(text,)e(the)i(tag)e(and)i (destination)g(argumen)o(ts)f(to)f(b)q(e)i(used)75 1917 y(for)k(a)f(send,)j(or)e(the)g(tag)f(and)i(source)f(argumen)o(ts)f(to)h (b)q(e)h(used)f(for)g(a)g(receiv)o(e.)36 b(In)20 b(addition,)j(this)75 1973 y(ob)s(ject)14 b(stores)g(information)g(ab)q(out)h(the)f(status)g (of)g(the)g(p)q(ending)j(comm)o(unication)e(op)q(eration)g(that)e(is)75 2030 y(p)q(erformed)i(with)h(this)g(ob)s(ject.)75 2347 y Fn(3.8.2)49 b(Comm)o(unication)14 b(initiation)75 2500 y FD(W)l(e)21 b(use)g(the)f(same)h(naming)g(con)o(v)o(en)o(tions)f(as)h (for)f(blo)q(c)o(king)h(comm)o(unication:)32 b(a)20 b(pre\014x)h(of)f Fs(R)h FD(\()p Fs(S)p FD(\))75 2556 y(is)f(used)f(for)f Fs(READ)o(Y)i FD(\()p Fs(SYNCHRONOUS)p FD(\))h(mo)q(de.)32 b(In)20 b(addition)g(a)e(pre\014x)i(of)e Fs(I)h FD(\(for)f Fs(IMMEDIA)l(TE)p FD(\))75 2613 y(indicates)f(that)d(the)h(call)i(is)e (non)o(blo)q(c)o(king.)166 2704 y(Initiate)h(a)f(standard)g(mo)q(de)g (non)o(blo)q(c)o(king)i(send.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 42 35 41 bop 75 -100 a Fp(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FD(35)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(ISEND\(buf,)15 b(count,)h(datat)o(yp)q(e,)h(dest,)f(tag,)f(comm)m(,)d(request\))117 139 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 249 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 358 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 468 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 577 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 686 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 796 y(OUT)108 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 937 y FB(int)23 b(MPI)p 245 937 15 2 v 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 937 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 994 y(int)h(tag,)f(MPI)p 683 994 V 17 w(Comm)g(comm,)g(MPI)p 1034 994 V 17 w(Comm)p 1147 994 V 17 w(request)g(*request\))75 1097 y(MPI)p 150 1097 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(REQUEST,)e(IERROR\))170 1154 y()h(BUF\(*\))170 1210 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1314 y FD(Initiate)16 b(a)f(ready)g(mo)q(de)h(non)o(blo)q (c)o(king)g(send.)75 1435 y Fs(MPI)p 160 1435 14 2 v 16 w(IRSEND\(buf,)g(count,)g(datat)o(yp)q(e,)g(dest,)g(tag,)f(comm)m(,) d(request\))117 1529 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1638 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1748 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1857 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1967 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 2076 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 2185 y(OUT)108 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 2327 y FB(int)23 b(MPI)p 245 2327 15 2 v 17 w(Irsend\(void*)f(buf,)i (int)f(count,)g(MPI)p 1026 2327 V 17 w(Datatype)g(datatype,)f(int)i (dest,)393 2384 y(int)g(tag,)f(MPI)p 683 2384 V 17 w(Comm)g(comm,)g (MPI)p 1034 2384 V 17 w(Comm)p 1147 2384 V 17 w(request)g(*request\))75 2487 y(MPI)p 150 2487 V 17 w(IRSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,) g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2544 y()g(BUF\(*\))170 2600 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 2704 y FD(Initiate)16 b(a)f(sync)o(hronous)g(mo)q(de)h(non) o(blo)q(c)o(king)g(send.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 43 36 42 bop 75 -100 a FD(36)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(ISSEND\(buf,)f(count,)i(datat)o(yp)q(e,)f(dest,)g (tag,)f(comm)m(,)d(request\))117 127 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 212 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 297 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h (bu\013er)g(elemen)o(t)e(\(handle\))117 382 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 467 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 551 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 636 y(OUT)108 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 766 y FB(int)23 b(MPI)p 245 766 15 2 v 17 w(Issend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 766 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 822 y(int)g(tag,)f(MPI)p 683 822 V 17 w(Comm)g(comm,)g(MPI)p 1034 822 V 17 w(Comm)p 1147 822 V 17 w(request)g(*request\))75 913 y(MPI)p 150 913 V 17 w(ISSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 970 y()g(BUF\(*\))170 1026 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1118 y FD(Initiate)16 b(a)f(non)o(blo)q(c)o(king)i(receiv)o (e.)75 1226 y Fs(MPI)p 160 1226 14 2 v 16 w(IRECV)e(\(buf,)h(count,)g (datat)o(yp)q(e,)g(source,)g(tag,)e(comm)n(,)d(request\))117 1308 y Ft(OUT)108 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 1393 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1478 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(handle\))117 1563 y(IN)155 b Fs(source)468 b Ft(rank)14 b(of)f(source)j(\(in)o(teger\))117 1648 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 1733 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 1817 y(OUT)108 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 1947 y FB(int)23 b(MPI)p 245 1947 15 2 v 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1947 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 2003 y(int)h(tag,)f(MPI)p 683 2003 V 17 w(Comm)g(comm,)g(MPI)p 1034 2003 V 17 w(Comm)p 1147 2003 V 17 w(request)g(*request\))75 2094 y(MPI)p 150 2094 V 17 w(IRECV\(BUF,)f(COUNT,)h(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2151 y()g(BUF\(*\))170 2207 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 2299 y FD(These)c(calls)g(allo)q (cate)h(a)e(comm)o(unication)h(request)f(and)h(asso)q(ciate)g(it)f (with)h(the)g(handle.)31 b(The)75 2355 y(request)11 b(can)f(b)q(e)i (used)f(later)g(to)f(query)g(the)h(status)f(of)g(the)h(comm)o (unication)g(or)f(w)o(ait)g(for)g(its)h(completion.)166 2416 y(A)17 b(non)o(blo)q(c)o(king)i(send)f(call)g(indicates)h(that)e (the)g(system)g(ma)o(y)g(start)f(cop)o(ying)h(data)g(out)g(of)g(the)75 2473 y(send)g(bu\013er.)24 b(The)17 b(sender)g(should)g(not)g(access)f (an)o(y)g(part)g(of)g(the)h(send)g(bu\013er)g(after)e(a)i(non)o(blo)q (c)o(king)75 2529 y(send)f(op)q(eration)f(is)h(called,)g(un)o(til)h (the)e(send)h(completes.)166 2591 y(A)d(non)o(blo)q(c)o(king)h(receiv)o (e)g(call)g(indicates)g(that)e(the)h(system)f(ma)o(y)g(start)g(writing) h(data)f(in)o(to)h(the)g(re-)75 2647 y(ceiv)o(e)f(bu\013er.)19 b(The)11 b(receiv)o(er)h(should)h(not)e(access)g(an)o(y)g(part)g(of)g (the)g(receiv)o(e)i(bu\013er)e(after)g(a)g(non)o(blo)q(c)o(king)75 2704 y(receiv)o(e)16 b(op)q(eration)g(is)f(called,)i(un)o(til)f(the)f (receiv)o(e)h(completes.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 44 37 43 bop 75 -100 a Fp(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FD(37)75 45 y Fn(3.8.3)49 b(Comm)o(unication)14 b(Completion)75 157 y FD(The)j(functions)g Fs(MPI)p 452 157 14 2 v 16 w(W)l(AIT)f FD(and)g Fs(MPI)p 768 157 V 16 w(TEST)h FD(are)f(used)h(to) f(complete)h(a)f(non)o(blo)q(c)o(king)h(comm)o(unica-)75 213 y(tion.)i(The)13 b(completion)h(of)e(a)g(send)h(op)q(eration)g (indicates)h(that)e(the)g(sender)h(is)g(no)o(w)f(free)h(to)f(up)q(date) h(the)75 270 y(lo)q(cations)k(in)f(the)h(send)f(bu\013er)g(\(the)g (send)g(op)q(eration)g(itself)h(lea)o(v)o(es)f(the)g(con)o(ten)o(t)g (of)f(the)h(send)h(bu\013er)75 326 y(unc)o(hanged\).)25 b(It)16 b(do)q(es)h(not)g(indicate)h(that)e(the)h(message)f(has)g(b)q (een)i(receiv)o(ed;)g(rather,)e(it)h(ma)o(y)f(ha)o(v)o(e)75 383 y(b)q(een)g(bu\013ered)g(b)o(y)f(the)h(comm)o(unication)g (subsystem.)k(Ho)o(w)o(ev)o(er,)14 b(if)i(a)f Fs(synchronous)i FD(mo)q(de)f(send)g(w)o(as)75 439 y(used,)f(the)g(completion)h(of)f (the)f(send)i(op)q(eration)f(indicates)h(that)e(a)h(matc)o(hing)g (receiv)o(e)h(w)o(as)e(initiated,)75 496 y(and)h(that)g(the)g(message)g (will)i(ev)o(en)o(tually)f(b)q(e)g(receiv)o(ed)g(b)o(y)f(this)h(matc)o (hing)f(receiv)o(e.)166 565 y(The)20 b(completion)h(of)e(a)g(receiv)o (e)i(op)q(eration)f(indicates)h(that)e(the)h(receiv)o(e)g(bu\013er)g (con)o(tains)g(the)75 622 y(receiv)o(ed)f(message,)f(and)g(that)f(the)h (status)f(ob)s(ject)h(is)g(set;)h(the)f(receiv)o(er)h(is)f(no)o(w)f (free)h(to)g(access)g(the)75 678 y(receiv)o(e)g(bu\013er.)27 b(It)17 b(do)q(es)h(not)f(indicate)i(that)d(the)i(matc)o(hing)f(send)h (op)q(eration)g(has)f(completed)h(\(but)75 735 y(indicates,)e(of)f (course,)g(that)f(the)i(send)f(w)o(as)g(initiated\).)75 852 y Fs(MPI)p 160 852 V 16 w(W)l(AIT\(request,)h(status\))117 942 y Ft(INOUT)62 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))117 1044 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\))75 1182 y FB(int)23 b(MPI)p 245 1182 15 2 v 17 w(Wait\(MPI)p 454 1182 V 16 w(Comm)p 566 1182 V 17 w(request)g(*request,)f(MPI)p 1084 1182 V 17 w(Status)h(*status\))75 1282 y(MPI)p 150 1282 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 1338 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1338 V 16 w(STATUS)p 979 1338 V 16 w(SIZE\),)g(IERROR)166 1438 y FD(A)14 b(call)h(to)f Fs(MPI)p 436 1438 14 2 v 15 w(W)l(AIT)h FD(returns)e(when)i(the)f(send) h(op)q(eration)f(iden)o(ti\014ed)i(b)o(y)e Fs(request)i FD(is)f(complete.)75 1494 y(If)21 b(the)g(comm)o(unication)h(ob)s(ject) f(asso)q(ciated)g(with)g(this)h(request)f(w)o(as)f(created)h(b)o(y)g(a) g(non)o(blo)q(c)o(king)75 1551 y(send)13 b(or)g(receiv)o(e)h(call,)g (then)f(the)g(ob)s(ject)f(is)h(deallo)q(cated)i(b)o(y)d(the)h(call)h (to)25 b Fs(MPI)p 1431 1551 V 16 w(W)l(AIT)13 b FD(and)g(the)g(request) 75 1607 y(b)q(ecomes)j(n)o(ull.)189 1780 y Fo(R)n(ationale.)39 b FD(MPI)15 b(alw)o(a)o(ys)f(n)o(ulli\014es)j(handles)f(to)f(ob)s (jects)f(that)g(ha)o(v)o(e)g(b)q(een)i(deallo)q(cated,)g(so)e(as)189 1836 y(to)g(a)o(v)o(oid)h(dangling)h(references.)21 b(\()p Fo(End)16 b(of)g(r)n(ationale.)p FD(\))166 2009 y(The)h(call)g(returns) g(in)g Fs(status)h FD(information)f(on)f(the)h(completed)g(op)q (eration.)24 b(The)17 b(status)f(ob)s(ject)75 2066 y(for)g(a)h(receiv)o (e)h(op)q(eration)g(can)f(b)q(e)h(queried)g(using)g(the)f(functions)h (describ)q(ed)h(in)f(section)g(3.3.1.)24 b(The)95 b Fm(CHANGE)75 2122 y FD(status)15 b(ob)s(ject)f(for)h(a)g(send)i(op)q(eration)e(ma)o (y)g(b)q(e)h(queried)h(as)e(w)o(ell)h(b)o(y)g(a)f(call)h(to)f Fs(MPI)p 1559 2122 V 16 w(TEST)p 1689 2122 V 17 w(CANCEL)75 2179 y FD(\(Section)c(3.9,)f(b)q(elo)o(w\).)19 b(Th)o(us,)11 b(one)g(alw)o(a)o(ys)f(need)i(to)e(pro)o(vide)h(a)f(v)m(alid)i(status)e (argumen)o(t)g(to)g Fs(MPI)p 1749 2179 V 16 w(W)l(AIT)75 2235 y FD(and)15 b(the)h(other)e(related)i(functions.)166 2305 y(It)11 b(is)h(allo)o(w)o(ed)g(to)f(call)h Fs(MPI)p 627 2305 V 16 w(W)l(AIT)f FD(with)h(a)f(n)o(ull)i Fs(request)g FD(argumen)o(t,)e(in)h(whic)o(h)g(case)f(the)h(op)q(eration)95 b Fm(CHANGE)75 2361 y FD(returns)17 b(immediately;)k(the)c Fs(status)j FD(argumen)o(t)d(is)h(set)f(to)g(return)g Fs(tag)h(=)g(MPI)p 1464 2361 V 16 w(ANY)p 1572 2361 V 17 w(T)l(A)o(G)p FD(,)g Fs(source)g(=)75 2418 y(MPI)p 160 2418 V 16 w(ANY)p 268 2418 V 17 w(SOURCE)p FD(,)e(and)g Fs(count)g(=)g(0)p FD(.)189 2591 y Fo(R)n(ationale.)52 b FD(This)18 b(mak)o(es)e Fs(MPI)p 772 2591 V 16 w(W)l(AIT)h FD(functionally)i(equiv)m(alen)o(t)g(to)e Fs(MPI)p 1528 2591 V 16 w(W)l(AIT)l(ALL)g FD(with)g(a)189 2647 y(list)e(of)f(length)h (one)f(and)h(adds)f(some)g(elegance.)21 b(Status)14 b(is)h(set)f(so)g (as)g(to)g(prev)o(en)o(t)g(errors)g(due)h(to)189 2704 y(access)g(to)g(stale)g Fs(status)i FD(information.)j(\()p Fo(End)c(of)g(r)n(ationale.)p FD(\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 45 38 44 bop 75 -100 a FD(38)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(TEST\(request,)g(\015ag,)f(status\))117 126 y Ft(INOUT)62 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))117 209 y(OUT)108 b Fs(\015ag)518 b Ft(\(logical\))117 291 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\))75 420 y FB(int)23 b(MPI)p 245 420 15 2 v 17 w(Test\(MPI)p 454 420 V 16 w(Comm)p 566 420 V 17 w(request)g(*request,)f(int)i(flag,)f(MPI)p 1323 420 V 17 w(Status)g(*status\))75 510 y(MPI)p 150 510 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 566 y(LOGICAL)g(FLAG)170 623 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 623 V 16 w(STATUS)p 979 623 V 16 w(SIZE\),)g(IERROR)166 713 y FD(A)17 b(call)i(to)e Fs(MPI)p 446 713 14 2 v 15 w(TEST)h FD(returns)f Fs(\015ag)h(=)f(true)h FD(if)g(the)g(op)q (eration)f(iden)o(ti\014ed)j(b)o(y)d Fs(request)i FD(is)f(com-)75 769 y(plete.)37 b(In)21 b(suc)o(h)g(a)f(case,)i(the)e(status)g(ob)s (ject)g(is)h(set)f(to)g(con)o(tain)h(information)g(on)f(the)h (completed)75 826 y(op)q(eration;)15 b(if)h(the)f(comm)o(unication)h (ob)s(ject)e(w)o(as)h(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)h (send)g(or)f(receiv)o(e,)h(then)75 882 y(it)i(is)g(deallo)q(cated)i (and)e(the)g(request)f(b)q(ecomes)i(n)o(ull.)29 b(The)18 b(call)h(returns)f Fs(\015ag=false)p FD(,)g(otherwise.)28 b(In)75 939 y(suc)o(h)16 b(a)e(case,)h(the)h(v)m(alue)g(of)f(the)g (status)f(ob)s(ject)h(is)h(unde\014ned.)166 999 y Fs(MPI)p 251 999 V 16 w(TEST)f FD(is)h(a)f(lo)q(cal,)h(non)o(blo)q(c)o(king)g (op)q(eration.)166 1059 y(The)11 b(return)g(status)f(ob)s(ject)h(for)f (a)h(receiv)o(e)h(op)q(eration)f(carries)g(information)g(that)g(can)g (b)q(e)g(accessed)75 1116 y(using)i(the)g(functions)g(describ)q(ed)i (in)e(section)g(3.3.1.)18 b(The)13 b(status)e(ob)s(ject)h(for)g(a)h (send)g(op)q(eration)g(carries)-2120 b Fm(CHANGE)75 1172 y FD(information)21 b(that)f(can)h(b)q(e)g(accessed)g(b)o(y)g(a)g(call) g(to)g Fs(MPI)p 1120 1172 V 15 w(TEST)p 1249 1172 V 17 w(CANCEL)g FD(\(Section)g(3.9,)g(b)q(elo)o(w\).)75 1229 y(Th)o(us,)h(one)f(alw)o(a)o(ys)f(need)i(to)e(pro)o(vide)i(a)e(v)m (alid)j(status)d(argumen)o(t)g(to)g Fs(MPI)p 1445 1229 V 16 w(TEST)h FD(and)g(the)g(other)75 1285 y(related)16 b(functions.)166 1345 y(It)11 b(is)h(allo)o(w)o(ed)g(to)e(call)j Fs(MPI)p 627 1345 V 15 w(TEST)f FD(with)f(a)g(n)o(ull)i Fs(request)g FD(argumen)o(t,)e(in)h(whic)o(h)g(case)f(the)h(op)q (eration)-2120 b Fm(CHANGE)75 1402 y FD(returns)17 b Fv(\015ag=true)p FD(,)h(and)f(the)g Fs(status)i FD(argumen)o(t)d(is)h (set)g(to)f(return)h Fs(tag)f(=)h(MPI)p 1465 1402 V 16 w(ANY)p 1573 1402 V 17 w(T)l(A)o(G)p FD(,)g Fs(source)h(=)75 1458 y(MPI)p 160 1458 V 16 w(ANY)p 268 1458 V 17 w(SOURCE)p FD(,)e(and)g Fs(count)g(=)g(0)p FD(.)166 1519 y(In)f(a)f(m)o (ulti-threaded)i(en)o(vironmen)o(t,)e(the)g(use)h(of)f(a)g(blo)q(c)o (king)i(receiv)o(e)f(op)q(eration)g(\()f Fs(MPI)p 1732 1519 V 15 w(W)l(AIT)p FD(\))75 1575 y(ma)o(y)k(allo)o(w)i(the)f(op)q (erating)g(system)g(to)g(de-sc)o(hedule)i(the)e(blo)q(c)o(k)o(ed)h (thread)f(and)h(sc)o(hedule)g(another)75 1631 y(thread)j(for)g (execution,)j(if)e(suc)o(h)g(is)f(a)o(v)m(ailable.)47 b(The)23 b(use)h(of)f(a)g(non)o(blo)q(c)o(king)h(receiv)o(e)h(op)q (eration)75 1688 y(\()p Fs(MPI)p 178 1688 V 15 w(TEST)p FD(\))e(allo)o(ws)f(the)g(user)g(to)g(sc)o(hedule)i(alternativ)o(e)e (activities)i(within)f(a)f(single)h(thread)f(of)75 1744 y(execution.)166 1805 y(Example:)75 1917 y FB(CALL)h (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1974 y(IF\(rank.EQ.0\))123 2030 y(THEN)170 2087 y(CALL)h(MPI_ISEND\(a\(1\),)e(10,)h(MPI_REAL,)g (1,)g(tag,)h(comm,)f(request,)g(ierr\))170 2143 y(****)h(do)f(some)h (computation)e(to)i(mask)f(latency)g(****)170 2200 y(CALL)h (MPI_WAIT\(request,)d(status,)i(ierr\))123 2256 y(ELSE)170 2312 y(CALL)h(MPI_IRECV\(a\(1\),)e(15,)h(MPI_REAL,)g(0,)g(tag,)h(comm,) f(request,)g(ierr\))170 2369 y(****)h(do)f(some)h(computation)e(to)i (mask)f(latency)g(****)170 2425 y(CALL)h(MPI_WAIT\(request,)d(status,)i (ierr\))75 2482 y(END)g(IF)166 2591 y FD(The)f(functions)h Fs(MPI)p 554 2591 V 15 w(W)l(AIT)f FD(and)g Fs(MPI)p 881 2591 V 16 w(TEST)g FD(can)g(b)q(e)g(used)h(to)e(complete)h(b)q(oth) g(sends)h(and)75 2647 y(receiv)o(es;)14 b(they)f(are)f(also)h(used)g (to)f(complete)h(an)o(y)g(other)f(non)o(blo)q(c)o(king)i(comm)o (unication)f(call)h(pro)o(vided)75 2704 y(b)o(y)h(MPI.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 46 39 45 bop 75 -100 a Fp(3.8.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b FD(39)75 45 y Fn(3.8.4)49 b(Semantics)13 b(of)k(Nonblo)q(cking)h (Comm)o(unications)75 131 y FD(The)11 b(seman)o(tics)f(of)h(non)o(blo)q (c)o(king)g(comm)o(unication)h(is)f(de\014ned)h(b)o(y)e(suitably)i (extending)g(the)e(de\014nitions)75 188 y(in)16 b(section)g(3.4)e(for)h (order)g(and)g(progress.)166 244 y(Non)o(blo)q(c)o(king)k(comm)o (unication)f(op)q(erations)g(are)g(ordered)g(according)g(to)f(the)h (execution)h(order)75 301 y(of)h(the)g(calls)h(that)e(initiate)j(the)e (comm)o(unication;)i(this)f(de\014nes)g(the)f(meaning)h(of)e(non-o)o(v) o(ertaking)75 357 y(comm)o(unication.)166 414 y(Example:)75 520 y FB(CALL)k(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 577 y(IF)h(\(RANK.EQ.0\))147 633 y(THEN)218 690 y(CALL)f(MPI_ISEND\(a,) g(1,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(h1,)h(ierr\))218 746 y(CALL)f(MPI_ISEND\(b,)g(1,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(h2,)h (ierr\))147 803 y(ELSE)94 b(!)24 b(rank.EQ.1)218 859 y(CALL)f(MPI_IRECV\(a,)g(1,)g(MPI_REAL,)g(0,)h(MPI_ANY_TAG,)e(comm,)h (h1,)g(ierr\))218 915 y(CALL)g(MPI_IRECV\(b,)g(1,)g(MPI_REAL,)g(0,)h (0,)f(comm,)g(h2,)h(ierr\))75 972 y(END)f(IF)75 1028 y(CALL)g(MPI_WAIT\(h1,status\))75 1085 y(CALL)g(MPI_WAIT\(h2,status\)) 166 1191 y FD(The)17 b(\014rst)f(send)h(of)f(pro)q(cess)h(zero)g(will)h (matc)o(h)e(the)h(\014rst)f(receiv)o(e)i(of)e(pro)q(cess)h(one,)g(ev)o (en)g(if)g(b)q(oth)75 1248 y(messages)e(are)h(sen)o(t)f(b)q(efore)h (pro)q(cess)g(one)g(executes)h(either)f(receiv)o(e.)22 b(This)17 b(precludes)g(an)f(implemen-)75 1304 y(tation)f(where)g (di\013eren)o(t)h(queues)g(are)f(used)g(to)g(p)q(ost)g(the)g(t)o(w)o(o) f(receiv)o(es.)166 1361 y(A)22 b(call)g(to)f Fs(MPI)p 458 1361 14 2 v 16 w(W)l(AIT)h FD(that)f(completes)h(a)f(receiv)o(e)i (will)g(ev)o(en)o(tually)g(terminate)e(and)h(return)75 1417 y(if)f(a)f(matc)o(hing)g(send)h(has)f(b)q(een)h(started,)g(unless) g(the)g(send)g(is)f(satis\014ed)h(b)o(y)f(another)g(receiv)o(e.)36 b(In)75 1474 y(particular,)20 b(if)f(the)g(matc)o(hing)g(send)g(is)g (non)o(blo)q(c)o(king,)i(then)e(the)g(receiv)o(e)g(should)h(complete)f (ev)o(en)g(if)75 1530 y(no)c(call)i(is)e(executed)i(b)o(y)e(the)g (sender)h(to)f(complete)h(the)f(send.)20 b(Similarly)l(,)e(a)d(call)h (to)f Fs(MPI)p 1651 1530 V 15 w(W)l(AIT)h FD(that)75 1587 y(completes)i(a)e(send)i(will)h(ev)o(en)o(tually)f(return)f(if)g (a)g(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)h(started,)f(unless)h (the)75 1643 y(receiv)o(e)e(is)g(satis\014ed)f(b)o(y)g(another)g(send,) g(and)g(ev)o(en)h(if)f(no)g(call)h(is)g(executed)g(to)e(complete)i(the) f(receiv)o(e.)166 1700 y(Example:)75 1806 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1863 y(IF)h(\(RANK.EQ.0\)) 147 1919 y(THEN)218 1976 y(CALL)f(MPI_ISEND\(a,)g(1,)g(MPI_REAL,)g(1,)h (0,)f(comm,)g(h,)h(ierr\))218 2032 y(CALL)f(MPI_RECV\(b,)g(1,)g (MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 2089 y(CALL)g(MPI_WAIT\(h,)g (status,)g(ierr\))147 2145 y(ELSE)94 b(!)24 b(rank.EQ.1)218 2202 y(CALL)f(MPI_RECV\(a,)g(1,)g(MPI_REAL,)g(0,)h(0,)f(comm,)g(ierr\)) 218 2258 y(CALL)g(MPI_SEND\(b,)g(1,)g(MPI_REAL,)g(0,)h(0,)f(comm,)g (ierr\))75 2315 y(END)g(IF)166 2421 y FD(This)16 b(co)q(de)f(should)i (not)d(deadlo)q(c)o(k)j(in)f(a)f(correct)f(MPI)h(implemen)o(tation.)166 2478 y(If)j Fs(MPI)p 299 2478 V 16 w(TEST)g FD(that)e(completes)j(a)e (receiv)o(e)i(is)f(rep)q(eatedly)h(called)g(with)f(the)g(same)f (argumen)o(ts,)75 2534 y(and)d(a)g(matc)o(hing)g(send)g(has)g(b)q(een)h (started,)e(then)h(the)g(call)h(will)g(ev)o(en)o(tually)g(return)f Fs(\015ag=true)p FD(,)h(unless)75 2591 y(the)i(send)g(is)g(satis\014ed) h(b)o(y)f(another)f(receiv)o(e.)25 b(If)17 b Fs(MPI)p 1020 2591 V 16 w(TEST)g FD(that)f(completes)i(a)e(send)i(is)f(rep)q (eatedly)75 2647 y(called)e(with)f(the)g(same)f(argumen)o(ts,)g(and)h (a)f(matc)o(hing)g(receiv)o(e)i(has)e(b)q(een)i(started,)e(then)h(the)g (call)g(will)75 2704 y(ev)o(en)o(tually)i(return)f Fs(\015ag=true)p FD(,)h(unless)g(the)g(receiv)o(e)g(is)f(satis\014ed)h(b)o(y)f(another)g (send.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 47 40 46 bop 75 -100 a FD(40)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FD(Non)o(blo)q(c)o (king)f(comm)o(unications)g(cannot)f(b)q(e)g(replaced)i(b)o(y)e(blo)q (c)o(king)h(comm)o(unications,)g(ev)o(en)f(in)75 102 y(the)h Fs(synchronous)i FD(comm)o(unication)f(mo)q(de.)k(Consider)c (the)f(follo)o(wing)h(example:)75 202 y FB(CALL)23 b (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 258 y(IF\(rank.EQ.0\))123 315 y(THEN)170 371 y(CALL)h(MPI_ISEND\(a\(1\),)e(count,)h(MPI_REAL,)f (1,)i(tag1,)f(comm,)g(request1,)g(ierr\))170 428 y(CALL)h (MPI_ISEND\(a\(2\),)e(count,)h(MPI_REAL,)f(1,)i(tag2,)f(comm,)g (request2,)g(ierr\))170 484 y(CALL)h(MPI_WAIT\(request1,)d(status,)i (ierr\))170 540 y(CALL)h(MPI_WAIT\(request2,)d(status,)i(ierr\))123 597 y(ELSE)170 653 y(CALL)h(MPI_IRECV\(a\(2\),)e(count,)h(MPI_REAL,)f (0,)i(tag2,)f(comm,)g(request2,)g(ierr\))170 710 y(CALL)h (MPI_IRECV\(a\(1\),)e(count,)h(MPI_REAL,)f(0,)i(tag1,)f(comm,)g (request1,)g(ierr\))170 766 y(CALL)h(MPI_WAIT\(request2,)d(status,)i (ierr\))170 823 y(CALL)h(MPI_WAIT\(request1,)d(status,)i(ierr\))75 879 y(END)g(IF)166 978 y FD(The)e(only)g(resource)g(requiremen)o(t)h (for)e(this)h(co)q(de)h(to)e(complete)h(is)h(the)f(abilit)o(y)h(to)e (ha)o(v)o(e)g(t)o(w)o(o)75 1035 y(p)q(ending)14 b(comm)o(unications.)19 b(If)13 b(one)f(replaces)h(the)f(non)o(blo)q(c)o(king)i(sends)f (\(receiv)o(es\))f(b)o(y)g(blo)q(c)o(king)h(sends)75 1091 y(\(receiv)o(es\),)18 b(and)f(deletes)h(the)g(calls)g(to)f Fs(MPI)p 861 1091 14 2 v 16 w(W)l(AIT)p FD(,)g(then)g(the)h(program)e (can)h(complete)h(only)g(if)g(the)75 1148 y(system)d(bu\013ers)h(at)e (least)i Fs(count)h FD(real)f(n)o(um)o(b)q(ers.)21 b(The)15 b(same)h(happ)q(ens)g(if)g(one)g(deletes)g(the)g(calls)h(that)75 1204 y(initiate)k(comm)o(unication)f(and)g(replaces)g(eac)o(h)g(call)g (to)f Fs(MPI)p 1165 1204 V 16 w(W)l(AIT)g FD(b)o(y)h(the)f(appropriate) h(blo)q(c)o(king)75 1260 y(send)c(or)e(receiv)o(e.)75 1389 y Fn(3.8.5)49 b(Multiple)17 b(Completions)75 1478 y FD(It)23 b(is)h(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)g(to)e(w)o (ait)h(for)g(the)g(completion)i(of)d(an)o(y)h(or)g(all)h(the)g(op)q (erations)f(in)75 1534 y(a)g(set,)h(rather)e(than)h(ha)o(ving)g(to)f(w) o(ait)h(for)f(a)g(sp)q(eci\014c)j(message.)43 b(A)23 b(call)h(to)e Fs(MPI)p 1597 1534 V 16 w(W)l(AIT)l(ANY)h FD(or)75 1591 y Fs(MPI)p 160 1591 V 16 w(TEST)l(ANY)e FD(can)f(b)q(e)g(used)g(to)f(w)o(ait)h(for)f(the)h(completion)g(of)g (one)g(out)f(of)g(sev)o(eral)h(op)q(erations;)75 1647 y(a)14 b(call)h(to)27 b Fs(MPI)p 347 1647 V 15 w(W)l(AIT)l(ALL)14 b FD(or)f Fs(MPI)p 701 1647 V 16 w(TEST)l(ALL)h FD(can)g(b)q(e)g(used)h (to)e(w)o(ait)g(for)h(all)g(p)q(ending)i(op)q(erations)e(in)75 1704 y(a)h(list.)75 1809 y Fs(MPI)p 160 1809 V 16 w(W)l(AIT)l(ANY)h (\(count,)g(a)o(rra)o(y)p 640 1809 V 14 w(of)p 691 1809 V 16 w(requests,)h(index,)f(status\))117 1887 y Ft(IN)155 b Fs(count)482 b Ft(list)14 b(length)f(\(in)o(teger\))117 1965 y(INOUT)62 b Fs(a)o(rra)o(y)p 416 1965 V 15 w(of)p 468 1965 V 16 w(requests)272 b Ft(arra)o(y)14 b(of)f(requests)j(\(arra) o(y)e(of)f(handles\))117 2042 y(OUT)108 b Fs(index)489 b Ft(index)12 b(of)f(handle)g(for)g(op)q(eration)h(that)f(completed)g (\(in)o(teger\))117 2120 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\))75 2246 y FB(int)23 b(MPI)p 245 2246 15 2 v 17 w(Waitany\(int)g(count,)g(MPI)p 788 2246 V 16 w(Comm)p 900 2246 V 17 w(request)g(*array)p 1252 2246 V 16 w(of)p 1316 2246 V 17 w(requests,)g(int)g(*index,)393 2302 y(MPI)p 468 2302 V 17 w(Status)g(*status\))75 2390 y(MPI)p 150 2390 V 17 w(WAITANY\(COUNT,)f(ARRAY)p 645 2390 V 16 w(OF)p 709 2390 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g (IERROR\))170 2446 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2446 V 17 w(OF)p 716 2446 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 2446 V 16 w(STATUS)p 1610 2446 V 16 w(SIZE\),)170 2503 y(IERROR)166 2591 y FD(Blo)q(c)o(ks)17 b(un)o(til)h(one)e(of)g (the)h(op)q(erations)f(asso)q(ciated)h(with)g(the)f(comm)o(unication)h (requests)g(in)g(the)75 2647 y(arra)o(y)c(has)h(completed.)20 b(Returns)15 b(in)g Fs(index)g FD(the)f(index)h(of)f(that)f(request)h (in)h(the)f(arra)o(y)f(and)h(returns)g(in)75 2704 y Fs(status)k FD(the)e(status)f(of)g(the)h(completing)h(comm)o(unication.)23 b(\(The)16 b(arra)o(y)e(is)j(indexed)g(from)e(zero,)h(b)q(oth)-2120 b Fm(CHANGE)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 48 41 47 bop 75 -100 a Fp(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FD(41)75 45 y(in)16 b(C)f(and)g(F)l(ortran.\))166 118 y(The)k(execution)h(of)e Fs(MPI)p 610 118 14 2 v 16 w(W)l(AIT)l (ANY\(count,)j(a)o(rra)o(y)p 1079 118 V 14 w(of)p 1130 118 V 16 w(requests,)h(index,)e(status\))g FD(has)f(the)f(same)75 175 y(e\013ect)10 b(as)g(the)g(execution)i(of)20 b Fs(MPI)p 655 175 V 16 w(W)l(AIT\(a)o(rra)o(y)p 897 175 V 14 w(of)p 948 175 V 16 w(requests[i],)13 b(status\))p FD(,)g(where)e Fs(i)f FD(is)h(the)f(v)m(alue)i(returned)75 231 y(b)o(y)h Fs(index)p FD(.)21 b(In)14 b(particular,)g(the)g(asso)q(ciated)f(comm)o (unication)i(ob)s(ject)d(is)i(deallo)q(cated,)h(and)f(the)g(handle)75 288 y(to)21 b(it)h(in)g Fs(a)o(rra)o(y)p 346 288 V 15 w(of)p 398 288 V 16 w(requests)i FD(is)e(set)f(to)g(n)o(ull.)41 b(If)22 b(more)f(then)h(one)g(op)q(eration)g(is)g(enabled)h(and)f(can) 75 344 y(terminate,)d(one)g(is)g(arbitrarily)g(c)o(hosen.)30 b(There)19 b(is)g(no)g(requiremen)o(t)g(that)f(the)g(c)o(hoice)i (satisfy)e(an)o(y)75 400 y(fairness)e(criterion.)166 474 y Fs(MPI)p 251 474 V 16 w(W)l(AIT)f FD(is)h(equiv)m(alen)o(t)g(to)f Fs(MPI)p 794 474 V 16 w(W)l(AIT)l(ANY)h FD(with)g(an)f(arra)o(y)f(of)g (length)i(one.)389 b Fm(CHANGE)75 594 y Fs(MPI)p 160 594 V 16 w(TEST)l(ANY\(count,)17 b(a)o(rra)o(y)p 626 594 V 14 w(of)p 677 594 V 16 w(requests,)g(index,)f(\015ag,)f(status\)) 117 688 y Ft(IN)155 b Fs(count)482 b Ft(list)14 b(length)f(\(in)o (teger\))117 796 y(INOUT)62 b Fs(a)o(rra)o(y)p 416 796 V 15 w(of)p 468 796 V 16 w(requests)272 b Ft(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 904 y(OUT)108 b Fs(index)489 b Ft(index)9 b(of)g(op)q(eration)g(that)h (completed,)f(or)g Fv(MPI)p 1651 904 13 2 v 14 w(UNDEFINED)905 961 y Ft(if)k(none)h(completed)g(\(in)o(teger\))117 1069 y(OUT)108 b Fs(\015ag)518 b Fv(TRUE)13 b Ft(if)g(one)i(of)e(the)h(op)q (erations)h(is)e(complete)h(\(logical\))117 1178 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\))75 1319 y FB(int)23 b(MPI)p 245 1319 15 2 v 17 w(Testany\(int)g(count,)g (MPI)p 788 1319 V 16 w(Comm)p 900 1319 V 17 w(request)g(*array)p 1252 1319 V 16 w(of)p 1316 1319 V 17 w(requests,)g(int)g(*index,)393 1375 y(int)h(*flag,)f(MPI)p 731 1375 V 16 w(Status)g(*status\))75 1478 y(MPI)p 150 1478 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 1478 V 16 w(OF)p 709 1478 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))170 1535 y(LOGICAL)g(FLAG)170 1591 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1591 V 17 w(OF)p 716 1591 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 1591 V 16 w(STATUS)p 1610 1591 V 16 w(SIZE\),)170 1648 y(IERROR)166 1751 y FD(Causes)11 b(either)h(one)f(or)g(none)h(of)f(the) g(op)q(erations)h(asso)q(ciated)f(with)h(the)f(comm)o(unication)h (requests)75 1807 y(to)21 b(return.)41 b(In)22 b(the)g(former)g(case,)h (it)f(returns)g Fs(\015ag)g(=)g(true)p FD(,)i(returns)e(in)h Fs(index)g FD(the)f(index)h(of)f(its)75 1864 y(request)12 b(in)h(the)f(arra)o(y)l(,)f(returns)g(in)i Fs(status)h FD(the)e(status)f(of)g(that)g(op)q(eration,)i(deallo)q(cates)g(the)f (asso)q(ciated)75 1920 y(comm)o(unication)g(ob)s(ject,)g(and)g(sets)f (the)h(request)g(to)f(n)o(ull.)20 b(\(The)11 b(arra)o(y)g(is)h(indexed) h(from)e(zero,)h(b)q(oth)g(in)96 b Fm(CHANGE)75 1977 y FD(C)12 b(and)h(F)l(ortran.\))18 b(In)13 b(the)f(latter)h(case,)f(it) h(returns)g Fs(\015ag)f(=)h(false)p FD(,)g(returns)f(a)h(v)m(alue)g(of) g Fv(MPI)p 1631 1977 13 2 v 14 w(UNDEFINED)75 2033 y FD(in)j Fs(index)g FD(and)g Fs(status)h FD(is)e(unde\014ned.)166 2106 y(The)k(execution)g(of)f Fs(MPI)p 609 2106 14 2 v 16 w(TEST)l(ANY\(count,)j(a)o(rra)o(y)p 1079 2106 V 15 w(of)p 1131 2106 V 16 w(requests,)g(index,)f(status\))g FD(has)f(the)f(same)75 2163 y(e\013ect)c(as)f(the)h(execution)h(of)e Fs(MPI)p 662 2163 V 16 w(TEST\(a)o(rra)o(y)p 905 2163 V 15 w(of)p 957 2163 V 16 w(requests[i],)j(\015ag,)d(status\))p FD(,)j(for)d Fs(i=0,)g(1)h(,...,)e(count-1)p FD(,)75 2219 y(in)i(some)e(arbitrary)h(order,)g(un)o(til)h(one)f(call)h (returns)f Fs(\015ag=true)p FD(,)h(or)e(all)i(fail.)20 b(In)14 b(the)f(former)f(case,)h Fs(index)75 2275 y FD(is)20 b(set)f(to)g(the)g(last)g(v)m(alue)i(of)e Fs(i)p FD(;)i(in)f(the)g (later)f(case,)h(it)g(is)f(set)h(to)e Fv(MPI)p 1330 2275 13 2 v 15 w(UNDEFINED)p FD(.)g(In)i(particular,)95 b Fm(CHANGE)75 2332 y Fs(MPI)p 160 2332 14 2 v 16 w(TEST)15 b FD(is)h(equiv)m(alen)o(t)h(to)d Fs(MPI)p 704 2332 V 16 w(TEST)l(ANY)j FD(with)e(an)g(arra)o(y)f(of)h(length)h(one.)166 2481 y Fl(Discussion:)33 b Fv(MPI)p 510 2481 13 2 v 14 w(TEST)p 628 2481 V 16 w(ANY)12 b Ft(pro)o(vides)h(t)o(w)o(o)g (indicators)g(of)f(success/failure:)19 b(the)14 b(v)n(alue)e(of)g Fv(\015ag)h Ft(and)95 b Fm(CHANGE)75 2531 y Ft(the)12 b(v)n(alue)g(of)f Fv(index)i Ft(\(i.e.,)e(whether)i Fv(index)g Ft(=)f Fv(MPI)p 870 2531 V 14 w(UNDEFINED)p Ft(\).)f(One)h(could)g(do)f (with)h(one)g(only;)f(e.g.,)g(get)h(rid)75 2581 y(of)i Fv(\015ag)p Ft(,)g(or)h(allo)o(w)e Fv(index)j Ft(to)e(b)q(e)i (unde\014ned)f(if)f(no)h(test)g(succeeded.)23 b(This)15 b(ma)o(y)e(buy)h(some)g(e\016ciency)h(and)g(ma)o(y)75 2631 y(sa)o(v)o(e)c(a)f(parameter,)g(at)g(the)h(exp)q(ense)i(of)c (increasing)i(the)g(di\013erence)h(b)q(et)o(w)o(een)g Fv(MPI)p 1384 2631 V 15 w(TEST)e Ft(and)h Fv(MPI)p 1666 2631 V 14 w(TEST)m(ANY)p Ft(.)166 2704 y(Opinions?)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 42 49 42 48 bop 75 -100 a FD(42)613 b Fp(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(W)l(AIT)l(ALL\()e(count,)i(a)o(rra)o(y)p 626 45 V 15 w(of)p 678 45 V 16 w(requests,)h(a)o(rra)o(y)p 971 45 V 15 w(of)p 1023 45 V 15 w(statuses\))117 124 y Ft(IN)155 b Fs(count)482 b Ft(lists)14 b(length)g(\(in)o(teger\))117 202 y(INOUT)62 b Fs(a)o(rra)o(y)p 416 202 V 15 w(of)p 468 202 V 16 w(requests)272 b Ft(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handles\))117 280 y(OUT)108 b Fs(a)o(rra)o(y)p 416 280 V 15 w(of)p 468 280 V 16 w(statuses)276 b Ft(arra)o(y)14 b(of)f(status)i(ob)r(jects)g(\(arra)o(y)f(of)f(Status\))75 406 y FB(int)23 b(MPI)p 245 406 15 2 v 17 w(Waitall\(int)g(count,)g (MPI)p 788 406 V 16 w(Comm)p 900 406 V 17 w(request)g(*array)p 1252 406 V 16 w(of)p 1316 406 V 17 w(requests,)393 462 y(MPI)p 468 462 V 17 w(Status)g(*array)p 796 462 V 16 w(of)p 860 462 V 17 w(statuses\))75 550 y(MPI)p 150 550 V 17 w(WAITALL\(COUNT,)f(ARRAY)p 645 550 V 16 w(OF)p 709 550 V 17 w(REQUESTS,)h(ARRAY)p 1085 550 V 16 w(OF)p 1149 550 V 17 w(STATUSES,)g(IERROR\))170 607 y(INTEGER)g(COUNT,)g (ARRAY)p 651 607 V 17 w(OF)p 716 607 V 17 w(REQUESTS\(*\),)170 663 y(ARRAY)p 293 663 V 17 w(OF)p 358 663 V 17 w(STATUSES\(MPI)p 663 663 V 15 w(STATUS)p 822 663 V 17 w(SIZE,*\),)g(IERROR)166 751 y FD(Blo)q(c)o(ks)12 b(un)o(til)h(all)g(comm)o(unication)g(op)q (erations)f(asso)q(ciated)g(with)g(requests)f(in)i(the)f(list)h (complete,)75 807 y(and)k(return)g(the)g(status)f(of)h(all)h(these)f (op)q(erations.)26 b(Both)16 b(arra)o(ys)g(ha)o(v)o(e)h(the)g(same)f(n) o(um)o(b)q(er)i(of)e(v)m(alid)75 864 y(en)o(tries.)24 b(The)16 b Fs(i)p FD(-th)h(en)o(try)f(in)h Fs(a)o(rra)o(y)p 694 864 14 2 v 14 w(of)p 745 864 V 16 w(statuses)i FD(is)e(set)f(to)g (the)h(return)f(status)f(of)h(the)h Fs(i)p FD(-th)f(op)q(eration.)75 920 y(All)g(comm)o(unication)g(ob)s(jects)f(are)g(deallo)q(cated)h(and) g(all)g(requests)f(are)g(set)g(to)g(n)o(ull.)166 978 y(The)f(execution)h(of)f Fs(MPI)p 596 978 V 16 w(W)l(AIT)l(ALL\(count,) g(a)o(rra)o(y)p 1046 978 V 15 w(of)p 1098 978 V 16 w(requests,)i(a)o (rra)o(y)p 1390 978 V 15 w(of)p 1442 978 V 16 w(statuses)g FD(has)e(the)g(same)75 1035 y(e\013ect)24 b(as)g(the)g(execution)h(of)f Fs(MPI)p 714 1035 V 16 w(W)l(AIT\(a)o(rra)o(y)p 956 1035 V 14 w(of)p 1007 1035 V 16 w(request[i],)k(a)o(rra)o(y)p 1331 1035 V 15 w(of)p 1383 1035 V 16 w(statuses[i]\))p FD(,)g(for)c Fs(i=0)g(,...,)75 1091 y(count-1)p FD(,)16 b(in)g(some)f(arbitrary)f(order.)75 1196 y Fs(MPI)p 160 1196 V 16 w(TEST)l(ALL\(count,)i(a)o(rra)o(y)p 613 1196 V 14 w(of)p 664 1196 V 16 w(requests,)h(\015ag,)e(a)o(rra)o(y)p 1054 1196 V 14 w(of)p 1105 1196 V 16 w(statuses\))117 1275 y Ft(IN)155 b Fs(count)482 b Ft(lists)14 b(length)g(\(in)o (teger\))117 1353 y(INOUT)62 b Fs(a)o(rra)o(y)p 416 1353 V 15 w(of)p 468 1353 V 16 w(requests)272 b Ft(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handle\))117 1431 y(OUT)108 b Fs(\015ag)518 b Ft(\(logical\))117 1509 y(OUT)108 b Fs(a)o(rra)o(y)p 416 1509 V 15 w(of)p 468 1509 V 16 w(statuses)276 b Ft(arra)o(y)14 b(of)f(status)i(ob)r(jects)g(\(arra)o (y)f(of)f(Status\))75 1635 y FB(int)23 b(MPI)p 245 1635 15 2 v 17 w(Testall\(int)g(count,)g(MPI)p 788 1635 V 16 w(Comm)p 900 1635 V 17 w(request)g(*array)p 1252 1635 V 16 w(of)p 1316 1635 V 17 w(requests,)g(int)g(*flag,)393 1691 y(MPI)p 468 1691 V 17 w(Status)g(*array)p 796 1691 V 16 w(of)p 860 1691 V 17 w(statuses\))75 1779 y(MPI)p 150 1779 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 1779 V 16 w(OF)p 709 1779 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 1779 V 16 w(OF)p 1292 1779 V 17 w(STATUSES,)g(IERROR\))170 1836 y(LOGICAL)g(FLAG)170 1892 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1892 V 17 w(OF)p 716 1892 V 17 w(REQUESTS\(*\),)170 1949 y(ARRAY)p 293 1949 V 17 w(OF)p 358 1949 V 17 w(STATUSES\(MPI)p 663 1949 V 15 w(STATUS)p 822 1949 V 17 w(SIZE,*\),)g(IERROR)166 2037 y FD(Returns)18 b Fs(\015ag)g(=)g(true)h FD(if)f(all)h(comm)o (unications)g(asso)q(ciated)f(with)g(requests)g(in)h(the)f(arra)o(y)e (ha)o(v)o(e)75 2093 y(completed.)30 b(In)19 b(this)f(case,)h(eac)o(h)f (status)g(en)o(try)g(is)g(set)g(to)g(the)g(status)g(of)g(the)g(corresp) q(onding)h(com-)75 2149 y(m)o(unication.)32 b(All)21 b(comm)o(unication)e(ob)s(jects)g(are)g(deallo)q(cated,)i(and)e(all)h (requests)f(are)g(set)f(to)h(n)o(ull.)75 2206 y(Otherwise,)k Fs(\015ag)e(=)g(false)h FD(is)f(returned,)i(no)e(request)g(is)h(mo)q (di\014ed)g(and)f(the)h(v)m(alues)g(of)f(the)g(status)75 2262 y(en)o(tries)16 b(are)e(unde\014ned.)22 b(This)16 b(is)g(a)f(non)o(blo)q(c)o(king)h(op)q(eration)g(that)e(returns)h (immediately)l(.)166 2320 y(Example:)75 2478 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2534 y(IF\(rank)g(<)h(2\)) 123 2591 y(THEN)214 b(!)24 b(rank=0)f(or)g(rank=1)170 2647 y(CALL)h(MPI_ISEND\(a,)e(n,)i(MPI_REAL,)e(2,)i(tag,)f(comm,)g (request,)g(ierr\))170 2704 y(****)h(do)f(some)h(computation)e(to)i (mask)f(latency)g(****)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 43 50 43 49 bop 75 -100 a Fp(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FD(43)170 45 y FB(CALL)24 b(MPI_WAIT\(request,)d(status,)i(ierr\))123 102 y(ELSE)214 b(!)24 b(rank=2)170 158 y(CALL)g(MPI_IRECV\(a,)e(n,)i (MPI_REAL,)e(0,)i(tag,)f(comm,)g(request_list\(0\),)f(ierr\))170 214 y(CALL)i(MPI_IRECV\(b,)e(n,)i(MPI_REAL,)e(1,)i(tag,)f(comm,)g (request_list\(1\),)f(ierr\))170 271 y(****)i(do)f(some)h(computation)e (to)i(mask)f(latency)g(****)170 327 y(CALL)h(MPI_WAITANY\(2,)e (request_list,)g(index,)h(status,)g(ierr\))170 384 y(IF\(index.EQ.0\)) 218 440 y(THEN)266 497 y(****)g(handle)g(message)g(from)g(process)g(0)h (****)266 553 y(CALL)f(MPI_WAIT\(request_list\(1\),)e(status,)i(ierr\)) 266 610 y(****)g(handle)g(message)g(from)g(process)g(1)h(****)218 666 y(ELSE)266 723 y(****)f(handle)g(message)g(from)g(process)g(1)h (****)266 779 y(CALL)f(MPI_WAIT\(request_list\(0\),)e(status,)i(ierr\)) 266 835 y(****)g(handle)g(message)g(from)g(process)g(0)h(****)170 892 y(END)g(IF)75 948 y(END)f(IF)166 1052 y FD(The)13 b(calls)h(in)o(tro)q(duced)f(in)h(this)f(subsection)h(can)f(b)q(e)g (used)g(to)f(w)o(ait)g(or)h(test)f(for)g(the)h(completion)g(of)75 1108 y(an)h(arbitrary)f(mix)h(of)f(non)o(blo)q(c)o(king)i(comm)o (unication)f(calls,)h(including)h(a)e(mix)g(of)f(sends)h(and)g(receiv)o (es)75 1164 y(and)h(of)g(an)o(y)g(additional)i(non)o(blo)q(c)o(king)f (comm)o(unication)g(calls)g(pro)o(vided)g(b)o(y)f(MPI.)166 1223 y(An)21 b Fs(a)o(rra)o(y)p 344 1223 14 2 v 14 w(of)p 395 1223 V 16 w(requests)h FD(argumen)o(t)e(that)g(is)g(passed)h(to)f (an)o(y)g(of)g(these)g(functions)h(ma)o(y)f(con)o(tain)75 1280 y(n)o(ull)e(en)o(tries.)24 b(These)17 b(en)o(tries)g(are)g (ignored)g(and)f(the)h(call)h(executes)f(as)f(if)h(it)g(w)o(as)f (supplied)j(an)d(arra)o(y)75 1336 y(of)h(requests)f(con)o(taining)i (only)g(the)f(non-n)o(ull)i(requests)e(for)f(activ)o(e)h(comm)o (unications)h(\(ho)o(w)o(ev)o(er,)e(see)75 1393 y(paragraph)f(b)q(elo)o (w)i(on)f Fs(a)o(rra)o(y)p 585 1393 V 15 w(of)p 637 1393 V 16 w(statuses)p FD(\).)24 b(As)16 b(a)g(request)g(in)h Fs(a)o(rra)o(y)p 1271 1393 V 15 w(of)p 1323 1393 V 16 w(requests)h FD(is)f(set)e(to)h(n)o(ull)h(when)75 1449 y(the)f(corresp)q(onding)h(op)q(eration)f(completes,)h(this)f(allo)o (ws)g(to)g(reuse)g(the)g(same)g(arra)o(y)e(for)i(completing)75 1506 y(in)g(successiv)o(e)f(calls)h(the)f(op)q(erations)g(asso)q (ciated)g(with)g(this)g(arra)o(y)l(.)k(Th)o(us,)14 b(the)h(last)g (example)g(can)g(b)q(e)75 1562 y(rewritten)g(as)g(follo)o(ws:)75 1739 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1795 y(IF\(rank)g(<)h(2\))123 1852 y(THEN)214 b(!)24 b(rank=0)f(or)g(rank=1)170 1908 y(CALL)h(MPI_ISEND\(a,)e(n,)i (MPI_REAL,)e(2,)i(tag,)f(comm,)g(request,)g(ierr\))170 1965 y(CALL)h(MPI_WAIT\(request,)d(status,)i(ierr\))123 2021 y(ELSE)214 b(!)24 b(rank=2)170 2078 y(CALL)g(MPI_IRECV\(a,)e(n,)i (MPI_REAL,)e(0,)i(tag,)f(comm,)g(request_list\(0\),)f(ierr\))170 2134 y(CALL)i(MPI_IRECV\(b,)e(n,)i(MPI_REAL,)e(1,)i(tag,)f(comm,)g (request_list\(1\),)f(ierr\))170 2190 y(CALL)i(MPI_WAITANY\(2,)e (request_list,)g(index,)h(status,)g(ierr\))266 2247 y(****)g(handle)g (first)g(arriving)g(message)47 b(****)170 2303 y(CALL)24 b(MPI_WAITANY\(2,)e(request_list,)g(index,)h(status,)g(ierr\))266 2360 y(****)g(handle)g(second)g(arriving)g(message)47 b(****)166 2478 y FD(A)12 b(call)i(to)d Fs(MPI)p 430 2478 V 16 w(W)l(AIT)l(ALL)h FD(and)h(a)f(successful)h(call)h(to)e Fs(MPI)p 1187 2478 V 15 w(TEST)l(ALL)h FD(will)h(set)e(all)h(status)f (en)o(tries)95 b Fm(CHANGE)75 2534 y FD(in)14 b Fs(a)o(rra)o(y)p 224 2534 V 14 w(of)p 275 2534 V 16 w(statuses)p FD(,)i(including)g (those)d(that)f(corresp)q(ond)i(to)e(n)o(ull)j(requests)e(in)h Fs(a)o(rra)o(y)p 1567 2534 V 14 w(of)p 1618 2534 V 16 w(requests)p FD(;)i(the)75 2591 y(en)o(tries)11 b(are)f(set)h(to)f(the) g(parameters)g(of)g(the)h(completing)h(request,)f(if)g(the)f(request)h (is)g(non-n)o(ull,)i(or)d(set)h(to)75 2647 y(the)g(parameters)g(of)g(a) g(\\n)o(ull)h(comm)o(unication")g(\()p Fs(tag)f FD(=)h Fv(MPI)p 1121 2647 13 2 v 14 w(ANY)p 1220 2647 V 15 w(T)m(A)o(G)p FD(,)e Fs(source)h FD(=)h Fv(MPI)p 1590 2647 V 14 w(ANY)p 1689 2647 V 15 w(SOURCE)p FD(,)75 2704 y Fs(count)k(=)g(0)p FD(\))f(if)g(the)g(request)h(is)f(n)o(ull.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 44 51 44 50 bop 75 -100 a FD(44)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FD(If)21 b Fs(MPI)p 302 45 14 2 v 16 w(W)l(AIT)l(ANY)h FD(is)f(called)h(with)f (an)g(empt)o(y)g Fs(a)o(rra)o(y)p 1144 45 V 14 w(of)p 1195 45 V 16 w(requests)i FD(argumen)o(t)d(\(i.e.,)h(an)g(arra)o(y)75 102 y(of)h(length)h(zero,)h(or)e(an)h(arra)o(y)e(with)i(all)g(en)o (tries)g(n)o(ull\))h(then)f(it)f(returns)h(immediately)h(with)f(the)75 158 y(v)m(alue)d Fv(MPI)p 273 158 13 2 v 14 w(UNDEFINED)e FD(in)i Fs(index)p FD(;)i(no)c(status)h(is)g(up)q(dated.)32 b Fs(MPI)p 1272 158 14 2 v 16 w(TEST)l(ANY)20 b FD(with)f(an)g(empt)o (y)g Fs(a)o(r-)75 214 y(ra)o(y)p 136 214 V 15 w(of)p 188 214 V 16 w(requests)25 b FD(returns)d Fv(MPI)p 619 214 13 2 v 14 w(UNDEFINED)g FD(in)h Fs(index)g FD(and)g Fs(\015ag)f(=)h(false)p FD(;)j(no)c(status)g(is)g(up)q(dated.)75 271 y Fs(MPI)p 160 271 14 2 v 16 w(W)l(AIT)l(ALL)15 b FD(with)h(an)f(empt)o(y)g Fs(a)o(rra)o(y)p 781 271 V 14 w(of)p 832 271 V 16 w(requests)j FD(argumen)o(t)c(returns)i (immediately)h(with)e(all)i(sta-)75 327 y(tuses)j(up)q(dated)g(to)f (the)h(status)e(of)i(a)f(n)o(ull)i(comm)o(unication.)34 b Fs(MPI)p 1283 327 V 15 w(TEST)l(ALL)20 b FD(with)g(an)f(empt)o(y)h Fs(a)o(r-)75 384 y(ra)o(y)p 136 384 V 15 w(of)p 188 384 V 16 w(requests)f FD(argumen)o(t)d(returns)h Fs(\015ag=true)h FD(and)f(up)q(dates)g(all)h(statuses)e(to)h(the)g(status)f(of)g(a)h(n)o (ull)75 440 y(comm)o(unication.)75 557 y Fs(Prop)q(osal)e(fo)o(r)f (additional)i(function)-245 596 y Fm(CHANGE)75 690 y Fs(MPI)p 160 690 V 16 w(TESTSOME\(incount,)h(a)o(rra)o(y)p 697 690 V 14 w(of)p 748 690 V 16 w(requests,)g(outcount,)f(a)o(rra)o(y) p 1237 690 V 15 w(of)p 1289 690 V 16 w(indices,)g(a)o(rra)o(y)p 1554 690 V 14 w(of)p 1605 690 V 16 w(statuses\))117 768 y Ft(IN)155 b Fs(incount)448 b Ft(length)14 b(of)f(arra)o(y)p 1176 768 13 2 v 15 w(of)p 1225 768 V 14 w(requests)j(\(in)o(teger\))117 840 y(INOUT)62 b Fs(a)o(rra)o(y)p 416 840 14 2 v 15 w(of)p 468 840 V 16 w(requests)272 b Ft(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handle\))117 911 y(OUT)108 b Fs(outcount)420 b Ft(n)o(um)o(b)q(er)13 b(of)h(completed)f(requests)j(\(in)o(teger\)) 117 983 y(OUT)108 b Fs(a)o(rra)o(y)p 416 983 V 15 w(of)p 468 983 V 16 w(indices)299 b Ft(arra)o(y)12 b(of)f(indices)h(of)f(op)q (erations)g(that)h(completed)f(\(arra)o(y)h(of)905 1040 y(in)o(teger\))117 1112 y(OUT)108 b Fs(a)o(rra)o(y)p 416 1112 V 15 w(of)p 468 1112 V 16 w(statuses)276 b Ft(arra)o(y)15 b(of)g(status)h(ob)r(jects)g(for)f(op)q(erations)g(that)h(completed)905 1168 y(\(arra)o(y)e(of)f(Status\))75 1293 y FB(int)23 b(MPI)p 245 1293 15 2 v 17 w(Testsome\(int)f(incount,)h(MPI)p 859 1293 V 17 w(Comm)p 972 1293 V 17 w(request)g(*array)p 1324 1293 V 16 w(of)p 1388 1293 V 17 w(requests,)393 1349 y(int)h(*outcount,)e(int)i(*array)p 994 1349 V 16 w(of)p 1058 1349 V 17 w(indices,)393 1406 y(MPI)p 468 1406 V 17 w(Status)f(*array)p 796 1406 V 16 w(of)p 860 1406 V 17 w(statuses\))75 1492 y(MPI)p 150 1492 V 17 w(TESTSOME\(INCOUNT,)e(ARRAY)p 716 1492 V 17 w(OF)p 781 1492 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 1492 V 16 w(OF)p 1459 1492 V 17 w(INDICES,)393 1549 y(ARRAY)p 516 1549 V 17 w(OF)p 581 1549 V 17 w(STATUSES,)f(IERROR\))170 1605 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 1605 V 17 w(OF)p 764 1605 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 1605 V 17 w(OF)p 1514 1605 V 17 w(INDICES\(*\),)170 1661 y(ARRAY)p 293 1661 V 17 w(OF)p 358 1661 V 17 w(STATUSES\(MPI)p 663 1661 V 15 w(STATUS)p 822 1661 V 17 w(SIZE,*\),)h(IERROR)166 1748 y FD(Returns)c(in)h Fs(outcount)h FD(the)e(n)o(um)o(b)q(er)h(of)e (requests)h(from)f(the)i(list)f Fs(a)o(rra)o(y)p 1445 1748 14 2 v 15 w(of)p 1497 1748 V 16 w(requests)i FD(that)d(ha)o(v)o(e) 75 1804 y(completed.)i(Returns)14 b(in)g(the)f(\014rst)g Fs(outcount)i FD(lo)q(cations)f(of)f(the)g(arra)o(y)f Fs(a)o(rra)o(y)p 1411 1804 V 14 w(of)p 1462 1804 V 16 w(indices)j FD(the)e(indices)i(of)75 1861 y(these)j(op)q(erations)f (\(index)h(within)h(the)e(arra)o(y)g Fs(a)o(rra)o(y)p 1003 1861 V 14 w(of)p 1054 1861 V 16 w(requests)p FD(;)j(the)e(arra)o (y)e(is)i(indexed)h(from)d(zero,)75 1917 y(b)q(oth)f(in)h(C)e(and)h(F)l (ortran\).)k(Returns)c(on)g(the)g(\014rst)f Fs(outcount)j FD(lo)q(cations)f(of)e(the)h(arra)o(y)e Fs(a)o(rra)o(y)p 1697 1917 V 15 w(of)p 1749 1917 V 16 w(status)75 1974 y FD(the)i(status)g(for)f(these)i(completed)g(op)q(erations.)166 2113 y Fl(Discussion:)32 b Ft(Alternativ)o(e)13 b(c)o(hoice:)18 b(Returns)13 b(in)f Fv(a)o(rra)o(y)p 1078 2113 13 2 v 15 w(of)p 1127 2113 V 14 w(status[i])g Ft(the)h(status)h(of)d(the)i Fv(i)p Ft(-th)g(request,)g(for)75 2169 y(eac)o(h)h(request)i(that)e (completed.)166 2308 y FD(This)i(is)f(a)g(non)o(blo)q(c)o(king)i(op)q (eration)e(that)g(returns)g(immediately)l(.)166 2365 y(The)f(execution)h(of)f Fs(MPI)p 596 2365 14 2 v 16 w(TESTSOME\(incount,)i(a)o(rra)o(y)p 1132 2365 V 14 w(of)p 1183 2365 V 16 w(requests,)h(outcount,)e(a)o(rra)o(y)p 1671 2365 V 15 w(of)p 1723 2365 V 16 w(indices,)75 2421 y(status\))k FD(has)d(the)i(same)e(e\013ect)h(as)g(the)g(execution)h (of)e Fs(MPI)p 1104 2421 V 16 w(TEST\(a)o(rra)o(y)p 1347 2421 V 14 w(of)p 1398 2421 V 16 w(requests[i],)k(\015ag,)d(status\))p FD(,)75 2478 y(for)h Fs(i=0,)g(1)h(,...,)e(count-1)p FD(,)j(in)f(some)f(arbitrary)g(order.)29 b(Whenev)o(er)19 b(a)f(call)h(succeeds,)h(the)f(v)m(alue)g(of)f Fs(i)75 2534 y FD(is)h(app)q(ended)g(to)f Fs(a)o(rra)o(y)p 489 2534 V 14 w(of)p 540 2534 V 16 w(indices)i FD(and)e(the)g(status)f(of)h (the)g(succeeding)i(op)q(eration)e(is)h(app)q(ended)h(to)75 2591 y Fs(a)o(rra)o(y)p 173 2591 V 15 w(of)p 225 2591 V 16 w(statuses)p FD(.)166 2647 y(If)c Fs(MPI)p 297 2647 V 16 w(TESTSOME)g FD(is)g(called)h(with)f(an)g(empt)o(y)f Fs(a)o(rra)o(y)p 1145 2647 V 15 w(of)p 1197 2647 V 16 w(requests)i FD(argumen)o(t)e(then)h(it)g(returns)75 2704 y(immediatelly)h(with)f Fs(outcount)h(=)f(0)p FD(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 45 52 45 51 bop 75 -100 a Fp(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FD(45)189 45 y Fo(R)n(ationale.)70 b FD(The)21 b(use)f(of)g Fs(MPI)p 785 45 14 2 v 16 w(TESTSOME)h FD(a)o(v)o(oids)f(the)h(o)o(v)o (erhead)f(of)f(rep)q(eated)i(calls)h(to)189 102 y Fs(MPI)p 274 102 V 15 w(TEST)l(ANY)14 b FD(that)f(w)o(ould)g(b)q(e)g(otherwise)g (required.)20 b(Also,)14 b(it)f(allo)o(ws)g(to)f(ac)o(hiev)o(e)h(some)g (fair-)189 158 y(ness)g(in)h(the)f(servicing)i(of)e(p)q(ending)i(comm)o (unications.)k(A)14 b(serv)o(er)e(that)h(uses)g Fs(MPI)p 1622 158 V 16 w(TESTSOME)189 214 y FD(to)k(w)o(ait)g(on)g(service)h (requests)g(from)f(all)h(its)g(clien)o(t)h(will)g(not)e(starv)o(e)f(an) o(y)i(clien)o(t.)28 b(The)17 b(formal)189 271 y(requiremen)o(t)c(is)g (that)e(is)i(a)f(comm)o(unication)h(is)g(enabled,)h(and)f Fs(MPI)p 1360 271 V 16 w(TESTSOME)g FD(is)g(rep)q(eatedly)189 327 y(called,)j(then)g(the)f(enabled)i(op)q(eration)e(will)i(ev)o(en)o (tually)f(complete.)21 b(\()p Fo(End)15 b(of)i(r)n(ationale.)p FD(\))75 506 y Fs(Alternative)f(p)o(rop)q(osal)189 612 y Fo(R)n(ationale.)441 595 y Fi(1)486 612 y FD(Some)h(applications)i (ma)o(y)d(p)q(ost)h(a)g(large)g(n)o(um)o(b)q(er)g(of)g(receiv)o(es)g (\(10's)f({)h(100's\))93 b Fm(CHANGE)189 668 y FD(and)18 b(w)o(ait)f(for)g(the)h(completion)h(of)e(an)o(y)l(.)27 b(This,)19 b(in)g(order)e(to)g(tak)o(e)g(adv)m(an)o(tage)g(of)h(ready)f (send)189 725 y(proto)q(col)k(and)g(amortize)g(for)f(load)h(im)o (balances.)39 b(The)21 b(use)g(of)g(a)f(sequen)o(tial)j(structure)d (\(an)189 781 y(arra)o(y)12 b(of)h(requests\))g(is)h(ine\016cien)o(t)i (as)d(it)h(has)f(to)g(b)q(e)h(searc)o(hed)g(sequen)o(tially)h(for)e (eac)o(h)h(call.)20 b(This)189 838 y(results)d(in)h(a)f(linear)h(o)o(v) o(erhead)e(p)q(er)i(in)o(v)o(o)q(cation)f(and)g(a)g(total)g(of)f Fw(O)q FD(\()p Fw(n)1428 821 y Fi(2)1448 838 y FD(\))g(time)i(for)e (handling)j Fw(n)189 894 y FD(comm)o(unications.)k(Instead,)17 b(if)f(the)h(list)g(of)f(requests)g(is)h(an)f(opaque)g(MPI)g(ob)s (ject,)g(then)g(more)189 951 y(e\016cien)o(t)k(data)f(structures)g(can) h(b)q(e)g(used,)h(reducing)g(the)f(o)o(v)o(erhead)f(p)q(er)h(comm)o (unication)h(to)189 1007 y Fw(O)q FD(\(log)8 b(log)g Fw(n)p FD(\),)20 b(where)g Fw(n)h FD(is)f(the)g(total)f(size)i(of)e (the)h(lists)h(\(constan)o(t)e(time)h(is)g(p)q(ossible,)j(if)d(the)189 1063 y(lists)14 b(are)g(disjoin)o(t\).)20 b(Also,)14 b(if)h(the)f(list)h(of)e(requests)h(is)h(an)f(opaque)g(ob)s(ject,)f (one)i(ma)o(y)e(reconsider)189 1120 y(the)20 b(decision)j(to)d(ha)o(v)o (e)g(functions)h(that)f(complete)h(comm)o(unications)h(n)o(ullify)g (nonp)q(ersisten)o(t)189 1176 y(requests:)d(this)d(b)q(eha)o(vior)g(is) g(essen)o(tial)g(only)f(for)g(arra)o(y)f(of)h(requests.)189 1272 y(An)d(in)o(terface)h(whic)o(h)g(allo)o(ws)g(incremen)o(tal)g(mo)q (di\014cations)h(to)e(these)g(lists)h(of)f(requests)h(is)g(prefer-)189 1329 y(able)j(to)e(rep)q(eated)i(uses)g(of)e(a)h(commit-lik)o(e)i (function)f(on)f(a)g(static)g(arra)o(y)f(since)j(these)e(lists)h(will) 189 1385 y(c)o(hange)11 b(gradually)g(in)h(man)o(y)e(applications)j (\(e.g.)k(n-b)q(o)q(dy)12 b(tree)f(algorithm\))g(\()p Fo(End)g(of)i(r)n(ationale.)p FD(\))166 1543 y(A)23 b(new)f(opaque)h (ob)s(ject,)h(a)e Fv(collection)i FD(is)f(added)g(to)f(MPI.)g(The)h (functions)h Fs(MPI)p 1648 1543 V 15 w(W)l(AIT)l(ANY)p FD(,)75 1599 y Fs(MPI)p 160 1599 V 16 w(W)l(AIT)l(ALL)p FD(,)17 b Fs(MPI)p 477 1599 V 16 w(TEST)l(ANY)j FD(and)e Fs(MPI)p 888 1599 V 16 w(TEST)l(ALL)g FD(accept)h(a)f Fs(collection)g FD(argumen)o(t,)g(whic)o(h)h(is)75 1655 y(a)i(handle)i(to)e(a)g(set)g(of)g(requests,)i(instead)f(of)f(the)g Fs(a)o(rra)o(y)p 1113 1655 V 15 w(of)p 1165 1655 V 16 w(requests)j FD(argumen)o(t)c(they)i(ha)o(v)o(e)f(no)o(w.)75 1712 y(Whenev)o(er)13 b(a)f(nonp)q(ersisten)o(t)h(request)f(is)h (satis\014ed)f(b)o(y)h(a)e(call)j(to)e Fs(MPI)p 1282 1712 V 15 w(W)l(AIT)l(ANY)i FD(or)d Fs(MPI)p 1646 1712 V 16 w(TEST)l(ANY)p FD(,)75 1768 y(it)16 b(is)g(dropp)q(ed)h(from)e (the)g(collection.)23 b(The)16 b(user)g(sp)q(eci\014es)h(an)f(index)h (for)e(eac)o(h)h(item)g(in)g(a)g(collection.)75 1825 y(When)f(a)f(call)h(to)f Fs(MPI)p 466 1825 V 16 w(W)l(AIT)p 595 1825 V 16 w(ANY)h FD(or)f(other)g(similar)i(function)f(satis\014es) f(a)g(request,)h(then)f(the)h(index)75 1881 y(asso)q(ciated)d(with)g (this)h(request)e(is)i(returned)f(b)o(y)g(the)g(call.)20 b(This)12 b(index)h(can)f(b)q(e)h(an)f(address)g(\(a)f(p)q(oin)o(ter\)) 75 1938 y(in)16 b(C,)f(and)g(can)g(b)q(e)h(used)g(directly)h(to)d (access)h(the)h(data)e(that)h(w)o(as)f(receiv)o(ed.)75 2052 y Fs(MPI)p 160 2052 V 16 w(COLLECTION)p 453 2052 V 16 w(CREA)l(TE\(collection\))117 2139 y Ft(OUT)108 b Fs(collection)409 b Ft(collection)14 b(\(handle\))75 2274 y FB(int)23 b(int)h(MPI)p 341 2274 15 2 v 17 w(Collection)p 598 2274 V 15 w(create\(MPI)p 853 2274 V 16 w(Comm)p 965 2274 V 17 w(collection)e(*collection\))75 2371 y(MPI)p 150 2371 V 17 w(COLLECTION)p 407 2371 V 16 w(CREATE\(COLLECTION,)f (IERROR\))170 2427 y(INTEGER)i(COLLECTION,)g(IERROR)166 2524 y FD(The)14 b(function)h Fs(MPI)p 520 2524 14 2 v 16 w(COLLECTION)p 813 2524 V 16 w(CREA)l(TE)g FD(creates)f(a)f (collection)j(ob)s(ject)d(for)g(m)o(ultiple)j(com-)75 2580 y(pletion)h(tests.)p 75 2661 720 2 v 127 2688 a Fz(1)144 2704 y Fh(Prop)q(osal)e(and)f(rationale)h(b)o(y)e(Salmon)h (and)g(Edelsohn)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 46 53 46 52 bop 75 -100 a FD(46)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(ADD)p 272 45 V 16 w(TO)p 352 45 V 17 w(COLLECTION\(request,)g(collection,)g(k)o(ey\))117 138 y Ft(IN)155 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))117 246 y(INOUT)62 b Fs(collection)409 b Ft(collection)14 b(\(handle\))117 353 y(IN)155 b Fs(k)o(ey)525 b Ft(user-sp)q(eci\014ed)17 b(k)o(ey)d(for)f(request)j(\(address\))75 494 y FB(int)23 b(int)h(MPI)p 341 494 15 2 v 17 w(Add)p 430 494 V 16 w(to)p 494 494 V 17 w(collection\(MPI)p 847 494 V 16 w(Comm)p 959 494 V 16 w(request)f(request,)g(MPI)p 1453 494 V 17 w(Comm)p 1566 494 V 16 w(collection)393 550 y(collection,)g(MPI)p 755 550 V 16 w(Aint)h(key\))75 653 y(MPI)p 150 653 V 17 w(ADD)p 239 653 V 17 w(TO)p 304 653 V 16 w(COLLECTION\(REQUEST,)e(COLLECTION,)g(KEY,)h(IERROR\))170 709 y(INTEGER)g(REQUEST,)g(COLLECTION,)f(KEY,)i(IERROR)166 812 y FD(The)15 b(function)g Fs(MPI)p 521 812 14 2 v 16 w(ADD)p 633 812 V 16 w(TO)p 713 812 V 17 w(COLLECTION)f FD(inserts)h Fs(request)h FD(in)o(to)f(the)f(set)g(of)g(requests)h (asso-)75 868 y(ciated)h(with)f Fs(collection)h FD(and)f(asso)q(ciates) h(the)f(user-sp)q(eci\014ed)i Fs(k)o(ey)f FD(with)f(it.)75 988 y Fs(MPI)p 160 988 V 16 w(DELETE)p 346 988 V 16 w(FROM)p 490 988 V 17 w(COLLECTION\(request,)h(collection\))117 1082 y Ft(IN)155 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))117 1189 y(INOUT)62 b Fs(collection)409 b Ft(collection)14 b(\(handle\))75 1330 y FB(int)23 b(int)h(MPI)p 341 1330 15 2 v 17 w(Delete)p 502 1330 V 16 w(from)p 614 1330 V 17 w(collection\(MPI)p 967 1330 V 15 w(Comm)p 1078 1330 V 17 w(request)e(request,)393 1386 y(MPI)p 468 1386 V 17 w(Comm)p 581 1386 V 17 w(collection)g(collection\))75 1489 y(MPI)p 150 1489 V 17 w(DELETE)p 311 1489 V 16 w(FROM)p 423 1489 V 17 w(COLLECTION\(REQUEST,)f(COLLECTION,)h(IERROR\))170 1545 y(INTEGER)h(REQUEST,)g(COLLECTION,)f(IERROR)166 1648 y FD(The)d(function)g Fs(MPI)p 529 1648 14 2 v 16 w(DELETE)p 715 1648 V 16 w(FROM)p 859 1648 V 17 w(COLLECTION)f FD(remo)o(v)o(es)g Fs(request)i FD(from)e(the)h(set)f(of)g(re-)75 1704 y(quests)d(asso)q(ciated)h(with)f Fs(collection)p FD(.)75 1824 y Fs(MPI)p 160 1824 V 16 w(W)l(AIT)l(ANY\(collection,)h (value,)f(status\))117 1917 y Ft(INOUT)62 b Fs(collection)409 b Ft(collection)14 b(\(handle\))117 2025 y(OUT)108 b Fs(k)o(ey)525 b Ft(user-sp)q(eci\014ed)17 b(k)o(ey)d(\(address\))117 2132 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\)) 75 2273 y FB(int)23 b(int)h(MPI)p 341 2273 15 2 v 17 w(Waitany\(MPI)p 622 2273 V 15 w(Comm)p 733 2273 V 17 w(collection)e(collection,)h(MPI)p 1371 2273 V 17 w(Aint)g(*key,)393 2329 y(MPI)p 468 2329 V 17 w(Status)g(*status\))75 2432 y(MPI)p 150 2432 V 17 w(WAITANY\(COLLECTION,)e(KEY,)i(STATUS,)g (IERROR\))170 2488 y(INTEGER)g(COLLECTION,)g(KEY,)g(STATUS\(MPI)p 1010 2488 V 16 w(STATUS)p 1170 2488 V 16 w(SIZE\),)g(IERROR)166 2591 y FD(The)11 b(function)g Fs(MPI)p 513 2591 14 2 v 16 w(W)l(AIT)l(ANY)h FD(w)o(aits)e(for)g(the)h(completion)h(of)e(one) h(request)g(from)f(the)h(collection.)75 2647 y(If)k(this)g(request)g (is)h(nonp)q(ersisten)o(t,)f(then)g(its)g(is)g(dropp)q(ed)h(from)e(the) h(collection.)22 b(The)15 b(k)o(ey)g(asso)q(ciated)75 2704 y(with)h(the)f(request)g(is)h(returned)f(in)h Fs(k)o(ey)p FD(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 47 54 47 53 bop 75 -100 a Fp(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FD(47)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(TEST)l(ANY\(collection,)16 b(\015ag,)f(k)o(ey)l(,)g(status\))117 125 y Ft(INOUT)62 b Fs(collection)409 b Ft(collection)14 b(\(handle\))117 206 y(OUT)108 b Fs(\015ag)518 b Fv(true)11 b Ft(if)d(one)i(of)f(the)h (op)q(erations)g(is)f(complete)g Fv(false)p Ft(,)h(otherwise)905 262 y(\(logical\))117 343 y(OUT)108 b Fs(k)o(ey)525 b Ft(user-sp)q(eci\014ed)17 b(k)o(ey)d(\(address\))117 423 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\)) 75 550 y FB(int)23 b(int)h(MPI)p 341 550 15 2 v 17 w(Testany\(MPI)p 622 550 V 15 w(Comm)p 733 550 V 17 w(collection)e(collection,)h(int)g (*flag,)g(MPI)p 1633 550 V 17 w(Aint)393 607 y(*key,)g(MPI)p 611 607 V 17 w(Status)g(*status\))75 696 y(MPI)p 150 696 V 17 w(TESTANY\(COLLECTION,)e(FLAG,)i(KEY,)h(STATUS,)e(IERROR\))170 752 y(INTEGER)h(COLLECTION,)g(FLAG,)g(KEY,)g(STATUS\(MPI)p 1153 752 V 16 w(STATUS)p 1313 752 V 16 w(SIZE\),)g(IERROR)166 842 y Fs(MPI)p 251 842 14 2 v 16 w(TEST)l(ANY)c FD(returns)f Fs(\015ag)g(=)g(true)g FD(if)h(an)o(y)e(of)h(the)g(op)q(erations)g (asso)q(ciated)g(with)g Fs(collection)75 898 y FD(has)f(completed.)28 b(In)19 b(this)f(case,)f(it)h(returns)g(in)g Fs(k)o(ey)g FD(the)f(user-sp)q(eci\014ed)j(v)m(alue)f(asso)q(ciated)f(with)g(the)75 955 y(request)d(handle)g(whic)o(h)h(completed,)f(in)h Fs(status)g FD(the)f(status)f(of)g(that)g(op)q(eration,)g(and)h(deallo) q(cates)h(the)75 1011 y(asso)q(ciated)f(comm)o(unications)h(ob)s(ject)f (if)h(ephemeral.)21 b(It)15 b(returns)g Fs(\015ag)g(=)g(false)p FD(,)g(otherwise.)75 1117 y Fs(MPI)p 160 1117 V 16 w(W)l(AIT)l (ALL\(collection,)g(a)o(rra)o(y)p 684 1117 V 14 w(of)p 735 1117 V 16 w(k)o(eys,)g(a)o(rra)o(y)p 953 1117 V 15 w(of)p 1005 1117 V 16 w(statuses\))117 1197 y Ft(INOUT)62 b Fs(collection)409 b Ft(collection)14 b(\(handle\))117 1278 y(OUT)108 b Fs(a)o(rra)o(y)p 416 1278 V 15 w(of)p 468 1278 V 16 w(k)o(eys)345 b Ft(arra)o(y)14 b(of)f(user-sp)q (eci\014ed)k(k)o(eys)d(\(arra)o(y)g(of)f(addresses\))117 1358 y(OUT)108 b Fs(a)o(rra)o(y)p 416 1358 V 15 w(of)p 468 1358 V 16 w(statuses)276 b Ft(arra)o(y)14 b(of)f(status)i(ob)r (jects)g(\(arra)o(y)f(of)f(Status\))75 1486 y FB(int)23 b(int)h(MPI)p 341 1486 15 2 v 17 w(Waitall\(MPI)p 622 1486 V 15 w(Comm)p 733 1486 V 17 w(collection)e(collection,)h(MPI)p 1371 1486 V 17 w(Aint)g(*array)p 1651 1486 V 16 w(of)p 1715 1486 V 17 w(keys,)393 1542 y(MPI)p 468 1542 V 41 w(Status)g(*array)p 820 1542 V 16 w(of)p 884 1542 V 17 w(statuses\))75 1631 y(MPI)p 150 1631 V 17 w(WAITALL\(COLLECTION,)e (ARRAY)p 764 1631 V 17 w(OF)p 829 1631 V 16 w(KEYS,)j(ARRAY)p 1109 1631 V 16 w(OF)p 1173 1631 V 17 w(STATUSES,)f(IERROR\))170 1688 y(INTEGER)g(COLLECTION,)g(ARRAY)p 771 1688 V 16 w(OF)p 835 1688 V 17 w(KEYS,)g(ARRAY)p 1115 1688 V 17 w(OF)p 1180 1688 V 17 w(STATUSES\(MPI)p 1485 1688 V 15 w(STATUS)p 1644 1688 V 17 w(SIZE,*\),)170 1744 y(IERROR)166 1833 y FD(The)14 b(function)g Fs(MPI)p 519 1833 14 2 v 16 w(W)l(AIT)l(ALL)f FD(blo)q(c)o(ks)h(un)o(til)h(all)f(comm)o (unication)g(op)q(erations)g(asso)q(ciated)g(with)75 1890 y(the)g(requests)g(in)h Fs(collection)g FD(complete.)20 b Fs(MPI)p 866 1890 V 16 w(W)l(AIT)l(ALL)13 b FD(returns)h(the)g (user-sp)q(eci\014ed)j(k)o(ey)d(asso)q(ciated)75 1946 y(with)21 b(eac)o(h)g(request)g(handle)h(in)f Fs(a)o(rra)o(y)p 764 1946 V 15 w(of)p 816 1946 V 16 w(k)o(eys)g FD(and)g(the)g(resp)q (ectiv)o(e)g(status)f(of)g(eac)o(h)h(op)q(eration)g(in)75 2003 y Fs(a)o(rra)o(y)p 173 2003 V 15 w(of)p 225 2003 V 16 w(statuses)p FD(.)h(All)16 b(ephemeral)h(comm)o(unication)f(ob)s (jects)e(are)h(deallo)q(cated.)75 2109 y Fs(MPI)p 160 2109 V 16 w(TEST)l(ALL\(collection,)g(\015ag,)g(a)o(rra)o(y)p 782 2109 V 14 w(of)p 833 2109 V 16 w(k)o(eys,)g(a)o(rra)o(y)p 1051 2109 V 15 w(of)p 1103 2109 V 16 w(statuses\))117 2189 y Ft(INOUT)62 b Fs(collection)409 b Ft(collection)14 b(\(handle\))117 2270 y(OUT)108 b Fs(\015ag)518 b Fv(true)15 b Ft(if)e(all)g(of)g(the)i(op)q(erations)f(are)g(complete)f (\(logical\))117 2350 y(OUT)108 b Fs(a)o(rra)o(y)p 416 2350 V 15 w(of)p 468 2350 V 16 w(k)o(eys)345 b Ft(arra)o(y)14 b(of)f(user-sp)q(eci\014ed)k(k)o(eys)d(\(arra)o(y)g(of)f(addresses\)) 117 2431 y(OUT)108 b Fs(a)o(rra)o(y)p 416 2431 V 15 w(of)p 468 2431 V 16 w(statuses)276 b Ft(arra)o(y)14 b(of)f(status)i(ob)r (jects)g(\(arra)o(y)f(of)f(Status\))75 2558 y FB(int)23 b(int)h(MPI)p 341 2558 15 2 v 17 w(Testall\(MPI)p 622 2558 V 15 w(Comm)p 733 2558 V 17 w(collection)e(collection,)h(int)g (*flag,)g(MPI)p 1633 2558 V 17 w(Aint)393 2614 y(*array)p 540 2614 V 17 w(of)p 605 2614 V 16 w(keys,)h(MPI)p 837 2614 V 16 w(Status)f(*array)p 1164 2614 V 17 w(of)p 1229 2614 V 17 w(statuses\))75 2704 y(MPI)p 150 2704 V 17 w(TESTALL\(COLLECTION,)e(FLAG,)i(ARRAY)p 907 2704 V 17 w(OF)p 972 2704 V 17 w(KEYS,)g(ARRAY)p 1252 2704 V 16 w(OF)p 1316 2704 V 17 w(STATUSES,)g(IERROR\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 48 55 48 54 bop 75 -100 a FD(48)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)170 45 y FB(INTEGER)23 b(COLLECTION,)g(FLAG,)g(ARRAY)p 914 45 15 2 v 16 w(OF)p 978 45 V 17 w(KEYS,)170 102 y(ARRAY)p 293 102 V 17 w(OF)p 358 102 V 17 w(STATUSES\(MPI)p 663 102 V 15 w(STATUS)p 822 102 V 17 w(SIZE,*\),)g(IERROR)166 188 y Fs(MPI)p 251 188 14 2 v 16 w(TEST)l(ALL)12 b FD(returns)g Fs(\015ag)h(=)g(true)g FD(if)f(all)i(of)e(the)g(op)q(erations)h(asso)q(ciated)g(with)f Fs(collection)h FD(ha)o(v)o(e)75 245 y(completed.)31 b(In)19 b(this)g(case,)g(it)f(returns)h(in)g(the)g(user-sp)q(eci\014ed) i(k)o(ey)d(asso)q(ciated)h(with)f(eac)o(h)h(request)75 301 y(handle)f(in)f Fs(a)o(rra)o(y)p 375 301 V 14 w(of)p 426 301 V 16 w(k)o(eys)g FD(and)f(the)h(resp)q(ectiv)o(e)g(status)f(of) f(eac)o(h)i(op)q(eration)f(in)h Fs(a)o(rra)o(y)p 1564 301 V 15 w(of)p 1616 301 V 16 w(statuses)p FD(.)26 b(All)75 357 y(ephemeral)17 b(comm)o(unication)g(ob)s(jects)f(are)g(deallo)q (cated.)25 b(It)16 b(returns)g Fs(\015ag)g(=)h(false)p FD(,)f(otherwise.)23 b(This)75 414 y(is)16 b(a)f(non)o(blo)q(c)o(king)h (op)q(erations)f(that)g(returns)g(immediately)l(.)75 518 y Fs(MPI)p 160 518 V 16 w(COLLECTION)p 453 518 V 16 w(SIZE\(collection,)g(count\))117 595 y Ft(IN)155 b Fs(collection)409 b Ft(collection)14 b(\(handle\))117 670 y(OUT)108 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(requests)i(in)d(collection)g(\(in)o(teger\))75 795 y FB(int)23 b(int)h(MPI)p 341 795 15 2 v 17 w(Collection)p 598 795 V 15 w(size\(MPI)p 805 795 V 17 w(Comm)p 918 795 V 16 w(collection)f(collection,int)f(*count\))75 881 y(MPI)p 150 881 V 17 w(COLLECTION)p 407 881 V 16 w(SIZE\(COLLECTION,)f(COUNT,)i(IERROR\))p Fs(I)p FB(NTEGER)f (COLLECTION,)g(COUNT,)393 938 y(IERROR)166 1024 y Fs(MPI)p 251 1024 14 2 v 16 w(COLLECTION)p 544 1024 V 16 w(SIZE)f FD(returns)h(the)f(n)o(um)o(b)q(er)h(of)f(requests)h(in)g(the)g (collection)h(for)e(whic)o(h)75 1081 y(there)16 b(is)h(an)f(incomplete) i(comm)o(unication.)24 b(The)16 b(coun)o(t)g(ma)o(y)g(b)q(e)g(used)h (as)f(the)g(size)i(of)d(the)i(arra)o(y)e(to)75 1137 y(pass)g(to)g Fs(MPI)p 315 1137 V 15 w(TEST)l(ALL)h FD(and)f Fs(MPI)p 706 1137 V 16 w(W)l(AIT)l(ALL)p FD(.)75 1241 y Fs(MPI)p 160 1241 V 16 w(COLLECTION)p 453 1241 V 16 w(FREE\(collection\))117 1318 y Ft(IN)155 b Fs(collection)409 b Ft(collection)14 b(\(handle\))75 1443 y FB(int)23 b(int)h(MPI)p 341 1443 15 2 v 17 w(Collection)p 598 1443 V 15 w(free\(MPI)p 805 1443 V 17 w(Comm)p 918 1443 V 16 w(collection)f(collection\))75 1529 y(MPI)p 150 1529 V 17 w(COLLECTION)p 407 1529 V 16 w(FREE\(COLLECTION,)e(IERROR\))170 1586 y(INTEGER)i(COLLECTION,)g (IERROR)166 1672 y FD(The)12 b(function)g Fs(MPI)p 515 1672 14 2 v 16 w(COLLECTION)p 808 1672 V 16 w(FREE)h FD(deallo)q(cates)f(the)g(collection)h(ob)s(ject.)18 b(It)12 b(is)g(erroneous)75 1728 y(to)j(initiate)h(a)f(m)o(ultiple)i (completion)f(test)f(with)h(a)e(collection)j(that)e(has)g(b)q(een)h (freed.)166 1868 y Fl(Discussion:)34 b Ft(Some)13 b(\(less)h(p)q(o)o(w) o(erful\))g(alternativ)o(es:)75 1924 y(Drop)f(the)g(k)o(ey:)18 b(the)13 b(user)h(will)e(need)h(to)g(use)h(message)e(tags,)h(in)f (order)i(to)f(iden)o(tify)f(requests)j(\(to)e(the)g(least,)g(this)75 1981 y(requires)i Fv(MPI)p 308 1981 13 2 v 14 w(GET)p 403 1981 V 15 w(T)m(A)o(G)e Ft(to)h(w)o(ork)f(after)h(a)g(send,)g(to)q (o\).)75 2037 y(Drop)g(the)p 255 2037 V 29 w(ADD)f(and)p 459 2037 V 29 w(DELETE.)h(Basically)m(,)e(this)i(amoun)o(ts)e(to)i(ha)o (ving)f(a)g(\\commit)p 1482 2037 V 12 w(arra)o(y")h(function.)75 2263 y Fq(3.9)59 b(Prob)r(e)20 b(and)f(Cancel)75 2365 y FD(The)d Fs(MPI)p 254 2365 14 2 v 16 w(PROBE)g FD(and)g Fs(MPI)p 605 2365 V 16 w(IPROBE)g FD(op)q(erations)g(allo)o(w)g (incoming)h(messages)e(to)g(b)q(e)h(c)o(hec)o(k)o(ed)g(for,)75 2421 y(without)g(actually)h(receiving)h(them.)k(The)17 b(user)f(can)g(then)h(decide)g(ho)o(w)f(to)g(receiv)o(e)h(them,)f (based)g(on)75 2478 y(the)i(information)f(returned)h(b)o(y)f(the)h (prob)q(e)g(\(basically)l(,)h(the)e(information)h(returned)g(b)o(y)f Fs(status)p FD(\).)28 b(In)75 2534 y(particular,)16 b(the)g(user)g(ma)o (y)f(allo)q(cate)h(memory)f(for)g(the)h(receiv)o(e)h(bu\013er,)e (according)h(to)f(the)h(length)g(of)75 2591 y(the)f(prob)q(ed)h (message.)166 2647 y(The)i Fs(MPI)p 347 2647 V 16 w(CANCEL)g FD(op)q(eration)g(allo)o(ws)g(p)q(ending)i(comm)o(unications)e(to)g(b)q (e)g(canceled.)30 b(This)18 b(is)75 2704 y(required)i(for)f(clean)o (up.)33 b(P)o(osting)19 b(a)g(send)h(or)f(a)g(receiv)o(e)h(ties)f(up)h (user)f(resources)h(\(send)f(or)g(receiv)o(e)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 49 56 49 55 bop 75 -100 a Fp(3.9.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b FD(49)75 45 y(bu\013ers\),)14 b(and)i(a)f(cancel)h(ma)o(y)f(b)q(e)g (needed)i(to)d(free)i(these)f(resources)g(gracefully)l(.)75 162 y Fs(MPI)p 160 162 14 2 v 16 w(IPROBE\(source,)h(tag,)f(comm)m(,)c (\015ag,)k(status\))117 251 y Ft(IN)155 b Fs(source)468 b Ft(source)15 b(rank,)f(or)27 b Fv(MPI)p 1281 251 13 2 v 15 w(ANY)p 1381 251 V 14 w(SOURCE)13 b Ft(\(in)o(teger\))117 352 y(IN)155 b Fs(tag)526 b Ft(tag)14 b(v)n(alue)f(or)28 b Fv(MPI)p 1226 352 V 14 w(ANY)p 1325 352 V 15 w(T)m(A)o(G)12 b Ft(\(in)o(teger\))117 453 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 553 y(OUT)108 b Fs(\015ag)518 b Ft(\(logical\))117 654 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\))75 791 y FB(int)23 b(MPI)p 245 791 15 2 v 17 w(Iprobe\(int)g(source,)g (int)g(tag,)g(MPI)p 1002 791 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 847 y(MPI)p 468 847 V 17 w(Status)g(*status\))75 947 y(MPI)p 150 947 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 1003 y(LOGICAL)g(FLAG)170 1059 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1059 V 16 w(STATUS)p 1218 1059 V 16 w(SIZE\),)g(IERROR)166 1159 y Fs(MPI)p 251 1159 14 2 v 16 w(IPROBE)c FD(returns)f Fs(\015ag)g(=)h(true)f FD(if)h(there)g(is)f(a)g(message)g(that)g(can)g (b)q(e)h(receiv)o(ed)g(and)g(that)75 1215 y(matc)o(hes)i(the)g(pattern) f(sp)q(eci\014ed)j(b)o(y)e(the)h(argumen)o(ts)e Fs(source)p FD(,)i Fs(tag)p FD(,)g(and)g Fs(comm)m FD(.)34 b(It)21 b(returns)g Fs(\015ag)75 1272 y(=)i(false)p FD(,)h(otherwise.)41 b(If)23 b Fs(MPI)p 633 1272 V 16 w(IPROBE)g FD(returns)f Fs(\015ag)g(=)h(true)p FD(,)h(then)f(the)f(status)g(ob)s(ject)g(can)g (b)q(e)75 1328 y(subsequen)o(tly)c(queried)g(using)g(the)f(functions)h (describ)q(ed)g(in)g(section)g(3.3.1)d(to)h(\014nd)i(the)f(source,)g (tag)75 1384 y(and)f(length)h(of)f(the)g(message.)23 b(The)16 b(v)m(alues)i(returned)e(b)o(y)h(these)f(queries)h(are)f(the)g (same)g(v)m(alues)i(that)75 1441 y(w)o(ould)12 b(ha)o(v)o(e)e(b)q(een)j (returned)e(b)o(y)h(a)f(call)h(to)e Fs(MPI)p 897 1441 V 16 w(RECV)i FD(executed)g(at)f(the)g(same)g(p)q(oin)o(t)h(in)g(the)f (program.)75 1497 y(The)k(return)h(status)e(ob)s(ject)h(is)g (unde\014ned)i(if)f Fs(\015ag)f(=)h(false)p FD(.)166 1567 y(A)d(subsequen)o(t)i(receiv)o(e)f(executed)g(with)g(the)g(same)f (con)o(text,)g(and)g(the)h(source)f(and)h(tag)f(returned)75 1623 y(b)o(y)g(querying)h(the)f(status)f(returned)i(b)o(y)f Fs(MPI)p 858 1623 V 16 w(IPROBE)h FD(will)h(receiv)o(e)f(the)f(message) f(that)h(w)o(as)f(matc)o(hed)75 1679 y(b)o(y)21 b(the)g(prob)q(e,)h(if) g(no)f(other)f(in)o(terv)o(ening)i(receiv)o(e)g(o)q(ccurred)f(after)g (the)g(prob)q(e.)37 b(If)21 b(the)g(receiving)75 1736 y(pro)q(cess)e(is)h(m)o(ulti-threaded,)h(it)f(is)f(the)h(user's)f(resp) q(onsibilit)o(y)i(to)e(ensure)h(that)e(the)i(last)f(condition)75 1792 y(holds.)166 1862 y(The)d Fs(source)h FD(argumen)o(t)e(of)h Fs(MPI)p 738 1862 V 16 w(PROBE)h FD(can)f(b)q(e)h Fv(MPI)p 1140 1862 13 2 v 14 w(ANY)p 1239 1862 V 15 w(SOURCE)p FD(,)d(and)i(the)h Fs(tag)f FD(argumen)o(t)75 1918 y(can)g(b)q(e)g Fv(MPI)p 298 1918 V 15 w(ANY)p 398 1918 V 14 w(T)m(A)o(G)p FD(,)f(so)g(that)g(one)h(can)g(prob)q(e)g(for)f(messages)h(from)f(an)h (arbitrary)f(source)h(and/or)75 1974 y(with)g(an)g(arbitrary)f(tag.)20 b(Ho)o(w)o(ev)o(er,)15 b(a)g(sp)q(eci\014c)j(comm)o(unication)e(con)o (text)f(m)o(ust)g(pro)o(vided)i(with)f(the)75 2031 y Fs(comm)9 b FD(argumen)o(t.)166 2100 y(A)17 b(call)h(to)e Fs(MPI)p 444 2100 14 2 v 16 w(PROBE)i FD(will)h(not)d(matc)o(h)g(a)h (message)g(that)f(has)g(already)i(b)q(een)g(matc)o(hed)e(b)o(y)h(a)95 b Fm(CHANGE)75 2157 y FD(previous)17 b(receiv)o(e)g(op)q(eration.)23 b(It)16 b(ma)o(y)f(matc)o(h)h(a)f(message)h(that)f(has)h(b)q(een)h (matc)o(hed)f(b)o(y)g(a)g(previous)75 2213 y(prob)q(e)f(op)q(eration)g (or)f(b)o(y)h(a)f(concurren)o(t)h(receiv)o(e)g(\(e.g.,)e(a)h(receiv)o (e)i(executed)g(b)o(y)e(a)g(concurren)o(t)h(thread)75 2269 y(in)h(a)f(m)o(ultithreaded)h(en)o(vironmen)o(t.)166 2421 y Fl(Implemen)o(tati)o(on)g(note:)47 b Ft(MPI)17 b(guaran)o(tees)h(that)f(successiv)o(e)i(messages)d(sen)o(t)i(from)d(a) i(source)h(to)e(a)75 2478 y(destination)10 b(within)f(the)i(same)f(con) o(text)h(are)f(receiv)o(ed)i(in)d(the)i(order)g(they)g(are)g(sen)o(t.) 17 b(Th)o(us,)11 b(MPI)f(m)o(ust)g(supp)q(ort,)75 2534 y(either)k(explicitly)e(or)h(implicitly)l(,)d(a)j(FIF)o(O)g(structure)i (to)e(manage)f(messages)h(b)q(et)o(w)o(een)h(eac)o(h)g(pair)e(of)h(pro) q(cesses,)75 2591 y(for)k(eac)o(h)g(con)o(text.)56 b Fv(MPI)p 516 2591 13 2 v 14 w(PROBE)16 b Ft(returns)i(information)c(on) j(the)h(\014rst)g(matc)o(hing)d(message)i(in)f(this)h(FIF)o(O;)75 2647 y(this)d(will)e(also)i(b)q(e)g(the)h(message)e(receiv)o(ed)i(b)o (y)f(the)g(\014rst)h(subsequen)o(t)h(receiv)o(e)f(with)e(the)i(same)e (source,)i(tag)e(and)75 2704 y(con)o(text)i(as)f(the)g(message)g(matc)o (hed)f(b)o(y)g Fv(MPI)p 806 2704 V 15 w(PROBE)p Ft(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 50 57 50 56 bop 75 -100 a FD(50)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(PROBE\(source,)g(tag,)f(comm)m(,)d(status\))117 122 y Ft(IN)155 b Fs(source)468 b Ft(source)15 b(rank,)f(or)27 b Fv(MPI)p 1281 122 13 2 v 15 w(ANY)p 1381 122 V 14 w(SOURCE)13 b Ft(\(in)o(teger\))117 197 y(IN)155 b Fs(tag)526 b Ft(tag)14 b(v)n(alue,)e(or)28 b Fv(MPI)p 1237 197 V 14 w(ANY)p 1336 197 V 15 w(T)m(A)o(G)13 b Ft(\(in)o(teger\))117 272 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 347 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\)) 75 472 y FB(int)23 b(MPI)p 245 472 15 2 v 17 w(Probe\(int)g(source,)g (int)g(tag,)h(MPI)p 979 472 V 16 w(Comm)g(comm,)f(MPI)p 1330 472 V 17 w(Status)g(*status\))75 558 y(MPI)p 150 558 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g(IERROR\))170 615 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 615 V 16 w(STATUS)p 1218 615 V 16 w(SIZE\),)g(IERROR)166 701 y Fs(MPI)p 251 701 14 2 v 16 w(PROBE)17 b FD(b)q(eha)o(v)o(es)g (lik)o(e)g Fs(MPI)p 769 701 V 16 w(IPROBE)g FD(except)g(that)f(it)g(is) h(a)f(blo)q(c)o(king)i(call)f(whic)o(h)g(returns)75 757 y(only)f(after)e(a)h(matc)o(hing)g(message)g(has)g(b)q(een)h(found.)166 814 y(The)10 b(MPI)g(implemen)o(tation)i(of)e Fs(MPI)p 805 814 V 15 w(PROBE)i FD(and)e Fs(MPI)p 1145 814 V 16 w(IPROBE)h FD(needs)g(to)e(guaran)o(tee)h(progress:)75 870 y(if)17 b(a)g(call)h(to)e Fs(MPI)p 385 870 V 16 w(PROBE)i FD(has)f(b)q(een)h(issued)g(b)o(y)f(a)f(pro)q(cess,)i(and)f(a)f(send)i (that)e(matc)o(hes)h(the)f(prob)q(e)75 927 y(has)j(b)q(een)i(initiated) g(b)o(y)f(some)f(pro)q(cess,)i(then)f(the)f(call)i(to)e Fs(MPI)p 1244 927 V 16 w(PROBE)i FD(will)g(return,)f(unless)h(the)75 983 y(message)c(is)i(receiv)o(ed)g(b)o(y)f(another)f(concurren)o(t)h (receiv)o(e)h(op)q(eration)f(\(that)e(is)j(executed)g(b)o(y)e(another) 75 1040 y(thread)e(at)g(the)h(probing)g(pro)q(cess\).)21 b(Similarly)l(,)d(if)e(a)f(pro)q(cess)h(busy)g(w)o(aits)f(with)h Fs(MPI)p 1573 1040 V 16 w(IPROBE)g FD(and)g(a)75 1096 y(matc)o(hing)e(message)g(has)g(b)q(een)h(issued,)g(then)f(the)g(call)h (to)f Fs(MPI)p 1179 1096 V 16 w(IPROBE)g FD(will)i(ev)o(en)o(tually)f (return)f Fs(\015ag)75 1153 y(=)i(true)f FD(unless)i(the)e(message)g (is)g(receiv)o(ed)i(b)o(y)e(another)g(concurren)o(t)g(receiv)o(e)h(op)q (eration.)75 1256 y Fs(MPI)p 160 1256 V 16 w(CANCEL\(request\))117 1334 y Ft(IN)155 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 1458 y FB(int)23 b(MPI)p 245 1458 15 2 v 17 w(Cancel\(MPI)p 502 1458 V 16 w(Comm)p 614 1458 V 17 w(request)g(*request\))75 1544 y(MPI)p 150 1544 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1601 y(INTEGER)h(REQUEST,)g(IERROR)166 1687 y FD(A)17 b(call)h(to)e Fs(MPI)p 444 1687 14 2 v 15 w(CANCEL)i FD(marks)e(for)g(cancellation)i (a)f(p)q(ending)h(non)o(blo)q(c)o(king)g(comm)o(unication)75 1744 y(op)q(eration)f(\(send)g(or)g(receiv)o(e\).)25 b(The)18 b(cancel)g(call)g(is)f(non)o(blo)q(c)o(king,)h(and)f(lo)q (cal.)27 b(It)17 b(returns)f(immedi-)75 1800 y(ately)l(,)f(p)q(ossibly) i(b)q(efore)e(the)h(comm)o(unication)g(is)f(actually)h(canceled.)166 1857 y(It)c(is)g(still)h(necessary)f(to)g(complete)g(a)g(comm)o (unication)g(that)f(has)h(b)q(een)h(mark)o(ed)e(for)h(cancellation,)75 1913 y(using)i(a)g(call)h(to)e Fs(MPI)p 450 1913 V 15 w(W)l(AIT)h FD(or)f Fs(MPI)p 728 1913 V 16 w(TEST)h FD(\(or)f(an)o(y)g (of)g(the)h(deriv)o(ed)h(op)q(erations\).)k(If)14 b(the)g(op)q(eration) 75 1970 y(has)e(b)q(een)h(canceled,)h(then)f(information)f(to)g(that)f (e\013ect)h(will)i(b)q(e)f(returned)f(in)h(the)g(status)e(argumen)o(t)g (of)75 2026 y(the)j(op)q(eration)g(that)f(completes)h(the)g(comm)o (unication.)20 b(If)14 b(a)g(comm)o(unication)g(is)g(mark)o(ed)g(for)f (cancel-)75 2083 y(lation,)j(then)h(a)e Fs(MPI)p 445 2083 V 16 w(W)l(AIT)h FD(call)h(for)f(that)f(comm)o(unication)h(is)h (guaran)o(teed)e(to)h(return,)f(irresp)q(ectiv)o(e)75 2139 y(of)i(the)g(activities)i(of)e(other)g(pro)q(cesses)h(\(i.e.,)f Fs(MPI)p 970 2139 V 16 w(W)l(AIT)g FD(b)q(eha)o(v)o(es)h(as)f(a)g(lo)q (cal)i(function\);)f(similarly)75 2195 y(if)g Fs(MPI)p 204 2195 V 16 w(TEST)f FD(is)h(rep)q(eatedly)g(called)h(in)f(a)f(busy)h (w)o(ait)e(lo)q(op)i(for)f(a)g(canceled)i(comm)o(unication,)f(then)75 2252 y Fs(MPI)p 160 2252 V 16 w(TEST)d FD(will)i(ev)o(en)o(tually)f(b)q (e)g(successful.)166 2308 y(Either)j(the)f(cancellation)i(succeeds,)f (or)f(the)g(comm)o(unication)h(succeeds,)h(but)e(not)g(b)q(oth.)29 b(If)18 b(a)75 2365 y(send)h(is)h(mark)o(ed)e(for)g(cancellation,)j (then)e(it)g(m)o(ust)f(b)q(e)i(the)f(case)g(that)f(either)h(the)g(send) g(completes)75 2421 y(normally)l(,)f(in)h(whic)o(h)f(case)f(the)h (message)f(sen)o(t)g(w)o(as)g(receiv)o(ed)h(at)f(the)h(destination)g (pro)q(cess,)g(or)f(that)75 2478 y(the)f(send)g(is)g(successfully)i (canceled,)f(in)g(whic)o(h)f(case)g(no)g(part)f(of)g(the)h(message)f(w) o(as)g(receiv)o(ed)i(at)e(the)75 2534 y(destination.)21 b(Then,)15 b(an)o(y)f(matc)o(hing)h(receiv)o(e)h(has)e(to)h(b)q(e)g (satis\014ed)h(b)o(y)e(another)h(send.)20 b(If)15 b(a)g(receiv)o(e)g (is)75 2591 y(mark)o(ed)d(for)f(cancellation,)k(then)d(it)h(m)o(ust)e (b)q(e)i(the)g(case)f(that)f(either)i(the)g(receiv)o(e)g(completes)g (normally)l(,)75 2647 y(or)18 b(that)g(the)g(receiv)o(e)i(is)f (successfully)h(canceled,)h(in)e(whic)o(h)h(case)e(no)h(part)f(of)g (the)g(receiv)o(e)i(bu\013er)e(is)75 2704 y(altered.)i(Then,)c(an)o(y)e (matc)o(hing)i(send)g(has)f(to)f(b)q(e)i(satis\014ed)g(b)o(y)f(another) g(receiv)o(e.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 51 58 51 57 bop 75 -100 a Fp(3.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)638 b FD(51)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(TEST)p 290 45 V 16 w(CANCELLED\(status,)17 b(\015ag\))117 139 y Ft(IN)155 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g (\(Status\))117 247 y(OUT)108 b Fs(\015ag)518 b Ft(\(logical\))75 388 y FB(int)23 b(MPI)p 245 388 15 2 v 17 w(Test)p 358 388 V 17 w(cancelled\(MPI)p 687 388 V 15 w(Status)g(status,)g(int)h (*flag\))75 491 y(MPI)p 150 491 V 17 w(TEST)p 263 491 V 16 w(CANCELLED\(STATUS,)e(FLAG,)h(IERROR\))170 547 y(LOGICAL)g(FLAG)170 604 y(INTEGER)g(STATUS\(MPI)p 604 604 V 16 w(STATUS)p 764 604 V 17 w(SIZE\),)g(IERROR)166 707 y FD(Returns)12 b Fs(\015ag)f(=)h(true)g FD(if)f(the)h(comm)o (unication)g(asso)q(ciated)f(with)h(the)g(status)e(ob)s(ject)h(w)o(as)f (canceled)75 763 y(successfully)l(.)27 b(In)18 b(suc)o(h)f(a)g(case,)g (all)g(other)g(\014elds)h(of)e Fs(status)j FD(\(suc)o(h)e(as)g Fs(count)h FD(or)e Fs(tag)p FD(\))h(are)f(unde\014ned.)75 819 y(Returns)i Fs(\015ag)f(=)h(false)p FD(,)f(otherwise.)27 b(If)18 b(a)f(receiv)o(e)h(op)q(eration)g(migh)o(t)f(b)q(e)h(canceled)h (then)e(one)h(should)75 876 y(call)41 b Fs(MPI)p 268 876 14 2 v 16 w(TEST)p 398 876 V 17 w(CANCELLED)20 b FD(\014rst,)h(to)e(c)o(hec)o(k)i(whether)f(the)g(op)q(eration)h(w)o(as) e(canceled,)k(b)q(efore)75 932 y(c)o(hec)o(king)16 b(on)f(the)g(other)g (\014elds)i(of)d(the)i(return)f(status.)166 1088 y Fl(Implemen)o(tati)o (on)h(note:)51 b Ft(If)18 b(a)f(send)h(op)q(eration)g(uses)h(an)e (\\eager")h(proto)q(col)f(\(data)h(is)f(transferred)75 1144 y(to)e(the)h(receiv)o(er)h(b)q(efore)f(a)g(matc)o(hing)d(receiv)o (e)k(is)e(p)q(osted\),)i(then)f(the)g(cancelation)f(of)g(this)g(send)i (ma)o(y)c(require)75 1201 y(comm)o(unication)j(with)j(the)h(in)o (tended)g(receiv)o(er,)h(in)e(order)h(to)f(free)h(allo)q(cated)f (bu\013ers.)36 b(On)19 b(some)g(systems)75 1257 y(this)14 b(ma)o(y)e(require)j(an)f(in)o(terrupt)h(to)e(the)i(in)o(tended)g (receiv)o(er.)20 b(Therefore,)15 b(it)e(is)h(exp)q(ected)i(that)e Fv(MPI)p 1705 1257 13 2 v 15 w(CANCEL)75 1314 y Ft(will)h(b)q(e)i(an)f (exp)q(ensiv)o(e)i(op)q(eration)e(on)g(man)o(y)f(systems,)i(to)f(b)q(e) h(used)g(only)f(exceptionally)m(.)25 b(Note)17 b(that,)f(while)75 1370 y(comm)o(unication)f(ma)o(y)h(b)q(e)i(needed)i(to)e(implemen)o(t)d Fv(MPI)p 1006 1370 V 14 w(CANCEL)p Ft(,)i(this)h(is)g(still)f(a)h(lo)q (cal)f(op)q(eration,)i(in)e(the)75 1427 y(sense)h(that)e(its)g (completion)e(do)q(es)j(not)f(dep)q(end)h(on)f(the)g(co)q(de)h (executed)h(b)o(y)e(other)g(pro)q(cesses.)27 b(If)16 b(pro)q(cessing)75 1483 y(is)d(required)i(on)e(another)h(pro)q(cess,)h (this)e(should)g(b)q(e)i(transparen)o(t)f(to)f(the)i(application)d (\(hence)j(the)f(need)g(for)f(an)75 1540 y(in)o(terrupt)i(and)e(an)h (in)o(terrupt)h(handler\).)75 1860 y Fq(3.10)59 b(P)n(ersistent)19 b(communication)f(requests)75 1993 y FD(Often)k(a)g(comm)o(unication)h (with)f(the)g(same)g(argumen)o(t)f(list)i(is)g(rep)q(eatedly)g (executed)g(within)g(the)75 2050 y(inner)c(lo)q(op)g(of)e(a)h(parallel) i(computation.)28 b(In)18 b(suc)o(h)h(a)f(situation,)g(it)h(ma)o(y)e(b) q(e)i(p)q(ossible)g(to)f(optimize)75 2106 y(the)h(comm)o(unication)g(b) o(y)g(binding)h(the)f(list)g(of)g(comm)o(unication)g(argumen)o(ts)f(to) g(a)g(comm)o(unication)75 2163 y(request)11 b(once)g(and,)g(then,)h (rep)q(eatedly)g(using)g(the)f(comm)o(unication)g(request)g(to)f (initiate)i(and)f(complete)75 2219 y(messages.)19 b(The)d(comm)o (unication)f(request)h(th)o(us)f(created)g(can)g(b)q(e)h(though)o(t)e (of)h(as)f(a)h(comm)o(unication)75 2275 y(p)q(ort)c(or)f(a)h(\\half-c)o (hannel")h(.)18 b(It)12 b(do)q(es)f(not)f(pro)o(vide)i(the)f(full)h (functionalit)o(y)h(of)d(a)h(con)o(v)o(en)o(tional)g(c)o(hannel,)75 2332 y(since)i(there)g(is)g(no)f(binding)i(of)e(the)g(send)h(p)q(ort)f (to)f(the)i(receiv)o(e)g(p)q(ort:)18 b(this)13 b(construct)f(allo)o(ws) g(reduction)75 2388 y(of)18 b(the)g(o)o(v)o(erhead)f(for)h(comm)o (unication)g(b)q(et)o(w)o(een)h(pro)q(cessor)e(and)h(comm)o(unication)h (con)o(troller,)g(but)75 2445 y(not)13 b(of)g(the)g(o)o(v)o(erhead)g (for)g(comm)o(unication)h(b)q(et)o(w)o(een)g(one)f(comm)o(unication)h (con)o(troller)g(and)g(another.)166 2518 y(A)j(comm)o(unication)g (request)g(is)g(created)g(using)h(one)f(of)f(the)h(four)f(follo)o(wing) i(calls.)26 b(These)17 b(calls)75 2574 y(in)o(v)o(olv)o(e)f(no)f(comm)o (unication.)166 2647 y(The)10 b(function)i Fs(MPI)p 513 2647 14 2 v 15 w(CREA)l(TE)p 697 2647 V 18 w(SEND)f FD(creates)f(a)g (comm)o(unication)h(request)f(for)g(a)g(standard)g(mo)q(de)75 2704 y(send)16 b(op)q(eration,)f(and)g(binds)i(to)d(it)i(all)g(the)f (argumen)o(ts)f(of)h(a)g(send)h(op)q(eration.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 52 59 52 58 bop 75 -100 a FD(52)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(CREA)l(TE)p 345 45 V 17 w(SEND\(buf,)g(count,)g (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm)m(,)d(request\))117 136 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 239 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 341 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 444 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 546 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 649 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 751 y(OUT)108 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 889 y FB(int)23 b(MPI)p 245 889 15 2 v 17 w(Create)p 406 889 V 17 w(send\(void*)f(buf,)h(int)h (count,)f(MPI)p 1139 889 V 17 w(Datatype)f(datatype,)h(int)h(dest,)393 946 y(int)g(tag,)f(MPI)p 683 946 V 17 w(Comm)g(comm,)g(MPI)p 1034 946 V 17 w(Comm)p 1147 946 V 17 w(request)g(*request\))75 1046 y(MPI)p 150 1046 V 17 w(CREATE)p 311 1046 V 16 w(SEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g(REQUEST,)g(IERROR\))170 1103 y()g(BUF\(*\))170 1159 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 1259 y FD(The)16 b(function)h Fs(MPI)p 524 1259 14 2 v 16 w(CREA)l(TE)p 709 1259 V 17 w(RSEND)g FD(creates)e(a)h(comm)o (unication)h(ob)s(ject)e(for)g(a)h(ready)g(mo)q(de)75 1316 y(send)g(op)q(eration.)75 1433 y Fs(MPI)p 160 1433 V 16 w(CREA)l(TE)p 345 1433 V 17 w(RSEND\(buf,)g(count,)g(datat)o(yp)q (e,)h(dest,)f(tag,)f(comm)m(,)d(request\))117 1524 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1627 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 1729 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 1832 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1934 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 2037 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 2139 y(OUT)108 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 2277 y FB(int)23 b(MPI)p 245 2277 15 2 v 17 w(Create)p 406 2277 V 17 w(rsend\(void*)f(buf,)h(int)h (count,)f(MPI)p 1163 2277 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 2334 y(int)h(tag,)f(MPI)p 683 2334 V 17 w(Comm)g(comm,)g(MPI)p 1034 2334 V 17 w(Comm)p 1147 2334 V 17 w(request)g(*request\))75 2434 y(MPI)p 150 2434 V 17 w(CREATE)p 311 2434 V 16 w(RSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 2491 y()g(BUF\(*\))170 2547 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)166 2647 y FD(The)15 b(function)i Fs(MPI)p 523 2647 14 2 v 15 w(CREA)l(TE)p 707 2647 V 18 w(SSEND)f FD(creates)f(a)g(comm)o(unication)h(ob)s(ject)e (for)h(a)g(sync)o(hronous)75 2704 y(mo)q(de)g(send)h(op)q(eration.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 53 60 53 59 bop 75 -100 a Fp(3.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)638 b FD(53)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(CREA)l(TE)p 345 45 V 17 w(SSEND\(buf,)16 b(count,)g(datat)o(yp)q (e,)h(dest,)f(tag,)f(comm)m(,)d(request\))117 122 y Ft(IN)155 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 198 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 273 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 424 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(\(in)o(teger\))117 499 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 575 y(OUT)108 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 699 y FB(int)23 b(MPI)p 245 699 15 2 v 17 w(Create)p 406 699 V 17 w(ssend\(void*)f(buf,)h(int)h (count,)f(MPI)p 1163 699 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 756 y(int)h(tag,)f(MPI)p 683 756 V 17 w(Comm)g(comm,)g(MPI)p 1034 756 V 17 w(Comm)p 1147 756 V 17 w(request)g(*request\))75 842 y(MPI)p 150 842 V 17 w(CREATE)p 311 842 V 16 w(SSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 899 y()g(BUF\(*\))170 955 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,) g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)166 1042 y FD(The)15 b(function)h Fs(MPI)p 522 1042 14 2 v 15 w(CREA)l(TE)p 706 1042 V 18 w(RECV)g FD(creates)e(a)h(comm)o(unication)g(ob)s(ject)f (for)h(a)f(receiv)o(e)i(op)q(era-)75 1098 y(tion.)75 1202 y Fs(MPI)p 160 1202 V 16 w(CREA)l(TE)p 345 1202 V 17 w(RECV\(buf,)g(count,)h(datat)o(yp)q(e,)f(source,)f(tag,)g(comm)m (,)d(request\))117 1279 y Ft(OUT)108 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1355 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(receiv)o(ed)j(\(in)o(teger\))117 1430 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 1505 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(source)j(or)d(MPI)p 1309 1505 13 2 v 16 w(ANY)p 1418 1505 V 15 w(SOUR)o(CE)h(\(in)o (teger\))117 1581 y(IN)155 b Fs(tag)526 b Ft(message)14 b(tag)f(or)h(MPI)p 1270 1581 V 15 w(ANY)p 1378 1581 V 16 w(T)m(A)o(G)f(\(in)o(teger\))117 1656 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 1731 y(OUT)108 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 1856 y FB(int)23 b(MPI)p 245 1856 15 2 v 17 w(Create)p 406 1856 V 17 w(recv\(void*)f(buf,)h(int)h(count,)f(MPI)p 1139 1856 V 17 w(Datatype)f(datatype,)h(int)h(source,)393 1912 y(int)g(tag,)f(MPI)p 683 1912 V 17 w(Comm)g(comm,)g(MPI)p 1034 1912 V 17 w(Comm)p 1147 1912 V 17 w(request)g(*request\))75 1999 y(MPI)p 150 1999 V 17 w(CREATE)p 311 1999 V 16 w(RECV\(BUF,)g (COUNT,)g(DATATYPE,)f(SOURCE,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 2055 y()g(BUF\(*\))170 2112 y(INTEGER)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 2198 y FD(A)18 b(comm)o(unication)g(\(send)g(or)g(receiv)o(e\))g(that)f (uses)h(a)g(prede\014ned)h(request)f(is)h(initiated)g(b)o(y)f(the)75 2255 y(function)e Fs(MPI)p 338 2255 14 2 v 16 w(ST)l(ART)p FD(.)75 2359 y Fs(MPI)p 160 2359 V 16 w(ST)l(ART\(request\))117 2436 y Ft(INOUT)62 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 2561 y FB(int)23 b(MPI)p 245 2561 15 2 v 17 w(Start\(MPI)p 478 2561 V 16 w(Comm)p 590 2561 V 17 w(request)g(*request\))75 2647 y(MPI)p 150 2647 V 17 w(START\(REQUEST,)f(IERROR\))170 2704 y(INTEGER)h (REQUEST,)g(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 54 61 54 60 bop 75 -100 a FD(54)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FD(The)11 b(comm)o(unication)h(ob)s(ject)f(asso)q(ciated)g(with)h Fs(request)h FD(should)f(b)q(e)g(an)f(ob)s(ject)g(that)g(w)o(as)f (created)75 102 y(b)o(y)k(one)g(of)g(the)g(previous)h(four)e (functions,)i(so)f(that)f(all)i(the)f(comm)o(unication)h(argumen)o(ts)e (are)h(already)75 158 y(de\014ned.)23 b(A)16 b(send)g(can)g(b)q(e)h (started)e(pro)o(vided)h(that)f(the)h(previous)h(send)f(using)h(the)e (same)h(ob)s(ject)f(has)75 214 y(completed,)d(or)f(as)f(so)q(on)h(as)f (the)h(ob)s(ject)f(is)i(created,)f(if)g(it)g(has)g(not)g(y)o(et)f(b)q (een)i(used)f(in)h(a)f(comm)o(unication.)75 271 y(In)k(addition,)h(if)f (the)f(comm)o(unication)i(mo)q(de)e(is)h Fs(ready)g FD(then)g(a)f(matc) o(hing)h(receiv)o(e)g(should)h(b)q(e)f(p)q(osted.)75 327 y(The)j(send)h(bu\013er)f(should)g(not)g(b)q(e)h(accessed)f(after)f (the)h(send)h(is)f(started,)g(and)g(un)o(til)h(the)f(op)q(eration)75 384 y(completes.)166 442 y(A)c(receiv)o(e)g(can)g(b)q(e)g(started)f (pro)o(vided)i(that)e(the)h(preceding)h(receiv)o(e)f(using)h(the)f (same)f(ob)s(ject)g(has)75 498 y(completed,)f(or)f(as)f(so)q(on)h(as)f (the)h(ob)s(ject)f(is)i(created,)f(if)g(it)g(has)g(not)g(y)o(et)f(b)q (een)i(used)f(in)h(a)f(comm)o(unication.)75 555 y(The)18 b(receiv)o(e)h(bu\013er)f(should)h(not)e(b)q(e)i(accessed)g(after)e (the)h(receiv)o(e)h(is)f(started,)g(un)o(til)h(the)f(op)q(eration)75 611 y(completes.)166 669 y(The)23 b(call)g(is)g(non)o(blo)q(c)o(king,)i (with)e(similar)h(seman)o(tics)e(to)g(the)g(non)o(blo)q(c)o(king)i (comm)o(unication)75 726 y(op)q(erations)e(describ)q(ed)i(in)e(section) g(3.8.)39 b(I.e.,)23 b(a)e(call)i(to)e Fs(MPI)p 1218 726 14 2 v 16 w(ST)l(ART)h FD(starts)f(a)g(comm)o(unication)75 782 y(in)g(the)e(same)h(manner)g(as)f(a)g(call)i(to)e Fs(MPI)p 838 782 V 16 w(ISEND)p FD(,)h Fs(MPI)p 1099 782 V 15 w(IRSEND)p FD(,)g Fs(MPI)p 1388 782 V 16 w(ISSEND)g FD(or)f Fs(MPI)p 1721 782 V 16 w(IRECV)p FD(,)75 839 y(resp)q(ectiv)o(ely)l(.)166 979 y Fl(Discussion:)34 b Ft(It)14 b(has)g(b)q(een)h(suggested)g(that)f Fv(MPI)p 1005 979 13 2 v 14 w(IST)m(ART)f Ft(w)o(ould)g(mak)o(e)g(this)h(p)q (oin)o(t)f(b)q(etter)75 1167 y Fs(MPI)p 160 1167 14 2 v 16 w(ST)l(ART)l(ALL\(count,)j(a)o(rra)o(y)p 641 1167 V 15 w(of)p 693 1167 V 16 w(requests\))117 1246 y Ft(IN)155 b Fs(count)482 b Ft(list)14 b(length)f(\(in)o(teger\))117 1324 y(INOUT)62 b Fs(a)o(rra)o(y)p 416 1324 V 15 w(of)p 468 1324 V 16 w(requests)272 b Ft(arra)o(y)14 b(of)f(requests)j(\(arra) o(y)e(of)f(handle\))75 1450 y FB(int)23 b(MPI)p 245 1450 15 2 v 17 w(Startall\(int)f(count,)h(MPI)p 811 1450 V 17 w(Comm)p 924 1450 V 17 w(request)g(*array)p 1276 1450 V 16 w(of)p 1340 1450 V 17 w(requests\))75 1538 y(MPI)p 150 1538 V 17 w(STARTALL\(COUNT,)f(ARRAY)p 669 1538 V 16 w(OF)p 733 1538 V 17 w(REQUESTS,)h(IERROR\))170 1595 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1595 V 17 w(OF)p 716 1595 V 17 w(REQUESTS\(*\),)f(IERROR)166 1683 y FD(Start)i(all)j(comm)o (unications)f(asso)q(ciated)g(with)f(requests)h(in)g Fs(a)o(rra)o(y)p 1399 1683 14 2 v 15 w(of)p 1451 1683 V 16 w(requests)p FD(.)52 b(A)26 b(call)h(to)75 1739 y Fs(MPI)p 160 1739 V 16 w(ST)l(ART)l(ALL\(count,)76 b(a)o(rra)o(y)p 701 1739 V 14 w(of)p 752 1739 V 16 w(requests\))65 b FD(has)d(the)h(same)f(e\013ect)h(as)f(calls)i(to)75 1796 y Fs(MPI)p 160 1796 V 16 w(ST)l(ART\(a)o(rra)o(y)p 431 1796 V 15 w(of)p 483 1796 V 16 w(requests[i]\))p FD(,)17 b(executed)f(for)e Fs(i=0)i(,...,)d(count-1)p FD(,)j(in)g(some)f(arbitrary)f(order.)166 1854 y(A)j(comm)o(unication)g (started)f(with)h(a)g(call)h(to)e Fs(MPI)p 1063 1854 V 16 w(ST)l(ART)h FD(or)f Fs(MPI)p 1377 1854 V 16 w(ST)l(ART)l(ALL)i FD(is)f(completed)75 1910 y(b)o(y)11 b(a)h(call)g(to)f Fs(MPI)p 385 1910 V 16 w(W)l(AIT)p FD(,)g Fs(MPI)p 620 1910 V 16 w(TEST)p FD(,)h(or)f(one)g(of)g(the)h(deriv)o(ed)h(functions) f(describ)q(ed)h(in)g(section)f(3.8.5.)75 1967 y(These)k(comm)o (unication)h(completion)g(functions)g(do)f(not)f(deallo)q(cate)j(the)e (request,)g(and)g(this)g(can)g(b)q(e)75 2023 y(reused)g(anew)g(b)o(y)f (a)h Fs(MPI)p 521 2023 V 15 w(ST)l(ART)h FD(call.)22 b(The)16 b(ob)s(ject)f(needs)h(to)f(b)q(e)i(explicitly)h(deallo)q (cated)f(b)o(y)f(a)f(call)75 2079 y(to)j(the)g(function)i Fs(MPI)p 482 2079 V 16 w(REQUEST)p 701 2079 V 17 w(FREE)p FD(,)f(b)q(elo)o(w.)30 b(W)l(e)19 b(th)o(us)f(ha)o(v)o(e)g(t)o(w)o(o)g (t)o(yp)q(es)g(of)g(comm)o(unication)75 2136 y(requests:)34 b Fx(p)q(ersisten)o(t)23 b FD(requests,)h(whic)o(h)f(are)f(allo)q (cated)i(b)o(y)e(a)g(call)i(to)e Fs(MPI)p 1503 2136 V 15 w(CREA)l(TE)p 1687 2136 V 18 w(xxx)p FD(,)i(and)75 2192 y(are)18 b(explicitly)j(deallo)q(cated)e(b)o(y)f Fs(MPI)p 746 2192 V 16 w(REQUEST)p 965 2192 V 18 w(FREE)p FD(,)g(and)g Fx(ephemeral)g FD(requests)g(that)f(p)q(ersist)75 2249 y(for)f(one)i(comm)o(unication)g(only:)24 b(they)17 b(are)g(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)h(comm)o(unication)g (initiation)75 2305 y(function,)e(and)f(are)g(freed)g(b)o(y)h(the)f (comm)o(unication)h(completion)g(call.)166 2363 y(If)21 b(a)g(call)i(to)e Fs(MPI)p 498 2363 V 15 w(W)l(AIT)p FD(,)g Fs(MPI)p 742 2363 V 16 w(TEST)p FD(,)g(or)g(an)o(y)g(of)g(the)g (deriv)o(ed)i(functions,)g(is)f(executed)g(on)-2120 b Fm(CHANGE)75 2420 y FD(a)22 b(p)q(ersisten)o(t)h(comm)o(unication)g (request)f(for)g(whic)o(h)h(there)g(is)g(no)f(p)q(ending)i(comm)o (unication)f(\(e.g.)75 2476 y(immediately)13 b(after)e(it)h(w)o(as)f (created)h(or)f(immediately)i(after)e(a)h(call)g(to)f Fs(MPI)p 1378 2476 V 16 w(W)l(AIT)p FD(\))h(then)g(the)f(outcome)75 2533 y(is)18 b(similar)h(to)e(that)g(whic)o(h)i(obtains)f(when)g(a)f(n) o(ull)i(request)f(w)o(as)f(passed)h(as)f(argumen)o(t:)24 b Fs(MPI)p 1749 2533 V 16 w(W)l(AIT)75 2589 y FD(returns)13 b(immediately)l(,)i Fs(MPI)p 581 2589 V 16 w(TEST)e FD(returns)g(with)g Fv(\015ag)f(=)g(true)p FD(,)i(etc.)19 b(As)13 b(for)g(calls)h(with)f(n) o(ull)h(requests,)75 2646 y(the)h(status)g(is)g(up)q(dated)h(to)f(the)g (status)g(of)f(a)h(n)o(ull)i(comm)o(unication.)166 2704 y(In)g(particular,)g(one)f(can)g(pass)g(to)g Fs(MPI)p 858 2704 V 16 w(W)l(AIT)l(ANY)h FD(and)f(the)h(other)f(functions)h(in)g (section)g(3.8.5)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 55 62 55 61 bop 75 -100 a Fp(3.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)638 b FD(55)75 45 y(an)23 b Fs(a)o(rra)o(y)p 244 45 14 2 v 14 w(of)p 295 45 V 16 w(request)i FD(argumen)o(t)c(where) i(some)g(of)f(the)h(requests)g(are)f(p)q(ersisten)o(t)h(ones,)h(and)f (ha)o(v)o(e)75 102 y(the)c(desired)h(outcome:)28 b(the)19 b(call)h(will)h(ignore)e(those)g(requests)g(in)h(the)f(arra)o(y)f(for)g (whic)o(h)i(there)f(are)75 158 y(no)e(p)q(ending)i(comm)o(unications)e (\(in)h(the)f(same)g(manner)g(that)f(these)h(functions)h(ignore)f(n)o (ull)i(request)75 214 y(en)o(tries\).)h(Th)o(us,)15 b(the)g(last)g (example)h(on)f(page)g(43)g(w)o(orks)f(correctly)h(with)h(p)q(ersisten) o(t)g(requests,)e(to)q(o.)75 326 y Fs(MPI)p 160 326 V 16 w(REQUEST)p 379 326 V 18 w(FREE\(request\))117 412 y Ft(INOUT)62 b Fs(request)452 b Ft(comm)o(unication)11 b(request)k(\(handle\))75 544 y FB(int)23 b(MPI)p 245 544 15 2 v 17 w(Request)p 430 544 V 16 w(free\(MPI)p 638 544 V 17 w(Comm)p 751 544 V 16 w(request)g(*request\))75 639 y(MPI)p 150 639 V 17 w(REQUEST)p 335 639 V 16 w(FREE\(REQUEST,)f (IERROR\))170 695 y(INTEGER)h(REQUEST,)g(IERROR)166 790 y FD(Mark)c Fs(request)i FD(for)e(deallo)q(cation.)35 b(The)20 b(request)f(ob)s(ject)h(is)g(deallo)q(cated)h(when)f(there)g (are)f(no)75 846 y(p)q(ending)e(comm)o(unications)f(in)o(v)o(olving)g (this)g(ob)s(ject,)e(at)h(whic)o(h)h(p)q(oin)o(t)f(the)h(request)f(b)q (ecomes)h(n)o(ull.)166 911 y(The)j(call)g(is)g(non)o(blo)q(c)o(king.)31 b(It)18 b(is)h(p)q(ermissible)i(to)d(call)h Fs(MPI)p 1249 911 14 2 v 16 w(REQUEST)p 1468 911 V 18 w(FREE\(request\))h FD(after)75 967 y(a)f(comm)o(unication)g(that)f(uses)h Fs(request)i FD(has)e(b)q(een)h(initiated,)h(but)e(b)q(efore)g(it)g (has)g(completed.)32 b(The)75 1024 y(ob)s(ject)15 b(will)j(b)q(e)f (deallo)q(cated)g(after)e(the)h(comm)o(unication)h(completes.)22 b(It)16 b(is)h(erroneous)f(to)f(initiate)i(a)75 1080 y(comm)o(unication)f(with)f(a)g(request)h(that)e(has)h(b)q(een)h (freed.)166 1221 y Fl(Discussion:)47 b Ft(It)18 b(is)f(impractical)e (to)i(ha)o(v)o(e)g(a)g(dela)o(y)o(ed)h(\\n)o(ulli\014cation")d(of)h Fv(request)p Ft(.)30 b(If)17 b(MPI)h(ma)o(y)m(,)d(at)95 b Fm(CHANGE)75 1271 y Ft(some)15 b(later)h(time,)e(assign)i(n)o(ull)f (to)g Fv(request)p Ft(,)j(then)e(the)h(user)g(cannot)f(deallo)q(cate)f (this)h(v)n(ariable)f(un)o(til)g(it)g(is)h(set)75 1321 y(to)h(n)o(ull.)28 b(In)17 b(particular,)h(if)e(this)h(is)h(an)f (automatic)e(v)n(ariable)i(in)f(a)i(subroutine,)g(then)g(the)g(user)g (cannot)g(exit)75 1370 y(the)c(subroutine)h(un)o(til)d(the)j(v)n (ariable)d(is)h(n)o(ull.)k(Also,)c(the)i(MPI)e(system)h(needs)h(to)e (\\remem)o(b)q(er")f(the)j(address)g(of)75 1420 y Fv(request)p Ft(,)g(so)f(that)g(when)h(the)f(comm)o(unication)c(ob)r(ject)15 b(is)f(dallo)q(cated,)f(the)h(p)q(oin)o(ter)g(to)g(it)g(is)f(n)o (ulli\014ed.)166 1478 y(This)h(lea)o(v)o(es)g(three)h(c)o(hoices:)166 1536 y(\(1\))f(A)g(request)h(is)f(nev)o(er)h(set)g(to)e(n)o(ull.)166 1594 y(\(2\))k(The)g(request)i(handle)d(is)h(set)h(to)f(n)o(ull)f(b)o (y)g(the)i(call)e(to)h Fv(MPI)p 1205 1594 13 2 v 14 w(REQUEST)p 1407 1594 V 14 w(FREE)p Ft(,)e(whic)o(h)i(also)f(deallo-)75 1644 y(cates)j(the)g(request)h(ob)r(ject.)32 b(The)19 b(call)e(to)h Fv(MPI)p 865 1644 V 14 w(REQUEST)p 1067 1644 V 14 w(FREE)f Ft(should)h(come)g(after)g(the)h(corresp)q(onding)75 1694 y(comm)o(unication)10 b(has)k(completed.)166 1752 y(\(3\))d(The)h(request)h(handle)f(is)f(set)h(to)g(n)o(ull)e(b)o(y)h Fv(MPI)p 948 1752 V 15 w(COMM)p 1093 1752 V 14 w(FREE)p Ft(;)f(the)i(request)h(ob)r(ject)f(is)g(freed)g(only)f(after)75 1802 y(an)o(y)h(activ)o(e)g(comm)o(unicatio)o(n)d(with)j(this)g(ob)r (ject)h(terminates.)18 b(Th)o(us,)12 b(the)h(call)e(to)p 1385 1802 V 27 w Fv(FREE)g Ft(can)h(o)q(ccur)h(b)q(efore)g(the)75 1851 y(call)g(to)h Fv(MPI)p 279 1851 V 14 w(W)m(AIT)f Ft(that)h(completes)g(the)g(comm)o(unication.)166 1909 y(In)19 b(order)h(to)e(call)g(\014rst)i Fv(MPI)p 641 1909 V 14 w(REQUEST)p 843 1909 V 14 w(FREE)p Ft(,)e(next)h Fv(MPI)p 1161 1909 V 14 w(W)m(AIT)p Ft(,)f(the)i(user)g(will)d(need)j (to)f(mak)o(e)e(an)75 1959 y(explicit)c(cop)o(y)h(of)g Fv(request)p Ft(.)19 b(He)c(or)f(she)g(will)f(b)q(e)h(left)g(with)g(a)f (dangling)f(reference)17 b(after)d(the)g(ob)r(ject)h(is)f(freed.)166 2024 y(I)g(supp)q(ort)h(\(2\):)j(I)c(don't)f(see)i(m)o(uc)o(h)e(more)g (p)q(erformance)h(or)g(con)o(v)o(enience)h(in)f(\(3\),)f(and)h(I)g (don't)f(think)h(\(1\))75 2080 y(is)g(acceptable.)166 2227 y FD(A)h(correct)f(in)o(v)o(o)q(cation)i(of)e(the)h(functions)h (describ)q(ed)g(in)g(this)f(section)h(will)g(o)q(ccur)g(in)f(a)g (sequence)75 2284 y(of)g(the)g(form)166 2349 y Fx(Create)i FD(\()p Fx(Start)h(Complete)p FD(\))725 2332 y Fg(\003)762 2349 y Fx(F)l(ree)e Fw(;)166 2413 y FD(where)g FA(\003)g FD(indicates)i(zero)e(or)f(more)h(rep)q(etitions.)24 b(If)16 b(the)g(same)g(comm)o(unication)h(ob)s(ject)f(is)g(used)75 2470 y(in)k(sev)o(eral)f(concurren)o(t)g(threads,)g(it)g(is)h(the)f (user)g(resp)q(onsibilit)o(y)i(to)e(co)q(ordinate)g(calls)h(so)f(that)f (the)75 2526 y(correct)d(sequence)h(is)g(ob)q(ey)o(ed.)166 2591 y(A)c(send)g(op)q(eration)f(initiated)j(with)d Fs(MPI)p 874 2591 14 2 v 16 w(ST)l(ART)i FD(can)e(b)q(e)i(matc)o(hed)e(with)h (an)o(y)f(receiv)o(e)i(op)q(eration)75 2647 y(and,)g(lik)o(ewise,)h(a)e (receiv)o(e)i(op)q(eration)e(initiated)i(with)f Fs(MPI)p 1098 2647 V 16 w(ST)l(ART)g FD(can)g(receiv)o(e)g(messages)f(generated) 75 2704 y(b)o(y)j(an)o(y)g(send)h(op)q(eration.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 56 63 56 62 bop 75 -100 a FD(56)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fq(3.11)59 b(Send-receive)75 147 y FD(The)18 b Fx(send-receiv)o(e)g FD(op)q(erations)g(com)o(bine)h(in)g(one)f(call)h(the)f(sending)i(of)d (a)h(message)g(to)f(one)h(desti-)75 204 y(nation)e(and)g(the)g (receiving)i(of)d(another)h(message,)f(from)g(another)h(destination,)g (p)q(ossibly)i(the)e(same.)75 260 y(A)k(send-receiv)o(e)j(op)q(eration) d(is)h(v)o(ery)f(useful)i(for)d(executing)j(a)e(shift)h(op)q(eration)f (across)g(a)g(c)o(hain)h(of)75 317 y(pro)q(cesses.)f(If)13 b(blo)q(c)o(king)h(sends)g(and)f(receiv)o(es)h(are)f(used)g(for)g(suc)o (h)g(shift,)g(then)h(one)f(needs)h(to)e(order)h(the)75 373 y(sends)k(and)g(receiv)o(es)h(correctly)f(\(e.g.,)e(ev)o(en)j(pro)q (cesses)f(send,)g(next)g(receiv)o(e,)h(o)q(dd)f(pro)q(cesses)g(receiv)o (e)75 430 y(\014rst,)g(next)g(send\))g(so)g(as)f(to)h(prev)o(en)o(t)f (cyclic)j(dep)q(endencies)h(that)c(ma)o(y)h(lead)g(to)g(deadlo)q(c)o (k.)26 b(When)17 b(a)75 486 y(send-receiv)o(e)d(op)q(eration)e(is)g (used,)h(the)f(comm)o(unication)h(subsystem)f(tak)o(es)f(care)g(of)h (these)g(issues.)20 b(The)75 542 y(send-receiv)o(e)14 b(op)q(eration)f(can)f(b)q(e)h(used)g(in)h(conjunction)f(with)g(the)f (functions)i(describ)q(ed)g(in)f(Chapter)f(6)75 599 y(in)19 b(order)f(to)g(p)q(erform)g(shifts)h(on)f(v)m(arious)h(logical)h(top)q (ologies.)30 b(Also,)19 b(a)f(send-receiv)o(e)i(op)q(eration)e(is)75 655 y(useful)e(for)f(implemen)o(ting)i(remote)e(pro)q(cedure)h(calls.) 166 712 y(A)g(message)f(sen)o(t)g(b)o(y)h(a)f(send-receiv)o(e)j(op)q (eration)d(can)h(b)q(e)h(receiv)o(ed)f(b)o(y)g(a)f(regular)h(receiv)o (e)h(op)q(er-)-2120 b Fm(CHANGE)75 769 y FD(ation)16 b(or)f(prob)q(ed)i(b)o(y)f(a)g(prob)q(e)g(op)q(eration;)g(a)g (send-receiv)o(e)i(op)q(eration)e(can)g(receiv)o(e)h(a)f(message)f(sen) o(t)75 825 y(b)o(y)g(a)g(regular)g(send)h(op)q(eration.)75 929 y Fs(MPI)p 160 929 14 2 v 16 w(SENDRECV\(sendbuf,)24 b(sendcount,)f(sendt)o(yp)q(e,)g(dest,)f(sendtag,)g(recvbuf,)g (recvcount,)f(recvt)o(yp)q(e,)75 986 y(source,)15 b(recvtag,)g(comm)m (,)d(status\))117 1063 y Ft(IN)155 b Fs(sendbuf)444 b Ft(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1139 y(IN)155 b Fs(sendcount)399 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1215 y(IN)155 b Fs(sendt)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(bu\013er)g(\(handle\))117 1291 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1366 y(IN)155 b Fs(sendtag)443 b Ft(send)15 b(tag)e(\(in)o(teger\))117 1442 y(OUT)108 b Fs(recvbuf)450 b Ft(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\)) 117 1518 y(IN)155 b Fs(recvcount)405 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1594 y(IN)155 b Fs(recvt)o(yp)q(e)430 b Ft(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(receiv)o(e)i(bu\013er)g(\(handle\))117 1670 y(IN)155 b Fs(source)468 b Ft(rank)14 b(of)f(source)j(\(in)o (teger\))117 1745 y(IN)155 b Fs(recvtag)449 b Ft(receiv)o(e)15 b(tag)f(\(in)o(teger\))117 1821 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 1897 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\))75 2022 y FB(int)23 b(MPI)p 245 2022 15 2 v 17 w(Sendrecv\(void)f (*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 2022 V 17 w(Datatype)f(sendtype,)393 2078 y(int)i(dest,)f(int)g(sendtag,)g(void)g (*recvbuf,)g(int)g(recvcount,)393 2135 y(MPI)p 468 2135 V 17 w(Datatype)g(recvtype,)f(int)i(source,)f(MPI)p 1297 2135 V 16 w(Datatype)g(recvtag,)393 2191 y(MPI)p 468 2191 V 17 w(Comm)g(comm,)g(MPI)p 819 2191 V 17 w(Status)g(*status\))75 2278 y(MPI)p 150 2278 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVBUF,)393 2335 y(RECVCOUNT,)h (RECVTYPE,)f(SOURCE,)h(RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 2391 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2447 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h (RECVTYPE,)170 2504 y(SOURCE,)g(RECV)h(TAG,)f(COMM,)g(STATUS\(MPI)p 986 2504 V 16 w(STATUS)p 1146 2504 V 16 w(SIZE\),)g(IERROR)166 2591 y FD(Execute)c(a)e(blo)q(c)o(king)j(send)f(and)f(receiv)o(e)h(op)q (eration.)29 b(Both)18 b(send)h(and)f(receiv)o(e)h(use)g(the)f(same)75 2647 y(comm)o(unicator,)c(but)g(p)q(ossibly)i(di\013eren)o(t)f(tags.)j (The)d(send)g(bu\013er)f(and)h(receiv)o(e)g(bu\013ers)g(are)f(disjoin)o (t,)75 2704 y(and)h(ma)o(y)g(ha)o(v)o(e)g(di\013eren)o(t)g(lengths)h (and)f(datat)o(yp)q(es.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 57 64 57 63 bop 75 -100 a Fp(3.11.)34 b(SEND-RECEIVE)1277 b FD(57)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(SENDRECV)p 408 45 V 17 w(REPLA)o(CE\(buf,)19 b(count,)f(datat)o(yp)q(e,)h(dest,)f (sendtag,)h(source,)f(recvtag,)f(comm)m(,)d(sta-)75 102 y(tus\))117 181 y Ft(INOUT)62 b Fs(buf)527 b Ft(initial)12 b(address)j(of)f(send)g(and)g(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 259 y(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)9 b(of)g(elemen)o(ts)h (in)f(send)i(and)e(receiv)o(e)i(bu\013er)g(\(in)o(teger\))117 338 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(and)f(receiv)o(e)h(bu\013er)g (\(handle\))117 417 y(IN)155 b Fs(dest)511 b Ft(rank)14 b(of)f(destination)h(\(in)o(teger\))117 495 y(IN)155 b Fs(sendtag)443 b Ft(send)15 b(message)f(tag)f(\(in)o(teger\))117 574 y(IN)155 b Fs(source)468 b Ft(rank)14 b(of)f(source)j(\(in)o (teger\))117 653 y(IN)155 b Fs(recvtag)449 b Ft(receiv)o(e)15 b(message)f(tag)f(\(in)o(teger\))117 731 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 810 y(OUT)108 b Fs(status)476 b Ft(status)15 b(ob)r(ject)g(\(Status\))75 936 y FB(int)23 b(MPI)p 245 936 15 2 v 17 w(Sendrecv)p 454 936 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 936 V 17 w(Datatype)g(datatype,)393 993 y(int)h(dest,)f(int)g (sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 993 V 16 w(Comm)h(comm,)393 1049 y(MPI)p 468 1049 V 17 w(Status)f(*status\))75 1138 y(MPI)p 150 1138 V 17 w(SENDRECV)p 359 1138 V 16 w(REPLACE\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g (SOURCE,)g(RECVTAG,)393 1194 y(COMM,)g(STATUS,)g(IERROR\))170 1251 y()g(BUF\(*\))170 1307 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)g(COMM,)170 1363 y(STATUS\(MPI)p 413 1363 V 16 w(STATUS)p 573 1363 V 17 w(SIZE\),)g(IERROR)166 1452 y FD(Execute)15 b(a)f(blo)q(c)o(king)i (send)f(and)g(receiv)o(e;)g(the)g(same)f(bu\013er)g(is)h(used)h(b)q (oth)e(for)g(the)h(send)g(and)g(for)75 1508 y(the)g(receiv)o(e,)h(so)f (that)f(the)i(message)e(sen)o(t)h(is)h(replaced)g(b)o(y)g(the)f (message)g(receiv)o(ed.)166 1566 y(The)f(seman)o(tics)g(of)f(a)h (send-receiv)o(e)h(op)q(eration)f(is)g(what)g(w)o(ould)g(obtain)g(if)g (the)g(caller)h(fork)o(ed)e(t)o(w)o(o)75 1623 y(concurren)o(t)19 b(threads,)f(one)h(to)f(execute)h(the)g(send,)g(and)g(one)g(to)e (execute)j(the)e(receiv)o(e,)i(follo)o(w)o(ed)f(b)o(y)75 1679 y(a)e(join)g(of)g(these)g(t)o(w)o(o)f(threads.)25 b(Additional)19 b(in)o(termediate)f(bu\013ering)g(is)f(needed)i(for)d (the)i(\\replace")75 1736 y(v)m(arian)o(t.)166 1794 y(A)11 b(send-receiv)o(e)i(cannot)f(b)q(e)g(implemen)o(ted)h(b)o(y)e(a)g(blo)q (c)o(king)i(send)f(follo)o(w)o(ed)g(b)o(y)f(a)g(blo)q(c)o(king)i (receiv)o(e)75 1850 y(or)i(a)g(blo)q(c)o(king)i(receiv)o(e,)f(follo)o (w)o(ed)g(b)o(y)g(a)f(blo)q(c)o(king)i(send.)22 b(Consider)16 b(the)f(follo)o(wing)i(co)q(de,)e(where)h(t)o(w)o(o)75 1907 y(pro)q(cesses)g(exc)o(hange)f(messages:)75 2024 y FB(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2080 y(IF)h(\(rank.EQ.0\))123 2137 y(THEN)170 2193 y(CALL)g (MPI_SENDRECV\(send_buff,)c(send_count,)j(MPI_REAL,)f(1,)i(send_tag,) 242 2250 y(recv_buff,)f(recv_count,)f(MPI_REAL,)h(1,)g(recv_tag,)g (comm,)g(status,)g(ierr\))123 2306 y(ELSE)170 2363 y(CALL)h (MPI_SENDRECV\(send_buff,)c(send_count,)j(MPI_REAL,)f(0,)i(send_tag,) 242 2419 y(recv_buff,)f(recv_count,)f(MPI_REAL,)h(0,)g(tag,)g(comm,)h (status,)f(ierr\))75 2475 y(END)g(IF)166 2591 y FD(If)16 b(the)f(send)h(receiv)o(es)h(are)e(replaced)i(either)f(b)o(y)f(blo)q(c) o(king)i(send,)f(follo)o(w)o(ed)g(b)o(y)g(blo)q(c)o(king)g(receiv)o(e,) 75 2647 y(or)g(blo)q(c)o(king)h(receiv)o(e,)g(follo)o(w)o(ed)g(b)o(y)f (blo)q(c)o(king)i(send,)f(then)f(the)h(co)q(de)g(ma)o(y)e(deadlo)q(c)o (k.)24 b(On)17 b(the)f(other)75 2704 y(hand,)f(send-receiv)o(e)i(can)f (b)q(e)f(implemen)o(ted)i(using)f(non)o(blo)q(c)o(king)h(sends)e(and)h (receiv)o(es.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 58 65 58 64 bop 75 -100 a FD(58)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fq(3.12)59 b(Null)21 b(p)n(ro)r(cesses)75 147 y FD(In)15 b(man)o(y)e(instances,)i (it)f(is)h(con)o(v)o(enien)o(t)g(to)e(sp)q(ecify)j(a)d(\\dumm)o(y")h (source)g(or)g(destination)h(for)e(comm)o(u-)75 203 y(nication.)29 b(This)18 b(simpli\014es)i(the)e(co)q(de)g(that)f(is)h(needed)i(for)d (dealing)i(with)f(b)q(oundaries,)h(e.g.,)e(in)i(the)75 259 y(case)c(of)g(a)g(non-circular)i(shift)e(done)h(with)f(calls)i(to)d (send-receiv)o(e.)166 316 y(The)f(sp)q(ecial)j(v)m(alue)26 b Fv(MPI)p 608 316 13 2 v 15 w(PROCNULL)12 b FD(can)i(b)q(e)g(used)g (instead)g(of)f(a)g(rank)g(wherev)o(er)g(a)g(source)g(or)g(a)75 372 y(destination)j(argumen)o(t)f(is)g(required)i(in)f(a)f(call.)21 b(A)15 b(comm)o(unication)h(with)g(pro)q(cess)29 b Fv(MPI)p 1649 372 V 15 w(PROCNULL)75 429 y FD(has)13 b(no)h(e\013ect:)k(a)13 b(send)h(to)25 b Fv(MPI)p 633 429 V 14 w(PROCNULL)13 b FD(succeeds)i(and)e(returns)g(as)g(so)q(on)h(as)f(p)q(ossible.)21 b(A)13 b(receiv)o(e)75 485 y(from)26 b Fv(MPI)p 271 485 V 14 w(PROCNULL)13 b FD(succeeds)i(and)f(returns)g(as)f(so)q(on)h(as)f (p)q(ossible)j(with)e(no)g(mo)q(di\014cations)h(to)e(the)75 542 y(receiv)o(e)18 b(bu\013er.)25 b(When)17 b(a)g(receiv)o(e)h(with)f Fs(source)h FD(=)32 b Fv(MPI)p 1090 542 V 15 w(PROCNULL)16 b FD(is)i(executed)f(then)h(the)f(status)75 598 y(ob)s(ject)e(returns) 30 b Fs(source)16 b FD(=)29 b Fv(MPI)p 661 598 V 14 w(PROCNULL)p FD(,)14 b Fs(tag)h FD(=)30 b Fv(MPI)p 1132 598 V 14 w(ANY)p 1231 598 V 15 w(T)m(A)o(G)14 b FD(and)h Fs(count=0)p FD(.)75 741 y Fq(3.13)59 b(Derived)19 b(datat)n(yp)r(es)75 843 y FD(Up)c(to)f(no)o(w,)g(all)i(p)q(oin)o(t)f(to)f(p)q(oin)o(t)h (comm)o(unication)h(in)o(v)o(olv)o(ed)f(only)g(con)o(tiguous)g (bu\013ers)g(con)o(taining)g(a)75 899 y(sequence)g(of)e(elemen)o(ts)h (of)f(the)g(same)g(t)o(yp)q(e.)19 b(This)14 b(is)g(to)q(o)f (constraining)h(on)f(t)o(w)o(o)f(accoun)o(ts:)19 b(One)14 b(often)75 956 y(w)o(an)o(ts)f(to)h(pass)g(messages)g(that)f(con)o (tain)i(v)m(alues)g(with)g(di\013eren)o(t)f(datat)o(yp)q(es)g(\(e.g.,)f (an)h(in)o(teger)h(coun)o(t,)75 1012 y(follo)o(w)o(ed)i(b)o(y)f(a)g (sequence)i(of)e(real)h(n)o(um)o(b)q(ers\);)f(and)h(one)g(often)f(w)o (an)o(ts)f(to)h(send)h(noncon)o(tiguous)g(data)75 1069 y(\(e.g.)h(a)12 b(sub-blo)q(c)o(k)i(of)e(a)g(matrix\).)18 b(One)13 b(solution)g(is)g(to)f(pro)o(vide)h(functions)g(that)e(pac)o (k)i(noncon)o(tiguous)75 1125 y(data)f(in)o(to)h(a)g(con)o(tiguous)g (bu\013er)g(at)f(the)h(sender)h(site)f(and)g(unpac)o(k)h(it)f(bac)o(k)g (at)f(the)h(receiv)o(er)h(site.)19 b(This)75 1182 y(has)f(the)g(disadv) m(an)o(tage)g(of)g(requiring)h(additional)h(memory-to-memory)c(cop)o(y) i(op)q(erations)g(at)g(b)q(oth)75 1238 y(sites,)12 b(ev)o(en)f(when)g (the)g(comm)o(unication)g(subsystem)g(has)f(scatter-gather)g (capabilities.)20 b(Instead,)12 b(MPI)75 1294 y(pro)o(vides)22 b(mec)o(hanisms)f(to)g(sp)q(ecify)h(more)f(general,)i(mixed)f(and)f (noncon)o(tiguous)g(comm)o(unication)75 1351 y(bu\013ers.)29 b(It)19 b(is)g(up)g(to)f(the)g(implemen)o(tation)i(to)e(decide)i (whether)e(data)g(should)h(b)q(e)h(\014rst)e(pac)o(k)o(ed)g(in)75 1407 y(a)e(con)o(tiguous)h(bu\013er)g(b)q(efore)g(b)q(eing)h (transmitted,)f(or)f(whether)h(it)g(can)g(b)q(e)g(collected)i(directly) f(from)75 1464 y(where)d(it)h(resides.)166 1520 y(The)e(general)h(mec)o (hanisms)g(pro)o(vided)g(here)f(allo)o(w)h(one)f(to)g(transfer)f (directly)l(,)j(without)e(cop)o(ying,)75 1577 y(ob)s(jects)j(of)g(v)m (arious)h(shap)q(e)h(and)f(size.)28 b(It)17 b(is)i(not)e(assumed)h (that)f(the)g(MPI)h(library)g(is)h(cognizan)o(t)e(of)75 1633 y(the)f(ob)s(jects)g(declared)h(in)g(the)f(host)g(language;)g(th)o (us,)g(if)g(one)g(w)o(an)o(ts)f(to)h(transfer)f(a)h(structure,)f(or)h (an)75 1690 y(arra)o(y)d(section,)i(it)f(will)i(b)q(e)f(necessary)f(to) g(pro)o(vide)h(in)g(MPI)f(a)g(de\014nition)i(of)e(a)g(comm)o(unication) h(bu\013er)75 1746 y(that)e(mimics)j(the)e(de\014nition)i(of)e(the)g (structure)g(or)f(arra)o(y)g(section)i(in)g(question.)20 b(These)15 b(facilities)h(can)75 1803 y(b)q(e)j(used)g(b)o(y)f(library) h(designers)h(to)d(de\014ne)j(comm)o(unication)f(functions)g(that)f (can)g(transfer)g(ob)s(jects)75 1859 y(de\014ned)g(in)f(the)f(host)g (language)g({)g(b)o(y)h(deco)q(ding)g(their)g(de\014nitions)h(as)e(a)o (v)m(ailable)i(in)f(a)f(sym)o(b)q(ol)h(table)75 1915 y(or)e(a)g(dop)q(e)g(v)o(ector.)20 b(Suc)o(h)15 b(higher-lev)o(el)j (comm)o(unication)e(functions)g(are)f(not)g(part)f(of)h(MPI.)166 1972 y(More)d(general)g(comm)o(unication)i(bu\013ers)e(are)g(sp)q (eci\014ed)i(b)o(y)f(replacing)g(the)g(basic)g(datat)o(yp)q(es)f(that) 75 2028 y(ha)o(v)o(e)e(b)q(een)i(used)f(so)f(far)g(with)h(deriv)o(ed)g (datat)o(yp)q(es)f(that)g(are)g(constructed)g(from)g(basic)h(datat)o (yp)q(es)f(using)75 2085 y(the)j(constructors)e(describ)q(ed)j(in)g (this)f(section.)19 b(These)13 b(metho)q(ds)f(of)g(constructing)h (deriv)o(ed)h(datat)o(yp)q(es)75 2141 y(can)h(b)q(e)h(applied)h (recursiv)o(ely)l(.)166 2198 y(A)e Fx(general)j(datat)o(yp)q(e)e FD(is)g(an)f(opaque)h(ob)s(ject)e(that)h(sp)q(eci\014es)i(t)o(w)o(o)d (things:)143 2291 y FA(\017)23 b FD(A)15 b(sequence)h(of)f(basic)h (datat)o(yp)q(es)143 2385 y FA(\017)23 b FD(A)15 b(sequence)h(of)f(in)o (teger)g(\(b)o(yte\))g(displacemen)o(ts)166 2478 y(The)22 b(displacemen)o(ts)g(are)g(not)f(required)h(to)f(b)q(e)h(p)q(ositiv)o (e,)i(distinct,)g(or)d(in)h(increasing)h(order;)75 2534 y(therefore)17 b(the)g(order)g(of)f(items)i(need)g(not)f(coincide)i (with)e(their)h(order)f(in)g(store,)g(and)g(an)g(item)h(ma)o(y)75 2591 y(app)q(ear)c(more)g(than)g(once.)20 b(W)l(e)14 b(call)h(suc)o(h)g(pair)f(of)g(sequences)h(\(or)e(sequence)j(of)d (pairs\))h(a)g Fx(t)o(yp)q(e)i(map)p FD(.)75 2647 y(The)22 b(sequence)g(of)f(basic)h(datat)o(yp)q(es)f(\(displacemen)o(ts)h (ignored\))g(is)f(the)h Fx(t)o(yp)q(e)i(signature)e FD(of)f(the)75 2704 y(datat)o(yp)q(e.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 59 66 59 65 bop 75 -100 a Fp(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FD(59)166 45 y(Let)189 142 y Fw(T)6 b(y)r(pemap)12 b FD(=)h FA(f)p FD(\()p Fw(ty)r(pe)562 149 y Fi(0)581 142 y Fw(;)8 b(disp)686 149 y Fi(0)705 142 y FD(\))p Fw(;)g(:::;)g FD(\()o Fw(ty)r(pe)905 149 y Ff(n)p Fg(\000)p Fi(1)971 142 y Fw(;)g(disp)1076 149 y Ff(n)p Fg(\000)p Fi(1)1144 142 y FD(\))p FA(g)p Fw(;)75 239 y FD(b)q(e)16 b(suc)o(h)f(a)g(t)o(yp)q(e)h(map,)e(where)i Fw(ty)r(pe)709 246 y Ff(i)738 239 y FD(are)f(basic)h(t)o(yp)q(es,)f(and)g Fw(disp)1233 246 y Ff(i)1262 239 y FD(are)g(displacemen)o(ts.)21 b(Let)189 336 y Fw(T)6 b(y)r(pesig)13 b FD(=)g FA(f)p Fw(ty)r(pe)517 343 y Fi(0)537 336 y Fw(;)8 b(:::;)g(ty)r(pe)702 343 y Ff(n)p Fg(\000)p Fi(1)768 336 y FA(g)75 433 y FD(b)q(e)14 b(the)f(asso)q(ciated)h(t)o(yp)q(e)f(signature.)19 b(This)14 b(t)o(yp)q(e)f(map,)g(together)g(with)g(a)g(base)g(address)h Fo(buf)p FD(,)f(sp)q(eci\014es)75 489 y(a)19 b(comm)o(unication)i (bu\013er:)28 b(the)20 b(comm)o(unication)h(bu\013er)e(that)g(consists) h(of)f Fw(n)h FD(en)o(tries,)h(where)f(the)75 546 y Fw(i)p FD(-th)e(en)o(try)g(is)g(at)g(address)g Fw(buf)f FD(+)12 b Fw(disp)777 553 y Ff(i)809 546 y FD(and)19 b(has)f(t)o(yp)q(e)g Fw(ty)r(pe)1174 553 y Ff(i)1188 546 y FD(.)29 b(A)19 b(message)e(assem)o(bled)i(from)f(suc)o(h)75 602 y(comm)o(unication)e (bu\013er)f(will)i(consist)e(of)g Fw(n)h FD(v)m(alues,)g(of)e(the)i(t)o (yp)q(es)f(de\014ned)h(b)o(y)g Fw(T)6 b(y)r(pesig)r FD(.)166 659 y(W)l(e)11 b(can)f(use)h(a)f(handle)i(to)e(a)g(general)h(datat)o (yp)q(e)f(as)h(an)f(argumen)o(t)g(in)h(a)f(send)i(or)e(receiv)o(e)h(op) q(eration,)75 716 y(in)j(replacemen)o(t)f(of)f(a)h(basic)g(datat)o(yp)q (e)g(argumen)o(t.)18 b(The)13 b(op)q(eration)g Fs(MPI)p 1359 716 14 2 v 16 w(SEND\(buf,)h(1,)e(datat)o(yp)q(e,...\))75 772 y FD(will)j(use)e(the)h(send)g(bu\013er)f(de\014ned)h(b)o(y)g(the)f (base)g(address)h Fs(buf)g FD(and)f(the)g(general)h(datat)o(yp)q(e)f (asso)q(ciated)75 828 y(with)f Fs(datat)o(yp)q(e)p FD(;)i(it)e(will)i (generate)d(a)h(message)f(with)h(the)g(t)o(yp)q(e)g(signature)g (determined)h(b)o(y)e(the)h Fs(datat)o(yp)q(e)75 885 y FD(argumen)o(t.)28 b Fs(MPI)p 390 885 V 16 w(RECV\(buf,)20 b(1,)e(datat)o(yp)q(e,...\))29 b FD(will)20 b(use)e(the)h(receiv)o(e)g (bu\013er)f(de\014ned)h(b)o(y)g(the)f(base)75 941 y(address)d Fs(buf)h FD(and)g(the)f(general)h(datat)o(yp)q(e)e(asso)q(ciated)i (with)f Fs(datat)o(yp)q(e)p FD(.)166 999 y(General)h(datat)o(yp)q(es)e (can)i(b)q(e)g(used)f(in)h(all)h(send)e(and)h(receiv)o(e)g(op)q (erations.)k(W)l(e)c(address)f(later)g(in)75 1055 y(section)h(3.13.5)d (the)i(case)h(where)f(the)g(second)h(argumen)o(t)e Fs(count)j FD(has)e(v)m(alue)h Fw(>)d FD(1.)166 1112 y(The)e(basic)g(datat)o(yp)q (es)f(presen)o(ted)i(in)f(section)g(3.2.1)f(are)g(particular)h(cases)g (of)f(a)h(general)g(datat)o(yp)q(e,)75 1169 y(and)19 b(are)f(prede\014ned.)32 b(Th)o(us,)19 b Fs(MPI)p 711 1169 V 16 w(INT)f FD(is)h(a)g(prede\014ned)h(handle)g(to)e(a)g(datat)o (yp)q(e)g(with)h(t)o(yp)q(e)g(map)75 1225 y FA(f)p FD(\()p Fs(int)p Fw(;)8 b FD(0\))p FA(g)p FD(,)j(with)i(one)g(en)o(try)f(of)g (t)o(yp)q(e)h Fs(int)g FD(and)f(displacemen)o(t)i(zero.)19 b(And)13 b(similarly)i(for)d(all)h(other)f(basic)75 1281 y(datat)o(yp)q(es.)166 1339 y(The)j Fx(exten)o(t)f FD(of)g(a)g(datat)o (yp)q(e)g(is)h(de\014ned)g(to)f(b)q(e)h(the)g(span)f(from)g(the)g (\014rst)g(b)o(yte)h(to)e(the)i(last)f(b)o(yte)75 1395 y(o)q(ccupied)19 b(b)o(y)d(en)o(tries)h(in)h(this)f(datat)o(yp)q(e,)f (rounded)h(up)g(to)f(satisfy)h(alignmen)o(t)g(requiremen)o(ts.)25 b(That)75 1452 y(is,)15 b(if)189 1549 y Fw(T)6 b(y)r(pemap)12 b FD(=)h FA(f)p FD(\()p Fw(ty)r(pe)562 1556 y Fi(0)581 1549 y Fw(;)8 b(disp)686 1556 y Fi(0)705 1549 y FD(\))p Fw(;)g(:::;)g FD(\()o Fw(ty)r(pe)905 1556 y Ff(n)p Fg(\000)p Fi(1)971 1549 y Fw(;)g(disp)1076 1556 y Ff(n)p Fg(\000)p Fi(1)1144 1549 y FD(\))p FA(g)p Fw(;)75 1645 y FD(then)283 1741 y Fw(l)q(b)p FD(\()p Fw(T)e(y)r(pemap)p FD(\))40 b(=)h(min)688 1770 y Ff(j)742 1741 y Fw(disp)826 1748 y Ff(j)844 1741 y Fw(;)272 1831 y(ub)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))40 b(=)h(max)693 1860 y Ff(j)743 1831 y FD(\()p Fw(disp)845 1838 y Ff(j)873 1831 y FD(+)10 b Fw(siz)r(eof)5 b FD(\()p Fw(ty)r(pe)1150 1838 y Ff(j)1169 1831 y FD(\)\))p Fw(;)21 b FD(and)189 1921 y Fw(extent)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))42 b(=)f Fw(ub)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))j FA(\000)h Fw(l)q(b)p FD(\()p Fw(T)c(y)r(pemap)p FD(\))j(+)h Fw(\017:)456 b FD(\(3.1\))75 2017 y(If)17 b Fw(ty)r(pe)206 2024 y Ff(i)237 2017 y FD(requires)g(alignmen)o(t)g(to)f(a)h(b)o(yte)f(address)h(that)f(is)h (is)g(a)f(m)o(ultiple)j(of)d Fw(k)1474 2024 y Ff(i)1488 2017 y FD(,)g(then)h Fw(\017)g FD(is)g(the)g(least)75 2074 y(nonnegativ)o(e)f(incremen)o(t)g(needed)g(to)f(round)g Fw(extent)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))16 b(to)e(the)h(next)h (m)o(ultiple)h(of)d(max)1794 2081 y Ff(i)1816 2074 y Fw(k)1840 2081 y Ff(i)1854 2074 y FD(.)166 2131 y(Example:)23 b(Assume)17 b(that)f Fw(T)6 b(y)r(pe)14 b FD(=)h FA(f)p FD(\()p Fs(double)q Fw(;)8 b FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(8\))p FA(g)15 b FD(\(a)h Fs(double)h FD(at)f(displacemen)o(t)i(zero,)75 2187 y(follo)o(w)o(ed)g(b)o(y)g(a)f Fs(cha)o(r)g FD(at)g(displacemen)o(t)j(eigh)o(t\).)27 b(Assume,)18 b(furthermore,)f(that)g(doubles)i(ha)o(v)o(e)e(to)g(b)q(e) 75 2244 y(strictly)e(aligned)g(at)f(addresses)h(that)e(are)h(m)o (ultiple)i(of)e(eigh)o(t.)20 b(Then,)15 b(the)f(exten)o(t)g(of)g(this)h (datat)o(yp)q(e)e(is)75 2300 y(16)d(\(9)g(rounded)h(to)f(the)h(next)g (m)o(ultiple)h(of)f(8\).)17 b(A)11 b(datat)o(yp)q(e)f(that)g(consists)h (of)f(a)g(c)o(haracter)g(immediately)75 2357 y(follo)o(w)o(ed)16 b(b)o(y)f(a)g(double)h(will)h(also)e(ha)o(v)o(e)g(an)g(exten)o(t)g(of)g (16.)189 2466 y Fo(R)n(ationale.)39 b FD(The)15 b(de\014nition)h(of)e (exten)o(t)g(is)h(motiv)m(ated)f(b)o(y)h(the)f(assumption)h(that)e(the) i(amoun)o(t)189 2523 y(of)i(padding)i(added)g(at)e(the)h(end)g(of)g (eac)o(h)g(structure)f(in)i(an)f(arra)o(y)e(of)i(structures)f(is)i(the) f(least)189 2579 y(needed)d(to)e(ful\014ll)j(alignmen)o(t)e(constrain)o (ts.)19 b(More)14 b(explicit)i(con)o(trol)d(of)g(the)h(exten)o(t)g(is)g (pro)o(vided)189 2636 y(in)i(section)g(3.13.3.)k(Suc)o(h)c(explicit)i (con)o(trol)e(is)g(needed)h(in)f(cases)g(where)g(the)g(assumption)g(do) q(es)189 2692 y(not)e(hold,)i(e.g.)j(where)d(union)g(t)o(yp)q(es)f(are) g(used.)21 b(\()p Fo(End)15 b(of)i(r)n(ationale.)p FD(\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 60 67 60 66 bop 75 -100 a FD(60)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fn(3.13.1)49 b(Datat)o(yp)q(e)17 b(constructo)o(rs)75 145 y Fs(Contiguous)47 b FD(The)16 b(simplest)i(datat)o(yp)q(e)e(constructor)f(is)i Fs(MPI)p 1171 145 14 2 v 16 w(TYPE)p 1304 145 V 17 w(CONTIGUOUS)h FD(whic)o(h)f(allo)o(ws)75 201 y(replication)g(of)e(a)g(datat)o(yp)q(e) f(in)o(to)h(con)o(tiguous)h(lo)q(cations.)75 312 y Fs(MPI)p 160 312 V 16 w(TYPE)p 293 312 V 17 w(CONTIGUOUS\(count,)h(oldt)o(yp)q (e,)f(newt)o(yp)q(e\))117 397 y Ft(IN)155 b Fs(count)482 b Ft(replication)14 b(coun)o(t)g(\(nonnegativ)o(e)g(in)o(teger\))117 486 y(IN)155 b Fs(oldt)o(yp)q(e)450 b Ft(old)13 b(datat)o(yp)q(e)i (\(handle\))117 576 y(OUT)108 b Fs(newt)o(yp)q(e)433 b Ft(new)15 b(datat)o(yp)q(e)f(\(handle\))75 707 y FB(int)23 b(MPI)p 245 707 15 2 v 17 w(Type)p 358 707 V 17 w(contiguous\(int)f (count,)h(MPI)p 972 707 V 17 w(Datatype)f(oldtype,)393 764 y(MPI)p 468 764 V 17 w(Datatype)h(*newtype\))75 857 y(MPI)p 150 857 V 17 w(TYPE)p 263 857 V 16 w(CONTIGUOUS\(COUNT,)f (OLDTYPE,)h(NEWTYPE,)f(IERROR\))170 914 y(INTEGER)h(COUNT,)g(OLDTYPE,)g (NEWTYPE,)g(IERROR)166 1007 y Fs(newt)o(yp)q(e)c FD(is)f(the)f(datat)o (yp)q(e)g(obtained)h(b)o(y)f(concatenating)h Fs(count)h FD(copies)f(of)f Fs(oldt)o(yp)q(e)p FD(.)27 b(Concate-)75 1064 y(nation)15 b(is)h(de\014ned)h(using)f Fo(extent)e FD(as)h(the)h(size)g(of)e(the)i(concatenated)f(copies.)166 1210 y Fl(Discussion:)40 b Ft(Here,)17 b(and)e(in)g(the)i(other)f (datat)o(yp)q(e)f(constructors,)j(the)e(input)f(datat)o(yp)q(e)h (argumen)o(t)f(of)-2121 b Fm(CHANGE)75 1267 y Ft(the)16 b(C)g(function)f(is)h(passed)h(b)o(y)e(v)n(alue,)g(and)h(the)g(output)g (datat)o(yp)q(e)g(argumen)o(t)f(is)h(passed)g(b)o(y)g(reference.)26 b(I.e.,)75 1323 y(\(...,oldt)o(yp)q(e,)12 b(*newt)o(yp)q(e\).)18 b(It)c(has)g(b)q(een)h(argued)g(that)f(it)f(is)h(more)f(con)o(v)o (enien)o(t)h(to)g(pass)g(b)q(oth)g(via)f(a)h(p)q(oin)o(ter.)166 1469 y FD(Example:)31 b(let)21 b Fs(oldt)o(yp)q(e)h FD(ha)o(v)o(e)e(t)o (yp)q(e)h(map)f FA(f)p FD(\()p Fs(double)q Fw(;)8 b FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(8\))p FA(g)p Fw(;)18 b FD(with)j(exten)o(t)g(16,)g(and)f(let)75 1526 y Fs(count)14 b FD(=)f(3.)19 b(The)d(t)o(yp)q(e)f(map)g(of)g(the)g(datat)o(yp)q(e)g (returned)g(b)o(y)h Fs(newt)o(yp)q(e)h FD(is)189 1656 y FA(f)p FD(\()p Fs(double)p Fw(;)8 b FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(8\))p Fw(;)g FD(\()p Fs(double)o Fw(;)g FD(16\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(24\))p Fw(;)g FD(\()p Fs(double)n Fw(;)g FD(32\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(40\))p FA(g)p FD(;)75 1778 y(i.e.,)15 b(alternating)g Fs(double)i FD(and)e Fs(cha)o(r)g FD(elemen)o(ts,)h (with)f(displacemen)o(ts)i(0)p Fw(;)8 b FD(8)p Fw(;)g FD(16)p Fw(;)f FD(24)p Fw(;)g FD(32)o Fw(;)h FD(4)o(0.)166 1842 y(In)16 b(general,)f(assume)g(that)g(the)g(t)o(yp)q(e)g(map)g(of)g Fs(oldt)o(yp)q(e)h FD(is)189 1971 y FA(f)p FD(\()p Fw(ty)r(pe)314 1978 y Fi(0)333 1971 y Fw(;)8 b(disp)438 1978 y Fi(0)457 1971 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 1978 y Ff(n)p Fg(\000)p Fi(1)723 1971 y Fw(;)h(disp)828 1978 y Ff(n)p Fg(\000)p Fi(1)895 1971 y FD(\))p FA(g)p Fw(;)75 2094 y FD(with)16 b(exten)o(t)e Fw(extent)p FD(.)22 b(Then)15 b Fs(newt)o(yp)q(e)i FD(has)e(a)g(t)o(yp)q(e)h(map)f(with)g Fs(count)c FA(\001)f Fs(n)16 b FD(en)o(tries)f(de\014ned)i(b)o(y:)189 2224 y FA(f)p FD(\()p Fw(ty)r(pe)314 2231 y Fi(0)333 2224 y Fw(;)8 b(disp)438 2231 y Fi(0)457 2224 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 2231 y Ff(n)p Fg(\000)p Fi(1)723 2224 y Fw(;)h(disp)828 2231 y Ff(n)p Fg(\000)p Fi(1)895 2224 y FD(\))p Fw(;)g FD(\()p Fw(ty)r(pe)1036 2231 y Fi(0)1055 2224 y Fw(;)g(disp)1160 2231 y Fi(0)1189 2224 y FD(+)i Fw(extent)p FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)1562 2231 y Ff(n)p Fg(\000)p Fi(1)1629 2224 y Fw(;)g(disp)1734 2231 y Ff(n)p Fg(\000)p Fi(1)1811 2224 y FD(+)j Fw(extent)p FD(\))p Fw(;)189 2373 y(:::;)d FD(\()p Fw(ty)r(pe)351 2380 y Fi(0)368 2373 y Fw(;)g(disp)473 2380 y Fi(0)502 2373 y FD(+)i Fw(extent)i FA(\001)e FD(\()p Fs(count)h FA(\000)f FD(1\)\))p Fw(;)e(:::;)g FD(\()o Fw(ty)r(pe)1129 2380 y Ff(n)p Fg(\000)p Fi(1)1195 2373 y Fw(;)g(disp)1300 2380 y Ff(n)p Fg(\000)p Fi(1)1378 2373 y FD(+)i Fw(extent)i FA(\001)d FD(\()p Fs(count)i FA(\000)g FD(1\)\))p FA(g)p Fw(:)75 2534 y Fs(V)o(ecto)o(r)45 b FD(The)16 b(function)g Fs(MPI)p 600 2534 14 2 v 16 w(TYPE)p 733 2534 V 17 w(VECTOR)g FD(is)g(a)f(more)g(general)h (constructor)e(that)h(allo)o(ws)g(repli-)75 2591 y(cation)20 b(of)g(a)g(datat)o(yp)q(e)f(in)o(to)h(lo)q(cations)h(that)e(consist)i (of)e(equally)j(spaced)f(blo)q(c)o(ks.)35 b(Eac)o(h)20 b(blo)q(c)o(k)g(is)75 2647 y(obtained)g(b)o(y)f(concatenating)h(the)f (same)g(n)o(um)o(b)q(er)h(of)f(copies)h(of)f(the)g(old)h(datat)o(yp)q (e.)32 b(The)19 b(spacing)75 2704 y(b)q(et)o(w)o(een)d(blo)q(c)o(ks)f (is)h(a)f(m)o(ultiple)i(of)e(the)g(exten)o(t)g(of)g(the)g(old)h(datat)o (yp)q(e.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 61 68 61 67 bop 75 -100 a Fp(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FD(61)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(VECTOR\()16 b(count,)g(blo)q(cklength,)g(stride,)g(oldt)o (yp)q(e,)g(newt)o(yp)q(e\))117 122 y Ft(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o (teger\))117 194 y(IN)155 b Fs(blo)q(cklength)371 b Ft(n)o(um)o(b)q(er) 16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 250 y(ger\))117 322 y(IN)155 b Fs(stride)484 b Ft(n)o(um)o(b)q(er)13 b(of)f(elemen)o(ts)h(b)q(et)o(w)o(een)h(start)g (of)f(eac)o(h)g(blo)q(c)o(k)g(\(in)o(te-)905 378 y(ger\))117 450 y(IN)155 b Fs(oldt)o(yp)q(e)450 b Ft(old)13 b(datat)o(yp)q(e)i (\(handle\))117 522 y(OUT)108 b Fs(newt)o(yp)q(e)433 b Ft(new)15 b(datat)o(yp)q(e)f(\(handle\))75 646 y FB(int)23 b(MPI)p 245 646 15 2 v 17 w(Type)p 358 646 V 17 w(vector\(int)f(count,) h(int)h(blocklength,)e(int)i(stride,)393 702 y(MPI)p 468 702 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 702 V 17 w(Datatype)h(*newtype\))75 789 y(MPI)p 150 789 V 17 w(TYPE)p 263 789 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)g (OLDTYPE,)f(NEWTYPE,)h(IERROR\))170 845 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)166 932 y FD(Example:)c(Assume,)12 b(again,)g(that)f Fs(oldt)o(yp)q(e)i FD(has)f(t)o(yp)q(e)g(map)g FA(f)p FD(\()p Fs(double)p Fw(;)c FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(8\))p FA(g)p Fw(;)i FD(with)i(exten)o(t)75 988 y(16.)20 b(A)c(call)h(to)d Fs(MPI)p 428 988 14 2 v 16 w(TYPE)p 561 988 V 17 w(VECTOR\()i(2,)f(3,)g (4,)g(oldt)o(yp)q(e,)i(newt)o(yp)q(e\))g FD(will)g(create)e(the)h (datat)o(yp)q(e)f(with)75 1045 y(t)o(yp)q(e)g(map)189 1121 y FA(f)p FD(\()p Fs(double)p Fw(;)8 b FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(8\))p Fw(;)g FD(\()p Fs(double)o Fw(;)g FD(16\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(24\))p Fw(;)g FD(\()p Fs(double)n Fw(;)g FD(32\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(40\))p Fw(;)189 1202 y FD(\()p Fs(double)q Fw(;)g FD(64\))p Fw(;)g FD(\()p Fs(cha)o(r)m Fw(;)g FD(72\))p Fw(;)g FD(\()p Fs(double)o Fw(;)g FD(80\))p Fw(;)g FD(\()p Fs(cha)o(r)m Fw(;)g FD(88\))p Fw(;)g FD(\()p Fs(double)o Fw(;)g FD(96\))p Fw(;)g FD(\()p Fs(cha)o(r)m Fw(;)g FD(104\))p FA(g)j FD(:)75 1284 y(t)o(w)o(o)j(blo)q(c)o(ks)i (with)f(three)h(copies)g(eac)o(h)f(of)g(the)g(old)h(t)o(yp)q(e,)f (starting)f(4*16)g(apart.)166 1340 y(A)h(call)h(to)f Fs(MPI)p 439 1340 V 16 w(TYPE)p 572 1340 V 17 w(VECTOR\(3,)g(1,)g(-2,)f (oldt)o(yp)q(e,)i(newt)o(yp)q(e\))h FD(will)g(create)e(the)g(datat)o (yp)q(e)189 1416 y FA(f)p FD(\()p Fs(double)p Fw(;)8 b FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(8\))p Fw(;)g FD(\()p Fs(double)o Fw(;)g FA(\000)p FD(32\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FA(\000)i(\000)g(\000)p FD(24\))p Fw(;)e FD(\()p Fs(double)p Fw(;)g FA(\000)p FD(64\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FA(\000)p FD(56\))p FA(g)p Fw(:)166 1492 y FD(In)16 b(general,)f(assume)g(that)g Fs(oldt)o(yp)q(e)h FD(has)f(t)o(yp)q(e)g(map)189 1568 y FA(f)p FD(\()p Fw(ty)r(pe)314 1575 y Fi(0)333 1568 y Fw(;)8 b(disp)438 1575 y Fi(0)457 1568 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 1575 y Ff(n)p Fg(\000)p Fi(1)723 1568 y Fw(;)h(disp)828 1575 y Ff(n)p Fg(\000)p Fi(1)895 1568 y FD(\))p FA(g)p Fw(;)75 1644 y FD(with)15 b(exten)o(t)e Fw(extent)p FD(.)21 b(The)15 b(newly)g(created)f(datat)o(yp)q(e)f(has)i (a)e(t)o(yp)q(e)i(map)f(with)g Fs(count)9 b FA(\001)f Fs(blo)q(cklength)i FA(\001)e Fw(n)75 1701 y FD(en)o(tries:)189 1777 y FA(f)p FD(\()p Fw(ty)r(pe)314 1784 y Fi(0)333 1777 y Fw(;)g(disp)438 1784 y Fi(0)457 1777 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 1784 y Ff(n)p Fg(\000)p Fi(1)723 1777 y Fw(;)h(disp)828 1784 y Ff(n)p Fg(\000)p Fi(1)895 1777 y FD(\))p Fw(;)189 1858 y FD(\()p Fw(ty)r(pe)291 1865 y Fi(0)310 1858 y Fw(;)g(disp)415 1865 y Fi(0)444 1858 y FD(+)i Fw(extent)p FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)817 1865 y Ff(n)p Fg(\000)p Fi(1)884 1858 y Fw(;)g(disp)989 1865 y Ff(n)p Fg(\000)p Fi(1)1067 1858 y FD(+)i Fw(extent)p FD(\))p Fw(;)e(:::;)189 1940 y FD(\()p Fw(ty)r(pe)291 1947 y Fi(0)310 1940 y Fw(;)g(disp)415 1947 y Fi(0)444 1940 y FD(+)i(\()p Fs(blo)q(cklength)i FA(\000)f FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)1182 1947 y Ff(n)p Fg(\000)p Fi(1)1249 1940 y Fw(;)g(disp)1354 1947 y Ff(n)p Fg(\000)p Fi(1)1431 1940 y FD(+)j(\()p Fs(blo)q(cklength)g FA(\000)g FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)189 2021 y FD(\()p Fw(ty)r(pe)291 2028 y Fi(0)310 2021 y Fw(;)e(disp)415 2028 y Fi(0)444 2021 y FD(+)i Fs(stride)h FA(\001)f Fw(extent)p FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)954 2028 y Ff(n)p Fg(\000)p Fi(1)1021 2021 y Fw(;)g(disp)1126 2028 y Ff(n)p Fg(\000)p Fi(1)1203 2021 y FD(+)j Fs(stride)g FA(\001)f Fw(extent)p FD(\))p Fw(;)e(:::;)189 2102 y FD(\()p Fw(ty)r(pe)291 2109 y Fi(0)310 2102 y Fw(;)g(disp)415 2109 y Fi(0)444 2102 y FD(+)i(\()p Fs(stride)h FD(+)g Fs(blo)q(cklength)h FA(\000)e FD(1\))g FA(\001)f Fw(extent)p FD(\))p Fw(;)f(:::;)189 2184 y FD(\()p Fw(ty)r(pe)291 2191 y Ff(n)p Fg(\000)p Fi(1)359 2184 y Fw(;)g(disp)464 2191 y Ff(n)p Fg(\000)p Fi(1)542 2184 y FD(+)i(\()p Fs(stride)h FD(+)f Fs(blo)q(cklength)i FA(\000)f FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)e(::::;)189 2265 y FD(\()p Fw(ty)r(pe)291 2272 y Fi(0)310 2265 y Fw(;)g(disp)415 2272 y Fi(0)444 2265 y FD(+)i Fs(stride)h FA(\001)f FD(\()p Fs(count)h FA(\000)g FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)189 2346 y FD(\()p Fw(ty)r(pe)291 2353 y Ff(n)p Fg(\000)p Fi(1)359 2346 y Fw(;)g(disp)464 2353 y Ff(n)p Fg(\000)p Fi(1)542 2346 y FD(+)i Fs(stride)h FA(\001)f FD(\()p Fs(count)h FA(\000)f FD(1\))g FA(\001)g Fw(extent)p FD(\))p Fw(;)e(:::;)189 2428 y FD(\()p Fw(ty)r(pe)291 2435 y Fi(0)310 2428 y Fw(;)g(disp)415 2435 y Fi(0)444 2428 y FD(+)i(\()p Fs(stride)h FA(\001)f FD(\()p Fs(count)h FA(\000)f FD(1\))g(+)g Fs(blo)q(cklength)i FA(\000)f FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)189 2509 y FD(\()p Fw(ty)r(pe)291 2516 y Ff(n)p Fg(\000)p Fi(1)359 2509 y Fw(;)g(disp)464 2516 y Ff(n)p Fg(\000)p Fi(1)542 2509 y FD(+)i(\()p Fs(stride)h FA(\001)e FD(\()p Fs(count)j FA(\000)e FD(1\))g(+)g Fs(blo)q(cklength)i FA(\000)e FD(1\))g FA(\001)g Fw(extent)p FD(\))p FA(g)166 2591 y FD(A)16 b(call)i(to)e Fs(MPI)p 443 2591 V 16 w(TYPE)p 576 2591 V 16 w(CONTIGUOUS\(count,)k(oldt)o(yp)q(e,)d(newt)o(yp)q(e\))h FD(is)f(equiv)m(alen)o(t)h(to)e(a)g(call)i(to)75 2647 y Fs(MPI)p 160 2647 V 16 w(TYPE)p 293 2647 V 17 w(VECTOR\(count,)g(1,)f (1,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p FD(,)h(or)e(to)g(a)h(call)g (to)f Fs(MPI)p 1495 2647 V 16 w(TYPE)p 1628 2647 V 17 w(VECTOR\(1,)75 2704 y(count,)g(n,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p FD(,)h Fs(n)e FD(arbitrary)l(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 62 69 62 68 bop 75 -100 a FD(62)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(Hvecto)o(r)45 b FD(The)19 b(function)h Fs(MPI)p 631 45 14 2 v 16 w(TYPE)p 764 45 V 17 w(HVECTOR)g FD(is)f(iden)o(tical)i(to)d Fs(MPI)p 1389 45 V 16 w(TYPE)p 1522 45 V 17 w(VECTOR)p FD(,)h(except)75 102 y(that)h Fs(stride)h FD(is)g(giv)o(en)g(in)g(b)o(ytes,)g(rather)f (then)g(in)i(elemen)o(ts.)36 b(The)20 b(use)h(for)f(b)q(oth)g(t)o(yp)q (es)h(of)f(v)o(ector)75 158 y(constructors)14 b(is)i(illustrated)h(in)f (section)f(3.13.7.)j(\()p Fs(H)d FD(is)h(used)g(for)e (\\heterogeneous"\).)75 266 y Fs(MPI)p 160 266 V 16 w(TYPE)p 293 266 V 17 w(HVECTOR\()i(count,)g(blo)q(cklength,)h(stride,)e(oldt)o (yp)q(e,)h(newt)o(yp)q(e\))117 347 y Ft(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o (teger\))117 430 y(IN)155 b Fs(blo)q(cklength)371 b Ft(n)o(um)o(b)q(er) 16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 486 y(ger\))117 570 y(IN)155 b Fs(stride)484 b Ft(n)o(um)o(b)q(er)13 b(of)g(b)o(ytes)h(b)q(et)o(w)o(een)h(start)f (of)f(eac)o(h)h(blo)q(c)o(k)f(\(in)o(teger\))117 653 y(IN)155 b Fs(oldt)o(yp)q(e)450 b Ft(old)13 b(datat)o(yp)q(e)i (\(handle\))117 736 y(OUT)108 b Fs(newt)o(yp)q(e)433 b Ft(new)15 b(datat)o(yp)q(e)f(\(handle\))75 864 y FB(int)23 b(MPI)p 245 864 15 2 v 17 w(Type)p 358 864 V 17 w(hvector\(int)f (count,)h(int)h(blocklength,)e(MPI)p 1306 864 V 17 w(Aint)h(stride,)393 921 y(MPI)p 468 921 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 921 V 17 w(Datatype)h(*newtype\))75 1011 y(MPI)p 150 1011 V 17 w(TYPE)p 263 1011 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)f(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1068 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)166 1158 y FD(Assume)15 b(that)g Fs(oldt)o(yp)q(e)h FD(has)f(t)o(yp)q(e)g (map)189 1272 y FA(f)p FD(\()p Fw(ty)r(pe)314 1279 y Fi(0)333 1272 y Fw(;)8 b(disp)438 1279 y Fi(0)457 1272 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 1279 y Ff(n)p Fg(\000)p Fi(1)723 1272 y Fw(;)h(disp)828 1279 y Ff(n)p Fg(\000)p Fi(1)895 1272 y FD(\))p FA(g)p Fw(;)75 1382 y FD(with)15 b(exten)o(t)e Fw(extent)p FD(.)21 b(The)15 b(newly)g(created)f(datat)o(yp)q(e)f(has)i(a)e(t)o(yp)q(e)i(map)f(with) g Fs(count)9 b FA(\001)f Fs(blo)q(cklength)i FA(\001)e Fw(n)75 1438 y FD(en)o(tries:)189 1552 y FA(f)p FD(\()p Fw(ty)r(pe)314 1559 y Fi(0)333 1552 y Fw(;)g(disp)438 1559 y Fi(0)457 1552 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 1559 y Ff(n)p Fg(\000)p Fi(1)723 1552 y Fw(;)h(disp)828 1559 y Ff(n)p Fg(\000)p Fi(1)895 1552 y FD(\))p Fw(;)189 1682 y FD(\()p Fw(ty)r(pe)291 1689 y Fi(0)310 1682 y Fw(;)g(disp)415 1689 y Fi(0)444 1682 y FD(+)i Fw(extent)p FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)817 1689 y Ff(n)p Fg(\000)p Fi(1)884 1682 y Fw(;)g(disp)989 1689 y Ff(n)p Fg(\000)p Fi(1)1067 1682 y FD(+)i Fw(extent)p FD(\))p Fw(;)e(:::;)189 1808 y FD(\()p Fw(ty)r(pe)291 1815 y Fi(0)310 1808 y Fw(;)g(disp)415 1815 y Fi(0)444 1808 y FD(+)i(\()p Fs(blo)q(cklength)i FA(\000)f FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)1182 1815 y Ff(n)p Fg(\000)p Fi(1)1249 1808 y Fw(;)g(disp)1354 1815 y Ff(n)p Fg(\000)p Fi(1)1431 1808 y FD(+)j(\()p Fs(blo)q(cklength)g FA(\000)g FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)189 1935 y FD(\()p Fw(ty)r(pe)291 1942 y Fi(0)310 1935 y Fw(;)e(disp)415 1942 y Fi(0)444 1935 y FD(+)i Fs(stride)q FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)794 1942 y Ff(n)p Fg(\000)p Fi(1)860 1935 y Fw(;)g(disp)965 1942 y Ff(n)p Fg(\000)p Fi(1)1042 1935 y FD(+)j Fs(stride)q FD(\))p Fw(;)d(:::;)189 2061 y FD(\()p Fw(ty)r(pe)291 2068 y Fi(0)310 2061 y Fw(;)g(disp)415 2068 y Fi(0)444 2061 y FD(+)i Fs(stride)h FD(+)g(\()p Fs(blo)q(cklength)h FA(\000)e FD(1\))g FA(\001)f Fw(extent)p FD(\))p Fw(;)f(:::;)189 2187 y FD(\()p Fw(ty)r(pe)291 2194 y Ff(n)p Fg(\000)p Fi(1)359 2187 y Fw(;)g(disp)464 2194 y Ff(n)p Fg(\000)p Fi(1)542 2187 y FD(+)i Fs(stride)h FD(+)g(\()p Fs(blo)q(cklength)g FA(\000)g FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)e(::::;)189 2313 y FD(\()p Fw(ty)r(pe)291 2320 y Fi(0)310 2313 y Fw(;)g(disp)415 2320 y Fi(0)444 2313 y FD(+)i Fs(stride)h FA(\001)f FD(\()p Fs(count)h FA(\000)g FD(1\)\))p Fw(;)d(:::;)189 2440 y FD(\()p Fw(ty)r(pe)291 2447 y Ff(n)p Fg(\000)p Fi(1)359 2440 y Fw(;)g(disp)464 2447 y Ff(n)p Fg(\000)p Fi(1)542 2440 y FD(+)i Fs(stride)h FA(\001)f FD(\()p Fs(count)h FA(\000)f FD(1\)\))p Fw(;)e(:::;)189 2566 y FD(\()p Fw(ty)r(pe)291 2573 y Fi(0)310 2566 y Fw(;)g(disp)415 2573 y Fi(0)444 2566 y FD(+)i Fs(stride)h FA(\001)f FD(\()p Fs(count)h FA(\000)g FD(1\))e(+)i(\()p Fs(blo)q(cklength)g FA(\000)g FD(1\))e FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)189 2692 y FD(\()p Fw(ty)r(pe)291 2699 y Ff(n)p Fg(\000)p Fi(1)359 2692 y Fw(;)g(disp)464 2699 y Ff(n)p Fg(\000)p Fi(1)542 2692 y FD(+)i Fs(stride)h FA(\001)f FD(\()p Fs(count)h FA(\000)f FD(1\))g(+)g(\()p Fs(blo)q(cklength)i FA(\000)e FD(1\))g FA(\001)g Fw(extent)p FD(\))p FA(g)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 63 70 63 69 bop 75 -100 a Fp(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FD(63)75 45 y Fs(Indexed)47 b FD(The)19 b(function)h Fs(MPI)p 629 45 14 2 v 16 w(TYPE)p 762 45 V 17 w(INDEXED)f FD(allo)o(ws)h(replication)h(of)d(an)i(old)f(datat)o(yp)q(e)g(in)o(to)g (a)75 102 y(sequence)c(of)e(con)o(tiguous)h(blo)q(c)o(ks,)g(where)g (eac)o(h)g(blo)q(c)o(k)h(can)f(con)o(tain)g(di\013eren)o(t)g(n)o(um)o (b)q(er)g(of)f(copies)i(and)75 158 y(ha)o(v)o(e)f(a)g(di\013eren)o(t)g (displacemen)o(t.)21 b(All)16 b(blo)q(c)o(k)f(displacemen)o(ts)g(are)f (m)o(ultiples)i(of)e(the)h(old)f(t)o(yp)q(e)h(exten)o(t.)75 262 y Fs(MPI)p 160 262 V 16 w(TYPE)p 293 262 V 17 w(INDEXED\()d(count,) i(a)o(rra)o(y)p 762 262 V 14 w(of)p 813 262 V 16 w(blo)q(cklengths,)h (a)o(rra)o(y)p 1185 262 V 15 w(of)p 1237 262 V 16 w(displacements,)e (oldt)o(yp)q(e,)h(newt)o(yp)q(e\))117 395 y Ft(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f (n)o(um)o(b)q(er)h(of)f(en)o(tries)i(in)905 452 y Fv(a)o(rra)o(y)p 992 452 13 2 v 15 w(of)p 1041 452 V 15 w(displacements)17 b Ft(and)h Fv(a)o(rra)o(y)p 1483 452 V 15 w(of)p 1532 452 V 15 w(blo)q(cklengths)i Ft(\(non-)905 508 y(negativ)o(e)14 b(in)o(teger\))117 584 y(IN)155 b Fs(a)o(rra)o(y)p 416 584 14 2 v 15 w(of)p 468 584 V 16 w(blo)q(cklengths)191 b Ft(n)o(um)o(b)q(er)18 b(of)f(elemen)o(ts)h(p)q(er)i(blo)q(c)o(k)d (\(arra)o(y)i(of)e(nonnegativ)o(e)905 640 y(in)o(tegers\))117 715 y(IN)155 b Fs(a)o(rra)o(y)p 416 715 V 15 w(of)p 468 715 V 16 w(displacements)162 b Ft(displacemen)o(t)19 b(for)h(eac)o(h)g(blo)q(c)o(k,)g(in)g(m)o(ultiples)e(of)h Fv(oldt)o(yp)q(e)905 772 y Ft(exten)o(t)c(\(arra)o(y)f(of)f(in)o (teger\))117 847 y(IN)155 b Fs(oldt)o(yp)q(e)450 b Ft(old)13 b(datat)o(yp)q(e)i(\(handle\))117 922 y(OUT)108 b Fs(newt)o(yp)q(e)433 b Ft(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1047 y FB(int)23 b(MPI)p 245 1047 15 2 v 17 w(Type)p 358 1047 V 17 w(indexed\(int)f (count,)h(int)h(*array)p 1068 1047 V 16 w(of)p 1132 1047 V 17 w(blocklengths,)393 1103 y(int)g(*array)p 636 1103 V 16 w(of)p 700 1103 V 17 w(displacements,)e(MPI)p 1147 1103 V 17 w(Datatype)g(oldtype,)393 1159 y(MPI)p 468 1159 V 17 w(Datatype)h(*newtype\))75 1246 y(MPI)p 150 1246 V 17 w(TYPE)p 263 1246 V 16 w(INDEXED\(COUNT,)f(ARRAY)p 757 1246 V 17 w(OF)p 822 1246 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 1246 V 16 w(OF)p 1357 1246 V 17 w(DISPLACEMENTS,)393 1302 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1359 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1359 V 17 w(OF)p 716 1359 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1359 V 17 w(OF)p 1323 1359 V 17 w(DISPLACEMENTS\(*\),)170 1415 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)166 1502 y FD(Example:)i(let)19 b Fs(oldt)o(yp)q(e)f FD(ha)o(v)o(e)g(t)o(yp)q(e)g(map)f FA(f)p FD(\()p Fs(double)q Fw(;)8 b FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(8\))p FA(g)p Fw(;)15 b FD(with)j(exten)o(t)g (16.)27 b(Let)18 b Fs(B)g(=)75 1558 y(\(3,)13 b(1\))h FD(and)g(let)h Fs(D)f(=)g(\(4,)f(0\))p FD(.)19 b(A)14 b(call)i(to)d Fs(MPI)p 855 1558 14 2 v 16 w(TYPE)p 988 1558 V 17 w(INDEXED\(2,)g(B,)h(D,)f(oldt)o(yp)q(e,)j(newt)o(yp)q(e\))f FD(returns)75 1615 y(a)g(datat)o(yp)q(e)g(with)g(t)o(yp)q(e)g(map)189 1709 y FA(f)p FD(\()p Fs(double)p Fw(;)8 b FD(64\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(72\))p Fw(;)g FD(\()p Fs(double)n Fw(;)g FD(80\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(88\))p Fw(;)g FD(\()p Fs(double)n Fw(;)g FD(96\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(104\))p Fw(;)189 1815 y FD(\()p Fs(double)q Fw(;)g FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)m Fw(;)g FD(8\))p FA(g)j FD(:)75 1922 y(three)k(copies)h(of) e(the)i(old)f(t)o(yp)q(e)g(starting)f(at)h(displacemen)o(t)h(64,)e(and) h(one)h(cop)o(y)e(starting)h(at)f(displace-)75 1978 y(men)o(t)h(0.)166 2035 y(In)h(general,)f(assume)g(that)g Fs(oldt)o(yp)q(e)h FD(has)f(t)o(yp)q(e)g(map)189 2129 y FA(f)p FD(\()p Fw(ty)r(pe)314 2136 y Fi(0)333 2129 y Fw(;)8 b(disp)438 2136 y Fi(0)457 2129 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 2136 y Ff(n)p Fg(\000)p Fi(1)723 2129 y Fw(;)h(disp)828 2136 y Ff(n)p Fg(\000)p Fi(1)895 2129 y FD(\))p FA(g)p Fw(;)75 2222 y FD(with)35 b(exten)o(t)g Fo(extent)p FD(.)79 b(Let)36 b Fs(B)f FD(b)q(e)h(the)f Fs(a)o(rra)o(y)p 1013 2222 V 15 w(of)p 1065 2222 V 16 w(blo)q(cklength)i FD(argumen)o(t)d(and)h Fs(D)g FD(b)q(e)h(the)75 2279 y Fs(a)o(rra)o(y)p 173 2279 V 15 w(of)p 225 2279 V 16 w(displacements)15 b FD(argumen)o(t.)k (The)c(newly)h(created)f(datat)o(yp)q(e)g(has)g Fw(n)c FA(\001)1450 2247 y Fe(P)1494 2260 y Fv(count)q Fg(\000)p Fi(1)1494 2292 y Ff(i)p Fi(=0)1646 2279 y Fs(B)p FD([)p Fs(i)p FD(])j(en)o(tries:)189 2373 y FA(f)p FD(\()p Fw(ty)r(pe)314 2380 y Fi(0)333 2373 y Fw(;)8 b(disp)438 2380 y Fi(0)467 2373 y FD(+)i Fs(D)p FD([)p Fs(0)p FD(])f FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)954 2380 y Ff(n)p Fg(\000)p Fi(1)1021 2373 y Fw(;)g(disp)1126 2380 y Ff(n)p Fg(\000)p Fi(1)1203 2373 y FD(+)j Fs(D)p FD([)p Fs(0)p FD(])d FA(\001)i Fw(extent)p FD(\))p Fw(;)e(:::;)189 2479 y FD(\()p Fw(ty)r(pe)291 2486 y Fi(0)310 2479 y Fw(;)g(disp)415 2486 y Fi(0)444 2479 y FD(+)i(\()p Fs(D)p FD([)p Fs(0)p FD(])f(+)h Fs(B)p FD([)p Fs(0)p FD(])g FA(\000)g FD(1\))g FA(\001)f Fw(extent)p FD(\))p Fw(;)f(:::;)g FD(\()p Fw(ty)r(pe)1178 2486 y Ff(n)p Fg(\000)p Fi(1)1245 2479 y Fw(;)g(disp)1350 2486 y Ff(n)p Fg(\000)p Fi(1)1428 2479 y FD(+)i(\()p Fs(D)p FD([)p Fs(0)p FD(])f(+)h Fs(B)p FD([)p Fs(0)p FD(])g FA(\000)g FD(1\))g FA(\001)f Fw(extent)p FD(\))p Fw(;)f(:::;)189 2586 y FD(\()p Fw(ty)r(pe)291 2593 y Fi(0)310 2586 y Fw(;)g(disp)415 2593 y Fi(0)444 2586 y FD(+)i Fs(D)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])e FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)1093 2593 y Ff(n)p Fg(\000)p Fi(1)1160 2586 y Fw(;)g(disp)1265 2593 y Ff(n)p Fg(\000)p Fi(1)1342 2586 y FD(+)j Fs(D)p FD([)p Fs(count)g FA(\000)f Fs(1)p FD(])f FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)189 2692 y FD(\()p Fw(ty)r(pe)291 2699 y Fi(0)310 2692 y Fw(;)g(disp)415 2699 y Fi(0)444 2692 y FD(+)i(\()p Fs(D)p FD([)p Fs(count)h FA(\000)f Fs(1)p FD(])g(+)g Fs(B)p FD([)p Fs(count)i FA(\000)e Fs(1)p FD(])g FA(\000)g FD(1\))f FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 64 71 64 70 bop 75 -100 a FD(64)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)189 45 y FD(\()p Fw(ty)r(pe)291 52 y Ff(n)p Fg(\000)p Fi(1)359 45 y Fw(;)8 b(disp)464 52 y Ff(n)p Fg(\000)p Fi(1)542 45 y FD(+)i(\()p Fs(D)p FD([)p Fs(count)h FA(\000)f Fs(1)p FD(])g(+)g Fs(B)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])e FA(\000)i FD(1\))e FA(\001)h Fw(extent)p FD(\))p FA(g)p Fw(:)166 138 y FD(A)j(call)g(to)f Fs(MPI)p 431 138 14 2 v 16 w(TYPE)p 564 138 V 17 w(VECTOR\(count,)j(blo)q(cklength,)g(stride,)e(oldt)o(yp)q (e,)h(newt)o(yp)q(e\))g FD(is)f(equiv)m(alen)o(t)75 194 y(to)i(a)f(call)j(to)d Fs(MPI)p 392 194 V 16 w(TYPE)p 525 194 V 17 w(INDEX\(count,)i(B,)f(D,)f(oldt)o(yp)q(e,)i(newt)o(yp)q (e\))h FD(where)189 278 y Fs(D)p FD([)p Fs(j)p FD(])11 b(=)i Fw(j)g FA(\001)d Fs(stride)16 b Fw(;)8 b(j)14 b FD(=)f(0)p Fw(;)8 b(:::;)g Fs(count)g FA(\000)j FD(1)p Fw(;)75 362 y FD(and)189 446 y Fs(B)p FD([)p Fs(j)p FD(])h(=)h Fs(blo)q(cklength)k Fw(;)8 b(j)14 b FD(=)f(0)p Fw(;)8 b(:::;)g Fs(count)h FA(\000)h FD(1)p Fw(:)75 565 y Fs(Hindexed)47 b FD(The)11 b(function)h Fs(MPI)p 643 565 V 15 w(TYPE)p 775 565 V 17 w(HINDEXED)f FD(is)g(iden)o(tical)i(to)d Fs(MPI)p 1383 565 V 15 w(TYPE)p 1515 565 V 17 w(INDEXED)p FD(,)g(except)75 621 y(that)19 b(blo)q(c)o(k)i(displacemen)o(ts)h(in)41 b Fs(a)o(rra)o(y)p 774 621 V 14 w(of)p 825 621 V 16 w(displacements)20 b FD(are)g(sp)q(eci\014ed)i(in)f(b)o(ytes,)g(rather)e(than)h(in)75 678 y(m)o(ultiples)d(of)e(the)g Fs(oldt)o(yp)q(e)h FD(exten)o(t.)75 781 y Fs(MPI)p 160 781 V 16 w(TYPE)p 293 781 V 17 w(HINDEXED\()21 b(count,)k(a)o(rra)o(y)p 814 781 V 14 w(of)p 865 781 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1248 781 V 15 w(of)p 1300 781 V 16 w(displacements,)d(oldt)o(yp)q(e,)h(new-)75 838 y(t)o(yp)q(e\))117 915 y Ft(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f(n)o(um)o(b)q (er)h(of)f(en)o(tries)i(in)905 971 y Fv(a)o(rra)o(y)p 992 971 13 2 v 15 w(of)p 1041 971 V 15 w(displacements)17 b Ft(and)h Fv(a)o(rra)o(y)p 1483 971 V 15 w(of)p 1532 971 V 14 w(blo)q(cklengths)i Ft(\(in)o(te-)905 1028 y(ger\))117 1101 y(IN)155 b Fs(a)o(rra)o(y)p 416 1101 14 2 v 15 w(of)p 468 1101 V 16 w(blo)q(cklengths)191 b Ft(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)h(in)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)f(of)g (nonnega-)905 1157 y(tiv)o(e)f(in)o(tegers\))117 1231 y(IN)155 b Fs(a)o(rra)o(y)p 416 1231 V 15 w(of)p 468 1231 V 16 w(displacements)162 b Ft(b)o(yte)14 b(displacemen)o(t)g(of)f (eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1304 y(IN)155 b Fs(oldt)o(yp)q(e)450 b Ft(old)13 b(datat)o(yp)q(e)i (\(handle\))117 1377 y(OUT)108 b Fs(newt)o(yp)q(e)433 b Ft(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1501 y FB(int)23 b(MPI)p 245 1501 15 2 v 17 w(Type)p 358 1501 V 17 w(hindexed\(int)f (count,)h(int)h(*array)p 1092 1501 V 16 w(of)p 1156 1501 V 17 w(blocklengths,)393 1558 y(MPI)p 468 1558 V 17 w(Aint)f(*array)p 748 1558 V 17 w(of)p 813 1558 V 17 w(displacements,)e(MPI)p 1259 1558 V 17 w(Datatype)i(oldtype,)393 1614 y(MPI)p 468 1614 V 17 w(Datatype)g(*newtype\))75 1701 y(MPI)p 150 1701 V 17 w(TYPE)p 263 1701 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 1701 V 17 w(OF)p 846 1701 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 1701 V 16 w(OF)p 1381 1701 V 17 w(DISPLACEMENTS,)393 1757 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1814 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1814 V 17 w(OF)p 716 1814 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1814 V 17 w(OF)p 1323 1814 V 17 w(DISPLACEMENTS\(*\),)170 1870 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)166 1957 y FD(Assume)15 b(that)g Fs(oldt)o(yp)q(e)h FD(has)f(t)o(yp)q(e)g(map)189 2041 y FA(f)p FD(\()p Fw(ty)r(pe)314 2048 y Fi(0)333 2041 y Fw(;)8 b(disp)438 2048 y Fi(0)457 2041 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 2048 y Ff(n)p Fg(\000)p Fi(1)723 2041 y Fw(;)h(disp)828 2048 y Ff(n)p Fg(\000)p Fi(1)895 2041 y FD(\))p FA(g)p Fw(;)75 2125 y FD(with)35 b(exten)o(t)g Fo(extent)p FD(.)79 b(Let)36 b Fs(B)f FD(b)q(e)h(the)f Fs(a)o(rra)o(y)p 1013 2125 14 2 v 15 w(of)p 1065 2125 V 16 w(blo)q(cklength)i FD(argumen)o(t)d(and)h Fs(D)g FD(b)q(e)h(the)75 2181 y Fs(a)o(rra)o(y)p 173 2181 V 15 w(of)p 225 2181 V 16 w(displacements)21 b FD(argumen)o(t.)39 b(The)22 b(newly)h(created)f(datat)o(yp)q(e)f(has)h(a)f(t)o(yp)q(e)h (map)f(with)i Fw(n)14 b FA(\001)75 2206 y Fe(P)119 2219 y Fv(count)q Fg(\000)p Fi(1)119 2251 y Ff(i)p Fi(=0)271 2238 y Fs(B)p FD([)p Fs(i)p FD(])g(en)o(tries:)189 2322 y FA(f)p FD(\()p Fw(ty)r(pe)314 2329 y Fi(0)333 2322 y Fw(;)8 b(disp)438 2329 y Fi(0)467 2322 y FD(+)i Fs(D)p FD([)p Fs(0)p FD(])o(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)794 2329 y Ff(n)p Fg(\000)p Fi(1)859 2322 y Fw(;)g(disp)964 2329 y Ff(n)p Fg(\000)p Fi(1)1042 2322 y FD(+)i Fs(D)p FD([)p Fs(0)p FD(])o(\))p Fw(;)e(:::;)189 2414 y FD(\()p Fw(ty)r(pe)291 2421 y Fi(0)310 2414 y Fw(;)g(disp)415 2421 y Fi(0)444 2414 y FD(+)i Fs(D)p FD([)p Fs(0)p FD(])f(+)i(\()p Fs(B)p FD([)p Fs(0)p FD(])e FA(\000)h FD(1\))g FA(\001)f Fw(extent)p FD(\))p Fw(;)f(:::;)g FD(\()p Fw(ty)r(pe)1178 2421 y Ff(n)p Fg(\000)p Fi(1)1245 2414 y Fw(;)g(disp)1350 2421 y Ff(n)p Fg(\000)p Fi(1)1428 2414 y FD(+)i Fs(D)p FD([)p Fs(0)p FD(])f(+)i(\()p Fs(B)p FD([)p Fs(0)p FD(])e FA(\000)h FD(1\))g FA(\001)f Fw(extent)p FD(\))p Fw(;)f(:::;)189 2507 y FD(\()p Fw(ty)r(pe)291 2514 y Fi(0)310 2507 y Fw(;)g(disp)415 2514 y Fi(0)444 2507 y FD(+)i Fs(D)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])o(\))p Fw(;)d(:::;)g FD(\()p Fw(ty)r(pe)933 2514 y Ff(n)p Fg(\000)p Fi(1)999 2507 y Fw(;)g(disp)1104 2514 y Ff(n)p Fg(\000)p Fi(1)1181 2507 y FD(+)i Fs(D)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])o(\))p Fw(;)d(:::;)189 2600 y FD(\()p Fw(ty)r(pe)291 2607 y Fi(0)310 2600 y Fw(;)g(disp)415 2607 y Fi(0)444 2600 y FD(+)i Fs(D)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])e(+)i(\()p Fs(B)p FD([)p Fs(count)g FA(\000)f Fs(1)p FD(])g FA(\000)g FD(1\))f FA(\001)h Fw(extent)p FD(\))p Fw(;)e(:::;)189 2692 y FD(\()p Fw(ty)r(pe)291 2699 y Ff(n)p Fg(\000)p Fi(1)359 2692 y Fw(;)g(disp)464 2699 y Ff(n)p Fg(\000)p Fi(1)542 2692 y FD(+)i Fs(D)p FD([)p Fs(count)h FA(\000)f Fs(1)p FD(])g(+)g(\()p Fs(B)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])e FA(\000)i FD(1\))e FA(\001)h Fw(extent)p FD(\))p FA(g)p Fw(:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 65 72 65 71 bop 75 -100 a Fp(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FD(65)75 45 y Fs(Struct)47 b(MPI)p 323 45 14 2 v 16 w(TYPE)p 456 45 V 16 w(STRUCT)23 b FD(is)g(the)f(most)g(general)h (constructor:)33 b(It)22 b(further)g(generalizes)i(the)75 102 y(previous)16 b(one)f(in)h(that)f(it)g(allo)o(ws)h(eac)o(h)f(blo)q (c)o(k)h(to)e(consist)i(of)f(replications)h(of)f(a)g(di\013eren)o(t)h (datat)o(yp)q(e.)75 205 y Fs(MPI)p 160 205 V 16 w(TYPE)p 293 205 V 17 w(STRUCT\(count,)c(a)o(rra)o(y)p 729 205 V 15 w(of)p 781 205 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1151 205 V 14 w(of)p 1202 205 V 16 w(displacements,)e(a)o(rra)o(y)p 1599 205 V 15 w(of)p 1651 205 V 15 w(t)o(yp)q(es,)i(new-)75 262 y(t)o(yp)q(e\))117 339 y Ft(IN)155 b Fs(count)482 b Ft(n)o(um)o(b)q(er)18 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))g({)g (also)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)905 395 y(in)c(arra)o(ys)h Fv(a)o(rra)o(y)p 1167 395 13 2 v 15 w(of)p 1216 395 V 14 w(t)o(yp)q(es)p Ft(,)h Fv(a)o(rra)o(y)p 1432 395 V 15 w(of)p 1481 395 V 14 w(displacements)e Ft(and)g Fv(a)o(r-)905 452 y(ra)o(y)p 959 452 V 15 w(of)p 1008 452 V 15 w(blo)q(cklengths)117 524 y Ft(IN)155 b Fs(a)o(rra)o(y)p 416 524 14 2 v 15 w(of)p 468 524 V 16 w(blo)q(cklength)208 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in) o(teger\))117 597 y(IN)155 b Fs(a)o(rra)o(y)p 416 597 V 15 w(of)p 468 597 V 16 w(displacements)162 b Ft(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o (teger\))117 669 y(IN)155 b Fs(a)o(rra)o(y)p 416 669 V 15 w(of)p 468 669 V 16 w(t)o(yp)q(es)327 b Ft(t)o(yp)q(e)20 b(of)f(elemen)o(ts)g(in)g(eac)o(h)h(blo)q(c)o(k)f(\(arra)o(y)g(of)g (handles)g(to)905 726 y(datat)o(yp)q(e)14 b(ob)r(jects\))117 798 y(OUT)108 b Fs(newt)o(yp)q(e)433 b Ft(new)15 b(datat)o(yp)q(e)f (\(handle\))75 923 y FB(int)23 b(MPI)p 245 923 15 2 v 17 w(Type)p 358 923 V 17 w(struct\(int)f(count,)h(int)h(*array)p 1044 923 V 16 w(of)p 1108 923 V 17 w(blocklengths,)393 979 y(MPI)p 468 979 V 17 w(Aint)f(*array)p 748 979 V 17 w(of)p 813 979 V 17 w(displacements,)e(MPI)p 1259 979 V 17 w(Datatype)i(*array)p 1635 979 V 16 w(of)p 1699 979 V 17 w(types,)393 1036 y(MPI)p 468 1036 V 17 w(Datatype)g (*newtype\))75 1122 y(MPI)p 150 1122 V 17 w(TYPE)p 263 1122 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 1122 V 17 w(OF)p 798 1122 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 1122 V 16 w(OF)p 1333 1122 V 17 w(DISPLACEMENTS,)393 1178 y(ARRAY)p 516 1178 V 17 w(OF)p 581 1178 V 17 w(TYPES,)h(NEWTYPE,)f(IERROR\))170 1235 y(INTEGER)h(COUNT,)g(ARRAY)p 651 1235 V 17 w(OF)p 716 1235 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 1235 V 17 w(OF)p 1323 1235 V 17 w(DISPLACEMENTS\(*\),)170 1291 y(ARRAY)p 293 1291 V 17 w(OF)p 358 1291 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)166 1378 y FD(Example:)d(Let)c Fs(t)o(yp)q(e1)g FD(ha)o(v)o(e)f(t)o(yp)q(e)g(map)189 1458 y FA(f)p FD(\()p Fs(double)p Fw(;)8 b FD(0\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(8\))p FA(g)p Fw(;)75 1539 y FD(with)k(exten)o(t)f(16.)18 b(Let)11 b Fs(B)h(=)g(\(2,)f(1,)g(3\))p FD(,)g Fs(D)h(=)f(\(0,)g(16,)h (26\))p FD(,)e(and)i Fs(T)f(=)h(\(MPI)p 1321 1539 14 2 v 15 w(FLO)o(A)l(T,)f(t)o(yp)q(e1,)i(MPI)p 1711 1539 V 16 w(CHAR\))p FD(.)75 1595 y(Then)j(a)f(call)h(to)e Fs(MPI)p 455 1595 V 16 w(TYPE)p 588 1595 V 17 w(STRUCT\(3,)h(B,)g(D,)g (T,)f(newt)o(yp)q(e\))j FD(returns)e(datat)o(yp)q(e)g(with)g(t)o(yp)q (e)h(map,)189 1676 y FA(f)p FD(\()p Fs(\015oat)o Fw(;)8 b FD(0\))p Fw(;)g FD(\()p Fs(\015oat)n Fw(;)g FD(4\))p Fw(;)g FD(\()p Fs(double)o Fw(;)g FD(16\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(24\))p Fw(;)g FD(\()p Fs(cha)o(r)m Fw(;)g FD(26\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(27\))p Fw(;)g FD(\()p Fs(cha)o(r)n Fw(;)g FD(28\))p FA(g)i FD(:)75 1756 y(t)o(w)o(o)g(copies)j(of)e Fs(MPI)p 421 1756 V 15 w(FLO)o(A)l(T)h FD(starting)f(at)g(0,)g(follo)o(w)o(ed)h(b)o(y)f (one)h(cop)o(y)f(of)g Fs(t)o(yp)q(e1)i FD(starting)e(at)f(16,)i(follo)o (w)o(ed)75 1813 y(b)o(y)i(three)g(copies)h(of)e Fs(MPI)p 520 1813 V 16 w(CHAR)p FD(,)i(starting)e(at)g(26.)19 b(\(W)l(e)14 b(assume)g(that)f(a)g(\015oat)h(o)q(ccupies)h(four)f(b)o (ytes.\))166 1869 y(In)i(general,)f(let)h Fs(T)f FD(b)q(e)h(the)f Fs(a)o(rra)o(y)p 742 1869 V 14 w(of)p 793 1869 V 16 w(t)o(yp)q(es)i FD(argumen)o(t,)d(where)31 b Fs(T[i])15 b FD(is)h(a)f(handle)h(to)189 1950 y Fw(ty)r(pemap)360 1957 y Ff(i)387 1950 y FD(=)d FA(f)p FD(\()p Fw(ty)r(pe)560 1931 y Ff(i)560 1961 y Fi(0)579 1950 y Fw(;)8 b(disp)684 1931 y Ff(i)684 1961 y Fi(0)702 1950 y FD(\))p Fw(;)g(:::;)g FD(\()p Fw(ty)r(pe)903 1931 y Ff(i)903 1961 y(n)924 1966 y Fd(i)935 1961 y Fg(\000)p Fi(1)982 1950 y Fw(;)g(disp)1087 1931 y Ff(i)1087 1961 y(n)1108 1966 y Fd(i)1120 1961 y Fg(\000)p Fi(1)1168 1950 y FD(\))p FA(g)p Fw(;)75 2030 y FD(with)i(exten)o(t)g Fw(extent)434 2037 y Ff(i)450 2030 y FD(.)18 b(Let)10 b Fs(B)h FD(b)q(e)g(the)f Fs(a)o(rra)o(y)p 826 2030 V 14 w(of)p 877 2030 V 16 w(blo)q(cklength)i FD(argumen)o(t)e(and)g Fs(D)g FD(b)q(e)h(the)f Fs(a)o(rra)o(y)p 1671 2030 V 15 w(of)p 1723 2030 V 16 w(displacements)75 2087 y FD(argumen)o(t.)19 b(Then)c(the)g(newly)h(created)f(datat)o(yp)q(e)f(has)h(a)f(t)o(yp)q(e) h(map)g(with)1395 2054 y Fe(P)1439 2067 y Fv(count)q Fg(\000)p Fi(1)1439 2099 y Ff(i)p Fi(=0)1591 2087 y Fs(B)p FD([)p Fs(i)p FD(])8 b FA(\001)h Fw(n)1715 2094 y Ff(i)1745 2087 y FD(en)o(tries)189 2174 y FA(f)p FD(\()p Fw(ty)r(pe)314 2155 y Fi(0)314 2185 y(0)333 2174 y Fw(;)f(disp)438 2155 y Fi(0)438 2185 y(0)467 2174 y FD(+)i Fs(D)p FD([)p Fs(0)p FD(])o(\))p Fw(;)e(:::;)g FD(\()p Fw(ty)r(pe)794 2155 y Fi(0)794 2185 y Ff(n)815 2190 y Fz(0)832 2174 y Fw(;)g(disp)937 2155 y Fi(0)937 2185 y Ff(n)958 2190 y Fz(0)986 2174 y FD(+)j Fs(D)p FD([)p Fs(0)p FD(])o(\))p Fw(;)d(:::;)189 2262 y FD(\()p Fw(ty)r(pe)291 2243 y Fi(0)291 2273 y(0)310 2262 y Fw(;)g(disp)415 2243 y Fi(0)415 2273 y(0)444 2262 y FD(+)i Fs(D)p FD([)p Fs(0)p FD(])f(+)i(\()p Fs(B)p FD([)p Fs(0)p FD(])e FA(\000)h FD(1\))g FA(\001)f Fw(extent)977 2269 y Fi(0)999 2262 y FD(\))p Fw(;)f(:::;)g FD(\()o Fw(ty)r(pe)1199 2243 y Fi(0)1199 2273 y Ff(n)1220 2278 y Fz(0)1237 2262 y Fw(;)g(disp)1342 2243 y Fi(0)1342 2273 y Ff(n)1363 2278 y Fz(0)1392 2262 y FD(+)i Fs(D)p FD([)p Fs(0)p FD(])f(+)i(\()p Fs(B)p FD([)p Fs(0)p FD(])e FA(\000)h Fs(1)p FD(\))g FA(\001)g Fw(extent)1926 2269 y Fi(0)1947 2262 y FD(\))p Fw(;)e(:::;)189 2356 y FD(\()p Fw(ty)r(pe)291 2337 y Fv(count)q Fg(\000)p Fi(1)291 2368 y(0)435 2356 y Fw(;)g(disp)540 2337 y Fv(count)p Fg(\000)p Fi(1)540 2368 y(0)693 2356 y FD(+)i Fs(D)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])o(\))p Fw(;)d(:::;)g FD(\()p Fw(ty)r(pe)1182 2337 y Fv(count)n Fg(\000)p Fi(1)1182 2368 y Ff(n)1203 2380 y Fv(count)o Fc(\000)p Fz(1)1339 2368 y Fg(\000)p Fi(1)1386 2356 y Fw(;)g(disp)1491 2337 y Fv(count)p Fg(\000)p Fi(1)1491 2368 y Ff(n)1512 2380 y Fv(count)p Fc(\000)p Fz(1)1650 2368 y Fg(\000)p Fi(1)1707 2356 y FD(+)i Fs(D)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])o(\))p Fw(;)d(:::;)189 2462 y FD(\()p Fw(ty)r(pe)291 2443 y Fv(count)q Fg(\000)p Fi(1)291 2474 y(0)435 2462 y Fw(;)g(disp)540 2443 y Fv(count)p Fg(\000)p Fi(1)540 2474 y(0)693 2462 y FD(+)i Fs(D)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])e(+)i(\()p Fs(B)p FD([)p Fs(count)g FA(\000)f Fs(1)p FD(])g FA(\000)g FD(1\))g FA(\001)f Fw(extent)1550 2470 y Fv(count)s Fg(\000)p Fi(1)1696 2462 y FD(\))p Fw(;)f(:::;)189 2550 y FD(\()p Fw(ty)r(pe)291 2530 y Fv(count)q Fg(\000)p Fi(1)291 2562 y Ff(n)312 2574 y Fv(count)q Fc(\000)p Fz(1)450 2562 y Fg(\000)p Fi(1)498 2550 y Fw(;)g(disp)603 2530 y Fv(count)o Fg(\000)p Fi(1)603 2562 y Ff(n)624 2574 y Fv(count)p Fc(\000)p Fz(1)761 2562 y Fg(\000)p Fi(1)818 2550 y FD(+)j Fs(D)p FD([)p Fs(count)g FA(\000)f Fs(1)p FD(])g(+)g(\()p Fs(B)p FD([)p Fs(count)h FA(\000)g Fs(1)p FD(])e FA(\000)h Fs(1)p FD(\))g FA(\001)g Fw(extent)1676 2558 y Fv(count)r Fg(\000)p Fi(1)1821 2550 y FD(\))p FA(g)166 2647 y FD(A)15 b(call)h(to)f Fs(MPI)p 439 2647 V 16 w(TYPE)p 572 2647 V 16 w(HINDEXED\()g(count,)h(B,)f(D,)g(oldt)o(yp)q(e,)h(newt)o(yp)q (e\))g FD(is)g(equiv)m(alen)o(t)h(to)d(a)h(call)75 2704 y(to)d Fs(MPI)p 213 2704 V 16 w(TYPE)p 346 2704 V 17 w(STRUCT\()h(count,)i(B,)e(D,)f(T,)h(newt)o(yp)q(e\))p FD(,)h(where)f(eac)o(h)g(en)o(try)g(of)g Fs(T)g FD(is)g(equal)h(to)e Fs(oldt)o(yp)q(e)p FD(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 66 73 66 72 bop 75 -100 a FD(66)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fn(3.13.2)49 b(Address)16 b(and)g(extent)f(functions)75 132 y FD(The)g(displacemen)o (ts)i(in)e(a)g(general)h(datat)o(yp)q(e)e(are)h(relativ)o(e)g(to)g (some)f(initial)j(bu\013er)e(address.)20 b Fx(Abso-)75 189 y(lute)15 b(addresses)c FD(can)h(b)q(e)h(substituted)f(for)g(these) g(displacemen)o(ts:)20 b(w)o(e)12 b(treat)f(them)h(as)g(displacemen)o (ts)75 245 y(relativ)o(e)17 b(to)e(\\address)h(zero",)g(the)g(start)f (of)h(the)g(address)h(space.)23 b(This)17 b(initial)h(address)e(zero)g (is)h(indi-)75 302 y(cated)f(b)o(y)g(the)g(constan)o(t)f Fv(MPI)p 601 302 13 2 v 14 w(BOTTOM)p FD(.)g(Th)o(us,)g(a)h(datat)o(yp) q(e)f(can)h(sp)q(ecify)h(the)f(absolute)g(address)g(of)75 358 y(the)e(en)o(tries)g(in)g(the)g(comm)o(unication)g(bu\013er,)g(in)h (whic)o(h)f(case)g(the)g Fs(buf)g FD(argumen)o(t)f(is)h(passed)g(the)g (v)m(alue)75 415 y Fv(MPI)p 152 415 V 14 w(BOTTOM)p FD(.)166 472 y(The)31 b(address)g(of)f(a)h(lo)q(cation)g(in)h(memory)e(can)h(b)q (e)h(found)f(b)o(y)g(in)o(v)o(oking)g(the)g(function)75 528 y Fs(MPI)p 160 528 14 2 v 16 w(ADDRESS)p FD(.)75 633 y Fs(MPI)p 160 633 V 16 w(ADDRESS\(lo)q(cation,)16 b(address\))117 710 y Ft(IN)155 b Fs(lo)q(cation)437 b Ft(lo)q(cation)13 b(in)h(caller)f(memory)f(\(c)o(hoice\))117 787 y(OUT)108 b Fs(address)449 b Ft(address)15 b(of)f(lo)q(cation)f (\(in)o(teger\))75 912 y FB(int)23 b(MPI)p 245 912 15 2 v 17 w(Address\(void*)f(location,)h(MPI)p 907 912 V 17 w(Aint)g(*address\))75 999 y(MPI)p 150 999 V 17 w (ADDRESS\(LOCATION,)e(ADDRESS,)i(IERROR\))170 1056 y()g (LOCATION\(*\))170 1112 y(INTEGER)g(ADDRESS,)g(IERROR)166 1199 y FD(Returns)16 b(the)f(\(b)o(yte\))f(address)i(of)e Fs(lo)q(cation)p FD(.)166 1256 y(Example:)147 1367 y FB(REAL)23 b(A\(100,100\))147 1423 y(INTEGER)f(I1,)i(I2,)f(DIFF)147 1480 y(CALL)g(MPI_ADDRESS\(A\(1,1\),)e(I1,)j(IERROR\))147 1536 y(CALL)f(MPI_ADDRESS\(A\(10,10\),)e(I2,)i(IERROR\))147 1593 y(DIFF)g(=)h(I2)f(-)h(I1)75 1649 y(!)g(The)f(value)g(of)h(DIFF)f (is)h(909*sizeofreal;)e(the)h(values)g(of)h(I1)f(and)h(I2)f(are)75 1706 y(!)h(implementation)e(dependent.)189 1815 y Fo(A)n(dvic)n(e)e(to) i(users.)73 b FD(C)20 b(users)h(ma)o(y)f(b)q(e)h(tempted)g(to)f(a)o(v)o (oid)h(the)f(usage)h(of)f Fs(MPI)p 1659 1815 14 2 v 16 w(ADDRESS)189 1871 y FD(and)e(rely)g(on)g(the)g(a)o(v)m(ailabilit)o(y)i (of)d(the)h(address)g(op)q(erator)f(&.)29 b(Note,)17 b(ho)o(w)o(ev)o(er,)h(that)f Fv(&)h Fo(c)n(ast-)189 1928 y(expr)n(ession)c FD(is)j(a)e(p)q(oin)o(ter,)i(not)e(an)h(address.)22 b(ANSI)17 b(C)f(do)q(es)g(not)g(require)g(that)g(the)g(v)m(alue)h(of)e (a)189 1984 y(p)q(oin)o(ter)g(\(or)g(the)g(p)q(oin)o(ter)h(cast)f(to)f Fv(int)p FD(\))h(b)q(e)h(the)g(absolute)f(address)h(of)e(the)i(ob)s (ject)e(p)q(oin)o(ted)j(at)d({)189 2041 y(although)j(this)g(is)h(the)f (common)g(case.)25 b(F)l(urthermore,)17 b(referencing)h(ma)o(y)e(not)h (ha)o(v)o(e)g(a)f(unique)189 2097 y(de\014nition)i(on)f(mac)o(hines)g (with)g(segmen)o(ted)g(address)g(space.)24 b(The)17 b(use)g(of)f Fs(MPI)p 1602 2097 V 16 w(ADDRESS)i FD(to)189 2153 y(\\reference")11 b(C)g(v)m(ariables)h(guaran)o(tees)e(p)q(ortabilit)o(y)i(to)f(suc)o(h)g (mac)o(hines)h(as)e(w)o(ell.)20 b(\()p Fo(End)12 b(of)g(advic)n(e)189 2210 y(to)k(users.)p FD(\))166 2320 y(The)f(follo)o(wing)h(auxiliary)h (functions)f(pro)o(vide)f(useful)i(information)e(on)g(deriv)o(ed)h (datat)o(yp)q(es.)75 2424 y Fs(MPI)p 160 2424 V 16 w(TYPE)p 293 2424 V 17 w(EXTENT\(datat)o(yp)q(e,)g(extent\))117 2502 y Ft(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(\(handle\))117 2578 y(OUT)108 b Fs(extent)471 b Ft(datat)o(yp)q(e)14 b(exten)o(t)h(\(in)o(teger\))75 2704 y FB(int)23 b(MPI)p 245 2704 15 2 v 17 w(Type)p 358 2704 V 17 w(extent\(MPI)p 615 2704 V 16 w(Datatype)f(datatype,)h(MPI)p 1156 2704 V 17 w(Aint)g(*extent\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 67 74 67 73 bop 75 -100 a Fp(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FD(67)75 45 y FB(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(EXTENT\(DATATYPE,)22 b(EXTENT,)h(IERROR\))170 102 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)166 189 y FD(Returns)14 b(the)g(exten)o(t)g(of)f(a)h(datat)o(yp)q(e)f(-)h(-)h ({)e(where)h(exten)o(t)g(is)g(as)g(de\014ned)h(in)g(Eq.)e(3.1)g(on)h (page)g(59.)75 293 y Fs(MPI)p 160 293 14 2 v 16 w(TYPE)p 293 293 V 17 w(SIZE\(datat)o(yp)q(e,)h(size\))117 371 y Ft(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(\(handle\))117 447 y(OUT)108 b Fs(size)519 b Ft(datat)o(yp)q(e)14 b(size)h(\(in)o(teger\))75 572 y FB(int)23 b(MPI)p 245 572 15 2 v 17 w(Type)p 358 572 V 17 w(size\(MPI)p 567 572 V 16 w(Datatype)g(datatype,)f(MPI)p 1108 572 V 17 w(Aint)h(*size\))75 659 y(MPI)p 150 659 V 17 w(TYPE)p 263 659 V 16 w(SIZE\(DATATYPE,)f(SIZE,)i(IERROR\))170 716 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)166 803 y FD(Returns)15 b(the)h(n)o(um)o(b)q(er)f(of)g(b)o(ytes)f(o)q(ccupied)j (b)o(y)e(en)o(tries)h(in)g(the)f(datat)o(yp)q(e)f({)h(i.e.)20 b(the)15 b(sum)g(of)g(the)75 859 y(sizes)h(of)f(the)g(datat)o(yp)q(e)g (elemen)o(ts.)75 964 y Fs(MPI)p 160 964 14 2 v 16 w(TYPE)p 293 964 V 17 w(COUNT\(datat)o(yp)q(e,)i(count\))117 1042 y Ft(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(\(handle\))117 1118 y(OUT)108 b Fs(count)482 b Ft(datat)o(yp)q(e)14 b(coun)o(t)g(\(in)o(teger\))75 1243 y FB(int)23 b(MPI)p 245 1243 15 2 v 17 w(Type)p 358 1243 V 17 w(count\(MPI)p 591 1243 V 16 w(Datatype)g(datatype,)f(int)i(*count\))75 1330 y(MPI)p 150 1330 V 17 w(TYPE)p 263 1330 V 16 w(COUNT\(DATATYPE,)e (COUNT,)h(IERROR\))170 1387 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)166 1474 y FD(Returns)16 b(the)f(n)o(um)o(b)q(er)g(of)g(en)o(tries)h(in)g (the)f(datat)o(yp)q(e.)75 1599 y Fn(3.13.3)49 b(Lo)o(w)o(er-b)q(ound)17 b(and)g(upp)q(er-b)q(ound)g(ma)o(rk)n(ers)75 1686 y FD(It)e(is)g(often) g(con)o(v)o(enien)o(t)g(to)g(de\014ne)h(explicitly)h(the)e(lo)o(w)o(er) g(b)q(ound)h(and)f(upp)q(er)h(b)q(ound)f(of)g(a)g(t)o(yp)q(e)g(map,)75 1743 y(and)20 b(o)o(v)o(erwrite)g(the)g(de\014nition)i(giv)o(en)f(b)o (y)f(Equation)g(3.1)g(on)g(page)g(59.)34 b(This)21 b(allo)o(ws)f(to)g (de\014ne)h(a)75 1799 y(datat)o(yp)q(e)12 b(that)g(has)h(\\holes")g(at) f(its)h(b)q(eginning)h(or)f(its)g(end;)g(or)g(a)f(datat)o(yp)q(e)g (with)h(en)o(tries)g(that)f(extend)75 1855 y(ab)q(o)o(v)o(e)k(the)g (upp)q(er)g(b)q(ound)h(or)f(b)q(elo)o(w)g(the)g(lo)o(w)o(er)g(b)q (ound.)23 b(Examples)16 b(of)g(suc)o(h)g(a)g(usage)g(are)f(pro)o(vided) 75 1912 y(in)h(section)g(3.13.7.)j(T)l(o)d(ac)o(hiev)o(e)g(this,)g(w)o (e)f(add)h(t)o(w)o(o)e(additional)j(\\pseudo-datat)o(yp)q(es")e Fs(MPI)p 1706 1912 14 2 v 16 w(LB)p FD(,)g(and)75 1968 y Fs(MPI)p 160 1968 V 16 w(UB)i FD(that)e(can)i(b)q(e)g(used,)g(resp)q (ectiv)o(ely)l(,)h(to)e(mark)f(the)i(lo)o(w)o(er)f(b)q(ound)h(or)f(the) h(upp)q(er)g(b)q(ound)g(of)f(a)75 2025 y(datat)o(yp)q(e.)j(These)c (pseudo-datat)o(yp)q(es)g(o)q(ccup)o(y)g(no)f(space)h(\()p Fw(extent)p FD(\()p Fs(MPI)p 1357 2025 V 17 w(LB)p FD(\))d(=)h Fw(extent)p FD(\()p Fs(MPI)p 1734 2025 V 17 w(UB)p FD(\))g(=)75 2081 y(0\).)18 b(They)c(do)f(not)f(a\013ect)g(the)h(size)h(or)f(coun)o (t)g(of)f(a)h(datat)o(yp)q(e,)f(and)h(do)g(not)g(a\013ect)f(the)h(the)g (con)o(ten)o(t)g(of)f(a)75 2138 y(message)i(created)h(with)h(this)f (datat)o(yp)q(e.)k(Ho)o(w)o(ev)o(er,)14 b(they)h(do)g(a\013ect)f(the)h (de\014nition)h(of)f(the)g(exten)o(t)f(of)75 2194 y(a)e(datat)o(yp)q(e) g(and,)h(therefore,)g(a\013ect)f(the)g(outcome)h(of)f(a)g(replication)i (of)f(this)g(datat)o(yp)q(e)f(b)o(y)g(a)g(datat)o(yp)q(e)75 2251 y(constructor.)166 2308 y(Example:)27 b(Let)19 b Fs(D)f(=)h(\(-3,)f(0,)h(6\))p FD(;)g Fs(T)f(=)h(\(MPI)p 990 2308 V 15 w(LB,)f(MPI)p 1173 2308 V 16 w(INT,)g(MPI)p 1378 2308 V 16 w(UB\))p FD(,)g(and)h Fs(B)f(=)h(\(1,)f(1,)h(1\))p FD(.)75 2364 y(Then)e(a)e(call)i(to)f Fs(MPI)p 459 2364 V 16 w(TYPE)p 592 2364 V 16 w(STRUCT\(3,)g(B,)g(D,)g(T,)f(t)o(yp)q (e1\))i FD(creates)e(a)h(new)g(datat)o(yp)q(e)g(that)f(has)h(an)75 2421 y(exten)o(t)f(of)g(9)g(\(from)f(-3)i(to)e(5\),)h(and)g(con)o (tains)h(an)f(in)o(teger)h(at)e(displacemen)o(t)j(0.)j(This)c(is)g(the) f(datat)o(yp)q(e)75 2477 y(de\014ned)g(b)o(y)g(the)f(sequence)h FA(f)p Fs(\(lb,)f(-3\),)f(\(int,)i(0\),)e(\(ub,)h(6\))p FA(g)g FD(.)19 b(If)c(this)f(t)o(yp)q(e)g(is)h(replicated)h(t)o(wice)e (b)o(y)g(a)g(call)75 2534 y(to)f Fs(MPI)p 214 2534 V 16 w(TYPE)p 347 2534 V 16 w(CONTIGUOUS\(2,)i(t)o(yp)q(e1,)f(t)o(yp)q (e2\))g FD(then)g(the)g(newly)g(created)g(t)o(yp)q(e)f(can)h(b)q(e)g (describ)q(ed)75 2590 y(b)o(y)f(the)g(sequence)h FA(f)p Fs(\(lb,)f(-3\),)g(\(int,)g(0\),)f(\(int,9\),)h(\(ub,)h(15\))p FA(g)d FD(.)20 b(\(En)o(tries)12 b(of)h(t)o(yp)q(e)g Fs(lb)g FD(or)g Fs(ub)h FD(can)f(b)q(e)h(deleted)75 2647 y(if)i(they)f(are)g(not)g(at)f(the)h(end-p)q(oin)o(ts)i(of)e(the)g (datat)o(yp)q(e.\))166 2704 y(In)h(general,)f(if)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 68 75 68 74 bop 75 -100 a FD(68)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)189 45 y Fw(T)6 b(y)r(pemap)12 b FD(=)h FA(f)p FD(\()p Fw(ty)r(pe)562 52 y Fi(0)581 45 y Fw(;)8 b(disp)686 52 y Fi(0)705 45 y FD(\))p Fw(;)g(:::;)g FD(\()o Fw(ty)r(pe)905 52 y Ff(n)p Fg(\000)p Fi(1)971 45 y Fw(;)g(disp)1076 52 y Ff(n)p Fg(\000)p Fi(1)1144 45 y FD(\))p FA(g)p Fw(;)75 147 y FD(then)16 b(the)f Fx(lo)o(w)o(er)h(b)q(ound)g FD(of)f Fw(T)6 b(y)r(pemap)15 b FD(is)g(de\014ned)i(to)e(b)q(e)189 282 y Fw(l)q(b)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))k(=)506 210 y Fe(\()560 254 y FD(min)636 261 y Ff(j)662 254 y Fw(disp)746 261 y Ff(j)1106 254 y FD(if)15 b(no)h(en)o(try)e(has)h (basic)h(t)o(yp)q(e)g Fs(lb)560 310 y FD(min)q FA(f)p Fw(disp)743 317 y Ff(j)788 310 y FD(:)28 b Fw(ty)r(pe)913 317 y Ff(j)944 310 y FD(=)13 b Fs(lb)p FA(g)42 b FD(otherwise)166 415 y(Similarly)l(,)17 b(the)e Fx(upp)q(er)j(b)q(ound)e FD(of)e Fw(T)6 b(y)r(pemap)15 b FD(is)h(de\014ned)h(to)d(b)q(e)189 549 y Fw(ub)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))11 b(=)518 477 y Fe(\()572 521 y FD(max)657 528 y Ff(j)682 521 y Fw(disp)766 528 y Ff(j)794 521 y FD(+)g Fw(siz)r(eof)5 b FD(\()p Fw(ty)r(pe)1072 528 y Ff(j)1090 521 y FD(\))57 b(if)15 b(no)h(en)o(try)e(has)h(basic)h(t)o(yp)q(e)g Fs(ub)572 577 y FD(max)p FA(f)p Fw(disp)764 584 y Ff(j)809 577 y FD(:)27 b Fw(ty)r(pe)933 584 y Ff(j)964 577 y FD(=)13 b Fs(ub)q FA(g)68 b FD(otherwise)166 682 y(Then)189 786 y Fw(extent)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))13 b(=)g Fw(ub)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))i FA(\000)j Fw(l)q(b)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))i(+)i Fw(\017)75 888 y FD(If)19 b Fw(ty)r(pe)208 895 y Ff(i)242 888 y FD(requires)h(alignmen)o(t)g(to)e(a)h(b)o(yte)g(address)g(that)g(is)h (a)e(m)o(ultiple)k(of)c Fw(k)1460 895 y Ff(i)1474 888 y FD(,)i(then)f Fw(\017)h FD(is)g(the)f(least)75 945 y(nonnegativ)o(e)d(incremen)o(t)g(needed)g(to)f(round)g Fw(extent)p FD(\()p Fw(T)6 b(y)r(pemap)p FD(\))16 b(to)e(the)h(next)h (m)o(ultiple)h(of)d(max)1794 952 y Ff(i)1816 945 y Fw(k)1840 952 y Ff(i)1854 945 y FD(.)166 1003 y(The)g(formal)f(de\014nitions)j (giv)o(en)e(for)f(the)h(v)m(arious)g(datat)o(yp)q(e)f(constructors)g (apply)i(no)o(w,)e(with)h(the)75 1060 y(amended)i(de\014nition)h(of)e Fx(exten)o(t)p FD(.)166 1194 y Fl(Discussion:)34 b Ft(The)14 b(follo)o(wing)d(additional)h(functions)i(w)o(ere)h(suggested:)-1594 b Fm(CHANGE)166 1246 y Fv(MPI)p 243 1246 13 2 v 14 w(TYPE)p 365 1246 V 14 w(LB\()15 b(datat)o(yp)q(e,)f(displacement\))75 1296 y Ft(Returns)h(the)f(lo)o(w)o(er)g(b)q(ound)g(of)f(the)h(t)o(yp)q (e)h(map)d(of)h(datat)o(yp)q(e.)166 1348 y Fv(MPI)p 243 1348 V 14 w(TYPE)p 365 1348 V 14 w(UB\()h(datat)o(yp)q(e,)g (displacement\))75 1398 y Ft(Returns)h(the)f(upp)q(er)h(b)q(ound)f(of)f (the)i(t)o(yp)q(e)f(map)f(of)g(datat)o(yp)q(e.)75 1615 y Fn(3.13.4)49 b(Comm)o(it)13 b(and)k(free)75 1704 y FD(A)g(datat)o(yp)q(e)f(ob)s(ject)h(has)g(to)f(b)q(e)h Fx(committed)h FD(b)q(efore)f(it)g(can)g(b)q(e)h(used)g(in)f(a)g(comm)o (unication.)26 b(The)75 1761 y(system)20 b(ma)o(y)g(\\compile")i(at)e (commit)g(time)h(an)g(in)o(ternal)g(represen)o(tation)g(for)f(the)h (datat)o(yp)q(e)f(that)75 1817 y(facilitates)c(comm)o(unication,)g (e.g.)k(c)o(hange)15 b(from)g(a)g(compacted)g(represen)o(tation)h(to)e (a)h(\015at)g(represen-)75 1874 y(tation)i(of)f(the)h(datat)o(yp)q(e,)g (and)g(select)g(the)g(most)g(con)o(v)o(enien)o(t)g(transfer)f(mec)o (hanism.)26 b(A)17 b(committed)75 1930 y(datat)o(yp)q(e)e(can)g(still)i (b)q(e)e(used)h(as)f(a)g(argumen)o(t)f(in)i(datat)o(yp)q(e)f (constructors.)166 1989 y(Note:)28 b(the)20 b(commit)f(op)q(eration)h (commits)f(the)h(datat)o(yp)q(e,)f(i.e.,)i(the)e(formal)g(description)i (of)e(a)75 2045 y(comm)o(unication)i(bu\013er,)h(not)e(the)h(con)o(ten) o(t)f(of)g(that)g(bu\013er.)36 b(Th)o(us,)22 b(after)e(a)g(datat)o(yp)q (e)g(has)h(b)q(een)75 2102 y(committed,)d(it)h(can)f(b)q(e)h(rep)q (eatedly)g(reused)g(to)e(comm)o(unicate)h(the)h(c)o(hanging)f(con)o (ten)o(t)g(of)f(a)h(bu\013er)75 2158 y(or,)c(indeed,)j(the)e(con)o(ten) o(t)g(of)g(di\013eren)o(t)g(bu\013ers,)g(with)g(di\013eren)o(t)h (starting)e(addresses.)75 2264 y Fs(MPI)p 160 2264 14 2 v 16 w(TYPE)p 293 2264 V 17 w(COMMIT\(datat)o(yp)q(e\))117 2343 y Ft(INOUT)62 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(that)g(is)g(committed)e(\(handle\))75 2470 y FB(int)23 b(MPI)p 245 2470 15 2 v 17 w(Type)p 358 2470 V 17 w(commit\(MPI)p 615 2470 V 16 w(Datatype)f(*datatype\))75 2559 y(MPI)p 150 2559 V 17 w(TYPE)p 263 2559 V 16 w(COMMIT\(DATATYPE,)g(IERROR\))170 2615 y(INTEGER)h(DATATYPE,)g(IERROR)166 2704 y FD(There)15 b(is)h(no)f(need)h(to)f(commit)g(basic)h(datat)o(yp)q(es;)e(they)h(are) g(\\pre-committed".)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 69 76 69 75 bop 75 -100 a Fp(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FD(69)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(FREE\(datat)o(yp)q(e\))117 123 y Ft(INOUT)62 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(that)g(is)g(freed)h (\(handle\))75 247 y FB(int)23 b(MPI)p 245 247 15 2 v 17 w(Type)p 358 247 V 17 w(free\(MPI)p 567 247 V 16 w(Datatype)g (*datatype\))75 334 y(MPI)p 150 334 V 17 w(TYPE)p 263 334 V 16 w(FREE\(DATATYPE,)f(IERROR\))170 391 y(INTEGER)h(DATATYPE,)g (IERROR)166 478 y FD(Marks)16 b(the)h(datat)o(yp)q(e)f(ob)s(ject)g (asso)q(ciated)h(with)g Fs(datat)o(yp)q(e)i FD(for)d(deallo)q(cation.) 26 b(The)17 b(ob)s(ject)f(will)75 534 y(b)q(e)g(deallo)q(cated)h(after) d(an)o(y)i(p)q(ending)h(comm)o(unication)f(that)e(uses)i(this)g(ob)s (ject)f(completes,)g(at)g(whic)o(h)75 590 y(p)q(oin)o(t)21 b Fs(datat)o(yp)q(e)g FD(b)q(ecomes)g(n)o(ull.)36 b(It)20 b(is)h(erroneous)f(to)f(use)i(a)f(datat)o(yp)q(e)f(as)h(argumen)o(t)f (to)h(an)g(MPI)75 647 y(function)c(after)e(it)i(w)o(as)e(freed.)166 786 y Fl(Discussion:)33 b Ft(Alternativ)o(e)13 b(de\014nition:)18 b(The)13 b(call)g(returns)h Fv(null)g Ft(in)f Fv(datat)o(yp)q(e)p Ft(.)19 b(See)14 b(previous)f(discussion)96 b Fm(CHANGE)75 843 y Ft(on)14 b(p)q(ersisten)o(t)h(requests.)166 982 y FD(Example:)75 1147 y FB(INTEGER)23 b(type1,)g(type2)75 1204 y(CALL)g(MPI_TYPE_CONTIGUOUS\(5,)e(MPI_REAL,)i(type1,)g(ierr\))409 1260 y(!)h(new)f(type)h(object)f(created)75 1316 y(CALL)g (MPI_TYPE_COMMIT\(type1,)e(ierr\))409 1373 y(!)j(now)f(type1)g(can)h (be)f(used)h(for)f(communication)75 1429 y(type2)g(=)h(type1)409 1486 y(!)g(type2)f(can)g(be)h(used)f(for)h(communication)409 1542 y(!)g(\(it)f(is)h(a)g(handle)f(to)g(same)h(object)f(as)g(type1\)) 75 1599 y(CALL)g(MPI_TYPE_VECTOR\(3,)f(5,)h(4,)h(MPI_REAL,)f(type1,)g (ierr\))409 1655 y(!)h(new)f(uncommitted)g(type)g(object)g(created)75 1712 y(CALL)g(MPI_TYPE_COMMIT\(type1,)e(ierr\))409 1768 y(!)j(now)f(type1)g(can)h(be)f(used)h(anew)f(for)h(communication)166 1876 y FD(F)l(reeing)19 b(a)f(datat)o(yp)q(e)f(do)q(es)i(not)f (a\013ect)f(an)o(y)h(other)g(datat)o(yp)q(e)f(that)h(w)o(as)f(built)j (from)d(the)i(freed)75 1933 y(datat)o(yp)q(e.)32 b(The)20 b(system)e(b)q(eha)o(v)o(es)i(as)f(if)h(input)g(datat)o(yp)q(e)f (argumen)o(ts)g(to)f(deriv)o(ed)j(datat)o(yp)q(e)e(con-)75 1989 y(structors)14 b(are)h(passed)g(b)o(y)h(v)m(alue.)166 2128 y Fl(Implemen)o(tati)o(on)j(note:)57 b Ft(The)20 b(implemen)o(tation)c(ma)o(y)i(k)o(eep)i(a)f(reference)j(coun)o(t)d(of) g(activ)o(e)g(com-)75 2185 y(m)o(unications)c(that)i(use)h(the)f(datat) o(yp)q(e,)h(in)e(order)i(to)f(decide)h(when)f(to)g(free)h(it.)27 b(Also,)17 b(one)g(ma)o(y)e(implem)o(en)o(t)75 2241 y(constructors)k (of)e(deriv)o(ed)h(datat)o(yp)q(es)f(so)h(that)f(they)h(k)o(eep)f(p)q (oin)o(ters)h(to)f(their)h(datat)o(yp)q(e)f(argumen)o(ts,)g(rather)75 2298 y(then)d(cop)o(ying)f(them.)k(In)d(suc)o(h)g(case,)g(one)g(needs)g (to)g(k)o(eep)g(trac)o(k)g(of)f(activ)o(e)g(datat)o(yp)q(e)h (de\014nition)f(references)j(in)75 2354 y(order)f(to)e(kno)o(w)h(when)g (a)g(datat)o(yp)q(e)g(ob)r(ject)h(can)f(b)q(e)g(freed.)75 2561 y Fn(3.13.5)49 b(Use)16 b(of)h(general)f(datat)o(yp)q(es)g(in)h (comm)n(unication)75 2647 y FD(Handles)g(to)f(deriv)o(ed)h(datat)o(yp)q (es)f(can)h(b)q(e)f(passed)h(to)f(a)g(comm)o(unication)h(call)g(wherev) o(er)f(a)g(datat)o(yp)q(e)75 2704 y(argumen)o(t)11 b(is)h(required.)19 b(A)12 b(call)g(of)f(the)h(form)e Fs(MPI)p 947 2704 14 2 v 16 w(SEND\(buf,)j(count,)g(datat)o(yp)q(e)g(,)f(...\))p FD(,)e(where)i Fs(count)i Fw(>)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 70 77 70 76 bop 75 -100 a FD(70)613 b Fp(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(1)p FD(,)c(is)h(in)o(terpreted)g(as)g(if)g(the)f(call)i(w)o(as)d(passed)i (a)f(new)h(datat)o(yp)q(e)f(whic)o(h)h(is)g(the)g(concatenation)g(of)f Fs(count)75 102 y FD(copies)k(of)f Fs(datat)o(yp)q(e)p FD(.)21 b(Th)o(us,)166 158 y Fs(MPI)p 251 158 14 2 v 16 w(SEND\(buf,)15 b(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm)m (\))d FD(is)k(equiv)m(alen)o(t)h(to)75 260 y FB (MPI_TYPE_CONTIGUOUS\(count,)j(datatype,)j(newtype\))75 317 y(MPI_TYPE_COMMIT\(newtype\))75 373 y(MPI_SEND\(buf,)f(1,)i (newtype,)e(dest,)i(tag,)f(comm\))166 475 y FD(The)14 b(same)g(applies)h(to)f(all)h(other)e(comm)o(unication)i(functions)g (that)e(ha)o(v)o(e)g(a)h Fs(count)h FD(and)f Fs(datat)o(yp)q(e)75 532 y FD(argumen)o(t.)166 588 y(Supp)q(ose)k(that)e(a)g(a)h(send)g(op)q (eration)g Fs(MPI)p 920 588 V 16 w(SEND\(buf,)h(count,)g(datat)o(yp)q (e.)26 b(dest,)18 b(tag,)f(comm)m(\))c FD(is)75 645 y(executed,)j (where)f Fs(datat)o(yp)q(e)i FD(ha)o(v)o(e)e(t)o(yp)q(e)g(map)189 747 y FA(f)p FD(\()p Fw(ty)r(pe)314 754 y Fi(0)333 747 y Fw(;)8 b(disp)438 754 y Fi(0)457 747 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 754 y Ff(n)p Fg(\000)p Fi(1)723 747 y Fw(;)h(disp)828 754 y Ff(n)p Fg(\000)p Fi(1)895 747 y FD(\))p FA(g)p Fw(;)166 849 y FD(with)20 b(exten)o(t)f Fw(extent)i FD(\(empt)o(y)e(en)o(tries)g(deleted\).)34 b(Then)20 b(the)g(send)g(bu\013er)f(used)h(b)o(y)g(the)f(send)75 905 y(op)q(eration)c(consists)h(of)f Fw(n)10 b FA(\001)g Fs(count)16 b FD(en)o(tries,)f(where)h(en)o(try)f Fw(i)9 b FA(\001)h Fw(n)h FD(+)f Fw(j)17 b FD(is)f(at)f(lo)q(cation)189 1008 y Fw(addr)282 1015 y Ff(i;j)333 1008 y FD(=)e Fs(buf)h FD(+)d Fw(extent)g FA(\001)f Fw(i)g FD(+)g Fw(disp)817 1015 y Ff(j)166 1110 y FD(and)15 b(has)h(t)o(yp)q(e)f Fw(ty)r(pe)522 1117 y Ff(j)540 1110 y FD(,)g(for)g Fw(i)d FD(=)i(0)p Fw(;)8 b(:::;)g Fs(co)o(unt)h FA(\000)i FD(1)j(and)i Fw(j)f FD(=)e(0)p Fw(;)8 b(:::;)g(n)g FA(\000)i FD(1.)20 b(These)c(en)o(tries)f(need)h(not)75 1166 y(b)q(e)g(con)o(tiguous,)f (nor)g(distinct;)h(their)f(order)g(can)h(b)q(e)g(arbitrary)l(.)166 1223 y(The)g(v)m(ariable)h(stored)e(at)g(address)g Fw(addr)879 1230 y Ff(i;j)933 1223 y FD(in)i(the)e(calling)j(program)c(should)j(b)q (e)f(of)f(a)g(t)o(yp)q(e)h(that)75 1279 y(matc)o(hes)d Fw(ty)r(pe)333 1286 y Ff(j)352 1279 y FD(,)h(where)g(t)o(yp)q(e)g(matc) o(hing)f(is)i(de\014ned)g(as)e(in)i(section)f(3.5.)19 b(The)14 b(message)f(sen)o(t)h(con)o(tains)75 1336 y Fw(n)c FA(\001)g Fs(count)17 b FD(en)o(tries,)e(where)g(en)o(try)g Fw(i)10 b FA(\001)g Fw(n)g FD(+)g Fw(j)18 b FD(has)d(t)o(yp)q(e)g Fw(ty)r(pe)1100 1343 y Ff(j)1119 1336 y FD(.)166 1392 y(Similarly)l(,)g(supp)q(ose)f(that)e(a)h(receiv)o(e)h(op)q(eration)f Fs(MPI)p 1100 1392 V 16 w(RECV\(buf,)h(count,)g(datat)o(yp)q(e,)h (source,)e(tag,)75 1449 y(comm)m(,)f(status\))17 b FD(is)e(executed,)h (where)g Fs(datat)o(yp)q(e)g FD(has)g(t)o(yp)q(e)f(map)189 1551 y FA(f)p FD(\()p Fw(ty)r(pe)314 1558 y Fi(0)333 1551 y Fw(;)8 b(disp)438 1558 y Fi(0)457 1551 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 1558 y Ff(n)p Fg(\000)p Fi(1)723 1551 y Fw(;)h(disp)828 1558 y Ff(n)p Fg(\000)p Fi(1)895 1551 y FD(\))p FA(g)p Fw(;)166 1653 y FD(with)k(exten)o(t)f Fw(extent)i FD(\(empt)o(y)e(en)o(tries)i(deleted\).)19 b(Then)12 b(the)g(receiv)o(e)h(bu\013er)f(used)g(for)f(this)h(receiv)o (e)75 1709 y(op)q(eration)j(consists)h(of)e(the)h Fw(n)c FA(\001)e Fs(count)16 b FD(en)o(tries,)f(where)h(en)o(try)e Fw(i)c FA(\001)f Fw(n)i FD(+)f Fw(j)17 b FD(is)f(at)e(lo)q(cation)i Fs(buf)e FD(+)c Fw(extent)h FA(\001)75 1766 y Fw(i)d FD(+)h Fw(disp)227 1773 y Ff(j)259 1766 y FD(and)15 b(has)f(t)o(yp)q(e) h Fw(ty)r(pe)613 1773 y Ff(j)631 1766 y FD(.)20 b(If)15 b(the)f(incoming)i(message)e(consists)g(of)g Fw(k)i FD(elemen)o(ts,)f (then)f(w)o(e)h(m)o(ust)75 1822 y(ha)o(v)o(e)f Fw(k)g FA(\024)f Fw(n)d FA(\001)f Fs(count)p FD(;)15 b(the)g Fw(i)9 b FA(\001)g Fw(n)h FD(+)f Fw(j)s FD(-th)15 b(elemen)o(t)g(of)g (the)f(message)h(should)h(ha)o(v)o(e)e(a)g(t)o(yp)q(e)h(that)f(matc)o (hes)75 1879 y Fw(ty)r(pe)159 1886 y Ff(j)178 1879 y FD(.)166 1935 y(Note)d(that)g(t)o(yp)q(e)h(matc)o(hing)f(is)h (de\014ned)h(according)f(to)f(the)h(t)o(yp)q(e)f(signature)h(of)f(the)h (corresp)q(onding)75 1992 y(datat)o(yp)q(es,)f(i.e.,)h(the)f(sequence)i (of)e(basic)h(t)o(yp)q(e)g(comp)q(onen)o(ts,)f(and)h(do)q(es)g(not)f (dep)q(end)i(on)e(the)g(datat)o(yp)q(e)75 2048 y(de\014nition.)166 2105 y(Example:)75 2195 y FB(...)75 2252 y(CALL)23 b (MPI_TYPE_CONTIGUOUS\()e(2,)j(MPI_REAL,)f(type2,)g(...\))75 2308 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e(4,)j(MPI_REAL,)f(type4,)g(...\)) 75 2365 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e(2,)j(type2,)f(type22,)g(...\)) 75 2421 y(...)75 2478 y(CALL)g(MPI_SEND\()g(a,)h(4,)f(MPI_REAL,)g (...\))75 2534 y(CALL)g(MPI_SEND\()g(a,)h(2,)f(type2,)g(...\))75 2591 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type22,)g(...\))75 2647 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type4,)g(...\))75 2704 y(...)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 71 78 71 77 bop 75 -100 a Fp(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FD(71)75 45 y FB(CALL)23 b(MPI_RECV\()g(a,)h(4,)f(MPI_REAL,)g (...\))75 102 y(CALL)g(MPI_RECV\()g(a,)h(2,)f(type2,)g(...\))75 158 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type22,)g(...\))75 214 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type4,)g(...\))166 305 y FD(>F)l(rom)14 b(a)g(t)o(yp)q(e)g(matc)o(hing)g(viewp)q(oin)o(t)h (an)o(y)f(of)g(the)g(sends)h(legally)g(matc)o(hes)f(an)o(y)g(of)f(the)h (receiv)o(es.)166 361 y(A)22 b(datat)o(yp)q(e)f(ma)o(y)g(sp)q(ecify)i (o)o(v)o(erlapping)g(en)o(tries.)40 b(If)22 b(suc)o(h)h(datat)o(yp)q(e) e(is)h(used)h(in)f(a)g(receiv)o(e)75 418 y(op)q(eration,)17 b(i.e.,)f(if)h(the)f(some)g(part)g(of)g(the)g(receiv)o(e)i(bu\013er)e (is)h(o)o(v)o(erwritten)f(more)g(than)g(once)h(b)o(y)f(the)75 474 y(receiv)o(e)g(op)q(eration,)f(then)h(the)f(outcome)g(of)g(the)g (receiv)o(e)h(is)g(unde\014ned.)166 531 y(Supp)q(ose,)j(again,)f(that)f Fs(MPI)p 683 531 14 2 v 15 w(RECV\(buf,)j(count,)f(datat)o(yp)q(e,)g (dest,)g(tag,)f(comm)m(,)d(status\))k FD(is)f(exe-)75 587 y(cuted,)d(where)h Fs(datat)o(yp)q(e)h FD(has)e(t)o(yp)q(e)g(map) 189 688 y FA(f)p FD(\()p Fw(ty)r(pe)314 695 y Fi(0)333 688 y Fw(;)8 b(disp)438 695 y Fi(0)457 688 y FD(\))p Fw(;)g(:::;)f FD(\()p Fw(ty)r(pe)657 695 y Ff(n)p Fg(\000)p Fi(1)723 688 y Fw(;)h(disp)828 695 y Ff(n)p Fg(\000)p Fi(1)895 688 y FD(\))p FA(g)p Fw(:)166 790 y FD(The)k(receiv)o(ed)g (message)f(need)h(not)f(\014ll)i(all)g(the)e(receiv)o(e)h(bu\013er,)g (nor)f(do)q(es)h(it)g(need)g(to)f(\014ll)i(a)e(n)o(um)o(b)q(er)75 846 y(of)i(lo)q(cations)h(whic)o(h)g(is)f(a)g(m)o(ultiple)i(of)e Fw(n)p FD(;)h(an)o(y)f(n)o(um)o(b)q(er)g Fw(k)i FD(of)d(basic)i(elemen) o(ts)g(can)f(b)q(e)h(receiv)o(ed,)h(where)75 903 y(0)d FA(\024)h Fw(k)h FA(\024)f Fs(count)e FA(\001)f Fw(n)p FD(.)166 959 y(The)k(n)o(um)o(b)q(er)g(of)f(elemen)o(ts)i(receiv)o(ed)g (can)f(b)q(e)g(retriev)o(ed)g(from)f Fs(status)j FD(using)e(the)g (query)g(function)75 1015 y Fs(MPI)p 160 1015 V 16 w(GET)p 264 1015 V 17 w(ELEMENTS)p FD(:)75 1119 y Fs(MPI)p 160 1119 V 16 w(GET)p 264 1119 V 17 w(ELEMENTS\()h(status,)h(datat)o(yp)q (e,)h(count\))117 1196 y Ft(IN)155 b Fs(status)476 b Ft(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 1271 y(IN)155 b Fs(datat)o(yp)q(e)424 b Ft(datat)o(yp)q(e)14 b(used)h(b)o(y)f(receiv)o(e)h(op)q(eration)f(\(handle\))117 1345 y(OUT)108 b Fs(count)482 b Ft(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(basic)f(elemen)o(ts)g(\(in)o(teger\))75 1470 y FB(int)23 b(MPI)p 245 1470 15 2 v 17 w(Get)p 334 1470 V 17 w(elements\(MPI)p 639 1470 V 16 w(Status)g(status,)f(MPI)p 1084 1470 V 17 w(Datatype)h(datatype,)g(int)g(*count\))75 1556 y(MPI)p 150 1556 V 17 w(GET)p 239 1556 V 17 w(ELEMENTS\(STATUS,)e (DATATYPE,)i(COUNT,)g(IERROR\))170 1613 y(INTEGER)g(STATUS\(MPI)p 604 1613 V 16 w(STATUS)p 764 1613 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 1699 y FD(The)13 b(previously)h(used)f(function)h Fs(MPI)p 834 1699 14 2 v 16 w(GET)p 938 1699 V 16 w(COUNT)g FD(has)f(a)f(di\013eren)o(t)h(b)q(eha)o(vior:)19 b(it)13 b(returns)g(the)75 1755 y(n)o(um)o(b)q(er)g(of)f(\\top)g(elemen)o(ts")h (receiv)o(ed.)21 b(In)13 b(the)g(previous)g(example,)h Fs(MPI)p 1368 1755 V 16 w(GET)p 1472 1755 V 16 w(COUNT)g FD(ma)o(y)e(return)75 1812 y(an)o(y)i(in)o(teger)h(v)m(alue)g Fw(k)q FD(,)g(where)f(0)f FA(\024)g Fw(k)g FA(\024)g Fs(count)r FD(.)19 b(If)c Fs(MPI)p 1047 1812 V 16 w(GET)p 1151 1812 V 16 w(COUNT)h FD(returns)e Fw(k)q FD(,)g(then)h(the)g(n)o (um)o(b)q(er)75 1868 y(of)f(basic)h(elemen)o(ts)f(receiv)o(ed)i(\(and)e (the)g(v)m(alue)h(returned)g(b)o(y)f Fs(MPI)p 1226 1868 V 16 w(GET)p 1330 1868 V 16 w(ELEMENTS)p FD(\))g(is)h Fw(n)8 b FA(\001)g Fw(k)q FD(.)19 b(If)c(the)75 1925 y(n)o(um)o(b)q(er)j(of)f(basic)h(elemen)o(ts)g(receiv)o(ed)h(is)f(not)f (a)h(m)o(ultiple)h(of)e Fw(n)p FD(,)h(i.e.)27 b(if)18 b(the)g(receiv)o(e)g(op)q(eration)g(has)75 1981 y(not)f(receiv)o(ed)i (an)f(in)o(tegral)g(n)o(um)o(b)q(er)g(of)f Fs(datat)o(yp)q(e)j FD(\\copies",)e(then)g Fs(MPI)p 1360 1981 V 16 w(GET)p 1464 1981 V 17 w(COUNT)g FD(returns)g(the)75 2038 y(v)m(alue)e Fv(MPI)p 269 2038 13 2 v 15 w(UNDEFINED)p FD(.)166 2094 y(Example:)75 2195 y FB(...)75 2252 y(CALL)23 b (MPI_TYPE_CONTIGUOUS\(2,)e(MPI_REAL,)i(Type2,)g(ierr\))75 2308 y(CALL)g(MPI_COMMIT\(Type2,)f(ierr\))75 2365 y(...)75 2421 y(CALL)h(MPI_RANK\(comm,)f(rank,)h(ierr\))75 2478 y(IF\(rank.EQ.0\))147 2534 y(THEN)218 2591 y(CALL)g(MPI_SEND\(a,)g(2,)g (MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 2647 y(CALL)g(MPI_SEND\(a,)g (3,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))147 2704 y(ELSE)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 72 79 72 78 bop 75 -100 a FD(72)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)218 45 y FB(CALL)23 b(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g(ierr\))218 102 y(CALL)g(MPI_GET_COUNT\(stat,)f(Type2,)h(i,)g(ierr\))119 b(!)24 b(returns)f(i=1)218 158 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e (Type2,)i(i,)h(ierr\))47 b(!)24 b(returns)f(i=2)218 214 y(CALL)g(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g(ierr\)) 218 271 y(CALL)g(MPI_GET_COUNT\(stat,)f(Type2,)h(i,)g(ierr\))119 b(!)24 b(returns)f(i=MPI_UNDEFINED)218 327 y(CALL)g (MPI_GET_ELEMENTS\(stat,)e(Type2,)i(i,)h(ierr\))47 b(!)24 b(returns)f(i=3)75 384 y(END)g(IF)166 500 y FD(The)15 b(function)h Fs(MPI)p 522 500 14 2 v 16 w(GET)p 626 500 V 17 w(ELEMENTS)f FD(can)g(also)g(b)q(e)g(used)h(after)e(a)h(prob)q(e)g (to)g(\014nd)h(the)f(n)o(um)o(b)q(er)75 556 y(of)20 b(elemen)o(ts)h(in) g(the)f(prob)q(ed)h(message.)34 b(Note)20 b(that)f(the)h(t)o(w)o(o)f (functions)i Fs(MPI)p 1507 556 V 16 w(GET)p 1611 556 V 17 w(COUNT)g FD(and)75 613 y Fs(MPI)p 160 613 V 16 w(GET)p 264 613 V 17 w(ELEMENTS)15 b FD(return)g(the)g(same)g(v)m (alues)i(when)e(they)h(are)e(used)i(with)g(basic)g(datat)o(yp)q(es.)189 729 y Fo(R)n(ationale.)38 b FD(The)14 b(extension)g(giv)o(en)g(to)f (the)h(de\014nition)h(of)e Fs(MPI)p 1303 729 V 16 w(GET)p 1407 729 V 16 w(COUNT)i FD(seems)e(natural:)189 785 y(one)j(w)o(ould)h (exp)q(ect)h(this)f(function)g(to)f(return)h(the)f(v)m(alue)i(of)e(the) h Fs(count)h FD(argumen)o(t,)e(when)h(the)189 842 y(receiv)o(e)e (bu\013er)f(is)h(\014lled.)22 b(Often)15 b(times)f Fs(datat)o(yp)q(e)i FD(represen)o(ts)f(a)f(basic)h(unit)g(of)f(data)g(one)g(w)o(an)o(ts)189 898 y(to)19 b(transfer;)j(e.g.,)e(a)g(record)g(in)h(an)f(arra)o(y)f(of) h(records)g(\(structures\).)34 b(One)21 b(should)g(b)q(e)g(able)189 954 y(to)f(\014nd)i(out)f(ho)o(w)f(man)o(y)h(comp)q(onen)o(ts)g(where)g (receiv)o(ed)i(without)e(b)q(othering)h(to)e(divide)j(b)o(y)189 1011 y(the)18 b(n)o(um)o(b)q(er)h(of)f(elemen)o(ts)i(in)f(eac)o(h)g (comp)q(onen)o(t.)30 b(Ho)o(w)o(ev)o(er,)18 b(on)h(other)f(o)q (ccasions,)h Fs(datat)o(yp)q(e)189 1067 y FD(is)h(used)h(to)f (de\014ned)h(a)f(complex)h(la)o(y)o(out)f(of)g(data)f(in)i(the)g (receiv)o(er)g(memory)l(,)f(and)h(do)q(es)f(not)189 1124 y(represen)o(t)i(a)g(basic)h(unit)g(of)f(data)g(for)f(transfers.)41 b(In)23 b(suc)o(h)f(cases,)i(one)e(needs)i(to)d(use)i(the)189 1180 y(function)16 b Fs(MPI)p 452 1180 V 15 w(GET)p 555 1180 V 17 w(ELEMENTS)p FD(.)f(\()p Fo(End)h(of)g(r)n(ationale.)p FD(\))166 1372 y Fl(Implemen)o(tati)o(on)i(note:)58 b Ft(The)19 b(curren)o(t)i(de\014nition)e(implies)e(that)i(a)g(receiv)o (e)h(cannot)g(c)o(hange)f(the)75 1422 y(v)n(alue)d(of)g(storage)h(b)q (et)o(w)o(een)h(the)f(en)o(tries)g(de\014ned)h(to)e(comp)q(ose)g(the)h (comm)o(unication)c(bu\013er.)27 b(In)17 b(particular,)75 1472 y(the)c(de\014nition)g(implies)e(that)i(padding)f(space)i(in)e(a)h (structure)i(should)d(not)h(b)q(e)g(mo)q(di\014ed)f(when)h(suc)o(h)h (structure)75 1522 y(is)f(copied)g(from)f(one)h(pro)q(cess)i(to)e (another.)18 b(This)13 b(w)o(ould)f(prev)o(en)o(ts)j(the)f(ob)o(vious)e (optimization)e(of)j(cop)o(ying)f(the)75 1571 y(structure,)i(together)f (with)e(the)i(padding,)e(as)h(one)g(con)o(tiguous)f(blo)q(c)o(k.)18 b(The)12 b(implemen)o(tatio)o(n)d(is)j(free)h(to)e(do)h(this)75 1621 y(optimization)h(when)j(it)f(do)q(es)i(not)e(impact)f(the)i (outcome)f(of)g(the)h(computation.)22 b(The)16 b(user)g(can)g(\\force") g(this)75 1671 y(optimization)11 b(b)o(y)j(explicitly)f(including)g (padding)g(as)h(part)g(of)f(the)h(message.)75 1887 y Fn(3.13.6)49 b(Co)o(rrect)15 b(use)h(of)g(addresses)75 1976 y FD(Successiv)o(ely)23 b(declared)f(v)m(ariables)h(in)e(C)g(or)g (F)l(ortran)e(are)i(not)g(necessarily)h(stored)f(at)f(con)o(tiguous)75 2033 y(lo)q(cations.)i(Th)o(us,)16 b(care)g(m)o(ust)f(b)q(e)h (exercised)i(that)d(displacemen)o(ts)i(do)f(not)f(cross)h(from)f(one)h (v)m(ariable)75 2089 y(to)e(another.)20 b(Also,)15 b(in)g(mac)o(hines)h (with)f(segmen)o(ted)g(address)g(space,)g(addresses)g(are)g(not)f (unique)j(and)75 2145 y(address)11 b(arithmetic)h(has)g(some)f(p)q (eculiar)i(prop)q(erties.)19 b(Th)o(us,)12 b(use)g(of)f Fx(addresses)p FD(,)f(i.e.)20 b(displacemen)o(ts)75 2202 y(relativ)o(e)c(to)e(the)i(start)e(address)29 b Fv(MPI)p 733 2202 13 2 v 14 w(BOTTOM)p FD(,)14 b(has)h(to)g(b)q(e)h(restricted.) 166 2260 y(V)l(ariables)g(b)q(elong)f(to)f(the)g(same)g Fx(sequen)o(tial)k(storage)c FD(if)h(they)f(b)q(elong)i(to)e(the)g (same)g(arra)o(y)l(,)g(to)75 2317 y(the)k(same)g Fs(COMMON)g FD(blo)q(c)o(k)h(in)g(F)l(ortran,)e(or)h(to)f(the)h(same)g(structure)f (in)i(C.)f(V)l(alid)h(addresses)f(are)75 2373 y(de\014ned)f(recursiv)o (ely)f(as)f(follo)o(ws:)131 2489 y(1.)22 b(The)g(function)g Fs(MPI)p 558 2489 14 2 v 16 w(ADDRESS)h FD(returns)e(a)h(v)m(alid)h (address,)h(when)e(passed)g(as)f(argumen)o(t)g(a)189 2546 y(v)m(ariable)16 b(of)f(the)g(calling)i(program.)131 2647 y(2.)22 b(The)d Fs(buf)h FD(argumen)o(t)f(of)g(a)g(comm)o (unication)h(function)g(ev)m(aluates)h(to)d(a)h(v)m(alid)i(address,)f (when)189 2704 y(passed)15 b(as)g(argumen)o(t)f(a)h(v)m(ariable)i(of)e (the)g(calling)i(program.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 73 80 73 79 bop 75 -100 a Fp(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FD(73)131 45 y(3.)22 b(If)15 b Fs(v)g FD(is)h(a)f(v)m(alid)i (address,)d(and)i Fs(i)f FD(is)g(an)h(in)o(teger,)f(then)g Fs(v+i)g FD(is)h(a)f(v)m(alid)i(address,)e(pro)o(vided)h Fs(v)f FD(and)189 102 y Fs(v+i)g FD(are)g(in)h(the)f(same)g(sequen)o (tial)h(storage.)131 196 y(4.)22 b(If)15 b Fs(v)g FD(is)h(a)f(v)m(alid) i(address)e(then)29 b Fv(MPI)p 820 196 13 2 v 15 w(BOTTOM)13 b(+)h(v)h FD(is)h(a)f(v)m(alid)i(address.)166 304 y(A)22 b(correct)g(program)g(uses)g(only)h(v)m(alid)i(addresses)d(to)g(iden)o (tify)i(the)e(lo)q(cations)h(of)f(en)o(tries)h(in)75 360 y(comm)o(unication)16 b(bu\013ers.)k(F)l(urthermore,)14 b(if)h Fs(u)h FD(and)f Fs(v)g FD(are)g(t)o(w)o(o)f(v)m(alid)j (addresses,)e(then)g(the)g(\(in)o(teger\))75 417 y(di\013erence)g Fs(u)f(-)g(v)g FD(can)g(b)q(e)g(computed)g(only)h(if)f(b)q(oth)g Fs(u)g FD(and)g Fs(v)g FD(are)f(in)i(the)f(same)f(sequen)o(tial)i (storage;)e(no)75 473 y(other)i(arithmetic)h(op)q(erations)f(can)g(b)q (e)h(meaningfully)h(executed)g(on)e(addresses.)166 530 y(The)22 b(ab)q(o)o(v)o(e)f(rules)i(imp)q(ose)f(no)g(constrain)o(ts)f (in)i(the)f(use)g(of)f(deriv)o(ed)i(datat)o(yp)q(es,)f(as)f(long)h(as) 95 b Fm(CHANGE)75 586 y FD(they)18 b(are)f(used)h(to)f(de\014ne)h(a)g (comm)o(unication)g(bu\013er)f(that)g(is)h(wholly)h(con)o(tained)f (within)h(the)e(same)75 643 y(sequen)o(tial)23 b(storage.)38 b(Ho)o(w)o(ev)o(er,)22 b(the)g(construction)g(of)f(a)h(comm)o (unication)g(bu\013er)g(that)f(con)o(tains)75 699 y(v)m(ariables)f (that)f(are)g(not)g(within)h(the)f(same)g(sequen)o(tial)h(storage)e(m)o (ust)h(ob)q(ey)g(certain)h(restrictions.)75 756 y(Basically)l(,)g(a)e (comm)o(unication)h(bu\013er)f(with)g(v)m(ariables)i(that)d(are)h(not)g (within)h(the)f(same)g(sequen)o(tial)75 812 y(storage)g(can)h(b)q(e)h (used)g(only)g(b)o(y)f(sp)q(ecifying)i(in)f(the)g(comm)o(unication)g (call)g Fs(buf)g(=)g(MPI)p 1651 812 14 2 v 16 w(BOTTOM)p FD(,)75 869 y Fs(count)i(=)g(1)p FD(,)g(and)f(using)h(a)f Fs(datat)o(yp)q(e)i FD(argumen)o(t)d(where)i(all)g(displacemen)o(ts)h (are)d(v)m(alid)j(\(absolute\))75 925 y(addresses.)d(A)15 b(datat)o(yp)q(e)f(with)h(displacemen)o(ts)h(that)e(are)h(absolute)g (addresses)g(not)f(within)i(the)f(same)75 982 y(sequen)o(tial)h (storage)e(can)i(only)f(b)q(e)h(constructed)g(using)g Fs(MPI)p 1139 982 V 15 w(TYPE)p 1271 982 V 17 w(STRUCT)p FD(.)189 1089 y Fo(A)n(dvic)n(e)f(to)i(users.)39 b FD(It)16 b(is)f(not)g(exp)q(ected)i(that)d(MPI)h(implemen)o(tations)i(will)f(b)q (e)g(able)g(to)f(detect)189 1145 y(erroneous,)22 b(\\out)f(of)g(b)q (ound")h(displacemen)o(ts)h({)f(unless)g(those)f(o)o(v)o(er\015o)o(w)g (the)g(user)h(address)189 1202 y(space)15 b({)h(since)g(the)g(MPI)g (call)g(ma)o(y)f(not)g(kno)o(w)g(the)h(exten)o(t)f(of)g(the)h(arra)o (ys)e(and)i(records)f(in)i(the)189 1258 y(host)d(program.)19 b(\()p Fo(End)d(of)g(advic)n(e)g(to)h(users.)p FD(\))166 1448 y Fl(Implemen)o(tati)o(on)d(note:)41 b Ft(There)16 b(is)f(no)g(need)h(to)f(distinguish)g(\(absolute\))h(addresses)h(and)e (\(relativ)o(e\))75 1505 y(displacemen)o(ts)h(on)h(a)g(mac)o(hine)e (with)i(con)o(tiguous)f(address)i(space:)25 b Fv(MPI)p 1274 1505 13 2 v 15 w(BOTTOM)16 b Ft(is)h(zero,)h(and)e(b)q(oth)h(ad-) 75 1561 y(dresses)d(and)e(displacemen)o(ts)g(are)g(in)o(tegers.)18 b(On)13 b(mac)o(hines)e(where)i(the)f(distinction)g(is)g(required,)g (addresses)j(are)75 1618 y(recognized)g(as)f(expressions)h(that)f(in)o (v)o(olv)o(e)f Fv(MPI)p 852 1618 V 14 w(BOTTOM)p Ft(.)75 1824 y Fn(3.13.7)49 b(Examples)75 1910 y FD(The)15 b(follo)o(wing)h (examples)g(illustrate)h(the)e(use)g(of)g(deriv)o(ed)h(datat)o(yp)q (es.)75 2031 y Fs(First)f(example:)43 b FD(Send)16 b(and)g(receiv)o(e)g (a)e(section)i(of)f(a)g(3D)g(arra)o(y)l(.)218 2139 y FB(REAL)23 b(a\(100,100,100\),)f(e\(9,9,9\))218 2195 y(INTEGER)h(oneslice,)g(twoslice,)f(threeslice,)h(sizeofreal,)f (myrank,)h(ierr)218 2252 y(MPI_STATUS)g(status)75 2365 y(C)143 b(extract)23 b(the)g(section)g(a\(1:17:2,)g(3:11,)g(2:10\))75 2421 y(C)143 b(and)23 b(store)h(it)f(in)h(e\(*,*,*\).)218 2534 y(CALL)f(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))218 2647 y(CALL)j(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 74 81 74 80 bop 75 -100 a FD(74)613 b Fp(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FB(C)119 b(create)23 b(datatype)g(for)g(a)h(1D)g(section)218 102 y(CALL)f(MPI_TYPE_VECTOR\()f(9,)i(1,)f(2,)h(MPI_REAL,)f(oneslice,)f (ierr\))75 214 y(C)119 b(create)23 b(datatype)g(for)g(a)h(2D)g(section) 218 271 y(CALL)f(MPI_TYPE_HVECTOR\(9,)f(1,)h(100*sizeofreal,)f (oneslice,)h(twoslice,ierr\))75 384 y(C)119 b(create)23 b(datatype)g(for)g(the)h(entire)f(section)218 440 y(CALL)g (MPI_TYPE_HVECTOR\()f(9,)i(1,)f(100*100*sizeofreal,)e(twoslice,)i(1,) 767 497 y(threeslice,)f(ierr\))218 610 y(CALL)h(MPI_TYPE_COMMIT\()f (threeslice,)h(ierr\))218 666 y(CALL)g(MPI_SENDRECV\(a\(1,3,2\),)e(1,)j (threeslice,)e(myrank,)h(0,)h(e,)f(9*9*9,)648 723 y(MPI_REAL,)f (myrank,)h(0,)h(MPI_COMM_WORLD,)e(status,)h(ierr\))75 844 y Fs(Second)17 b(example:)42 b FD(Cop)o(y)15 b(the)g(\(strictly\))g (lo)o(w)o(er)g(triangular)g(part)g(of)g(a)g(matrix.)218 951 y FB(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 1007 y(INTEGER)47 b(disp\(100\),)22 b(blocklen\(100\),)g(ltype,)h(myrank,)g (ierr)218 1064 y(MPI_STATUS)g(status)75 1177 y(C)119 b(copy)23 b(lower)h(triangular)e(part)h(of)h(array)f(a)75 1233 y(C)119 b(onto)23 b(lower)h(triangular)e(part)h(of)h(array)f(b)218 1346 y(CALL)g(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))75 1459 y(C)119 b(compute)23 b(start)g(and)h(size)f(of)h(each)f(column)218 1515 y(DO)h(i=1,)f(100)266 1572 y(disp\(i\))g(=)h(100*\(i-1\))e(+)i(i) 266 1628 y(block\(i\))f(=)g(100-i)218 1685 y(END)h(DO)75 1798 y(C)119 b(create)23 b(datatype)g(for)g(lower)h(triangular)e(part) 218 1854 y(CALL)h(MPI_TYPE_INDEX\()f(100,)i(block,)f(disp,)g(MPI_REAL,) f(ltype,)i(ierr\))218 1967 y(CALL)f(MPI_TYPE_COMMIT\(ltype,)e(ierr\)) 218 2024 y(CALL)i(MPI_SENDRECV\()g(a,)g(1,)h(ltype,)f(myrank,)g(0,)g (b,)h(1,)552 2080 y(ltype,)f(myrank,)g(0,)h(MPI_COMM_WORLD,)e(status,)h (ierr\))75 2201 y Fs(Third)15 b(example:)43 b FD(T)l(ransp)q(ose)15 b(a)g(matrix)218 2308 y FB(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 2365 y(INTEGER)g(row,)g(xpose,)g(sizeofreal,)g(myrank,)g(ierr)218 2421 y(MPI_STATUS)g(status)75 2534 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 2647 y(CALL)g (MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 75 82 75 81 bop 75 -100 a Fp(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FD(75)218 45 y FB(CALL)23 b(MPI_TYPE_EXTENT\()f(MPI_REAL,)h (sizeofreal,)f(ierr\))75 158 y(C)119 b(create)23 b(datatype)g(for)g (one)h(row)218 214 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f (MPI_REAL,)g(row,)g(ierr\))75 327 y(C)119 b(create)23 b(datatype)g(for)g(matrix)g(in)h(row-major)f(order)218 384 y(CALL)g(MPI_TYPE_HVECTOR\()f(100,)h(1,)h(sizeofreal,)e(row,)i (xpose,)f(ierr\))218 497 y(CALL)g(MPI_TYPE_COMMIT\()f(xpose,)h(ierr\)) 75 610 y(C)119 b(send)23 b(matrix)h(in)f(row-major)g(order)g(and)g (receive)g(in)h(column)f(major)g(order)218 666 y(CALL)g(MPI_SENDRECV\() g(a,)g(1,)h(xpose,)f(myrank,)g(0,)g(b,)h(100*100,)457 723 y(MPI_REAL,)e(myrank,)h(0,)h(MPI_COMM_WORLD,)e(status,)h(ierr\))166 820 y FD(Another)15 b(approac)o(h)g(to)f(the)i(transp)q(ose)f(problem:) 218 907 y FB(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 963 y(INTEGER)47 b(disp\(2\),)23 b(blocklen\(2\),)f(type\(2\),)h(row,)g (row1,)g(sizeofreal)218 1020 y(INTEGER)47 b(myrank,)23 b(ierr)218 1076 y(MPI_STATUS)g(status)218 1189 y(CALL)g (MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))75 1302 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 1415 y(CALL)g (MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 1528 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 1584 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f(MPI_REAL,)g(row,)g (ierr\))75 1697 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row,)f (with)h(the)f(extent)g(of)h(one)f(real)g(number)218 1754 y(disp\(1\))g(=)h(0)218 1810 y(disp\(2\))f(=)h(sizeofreal)218 1867 y(type\(1\))47 b(=)24 b(row)218 1923 y(type\(2\))47 b(=)24 b(MPI_UB)218 1980 y(blocklen\(1\))46 b(=)24 b(1)218 2036 y(blocklen\(2\))46 b(=)24 b(1)218 2092 y(CALL)f(MPI_TYPE_STRUCT\() f(2,)i(blocklen,)e(disp,)i(type,)f(row1,)g(ierr\))218 2205 y(CALL)g(MPI_TYPE_COMMIT\()f(row1,)h(ierr\))75 2318 y(C)119 b(send)23 b(100)h(rows)f(and)h(receive)f(in)g(column)g(major)g (order)218 2375 y(CALL)g(MPI_SENDRECV\()g(a,)g(100,)g(row1,)h(myrank,)f (0,)g(b,)h(100*100,)457 2431 y(MPI_REAL,)e(myrank,)h(0,)h (MPI_COMM_WORLD,)e(status,)h(ierr\))75 2550 y Fs(F)o(ourth)15 b(example:)43 b FD(manipulate)17 b(an)e(arra)o(y)f(of)g(structures.)75 2647 y FB(struct)23 b(Partstruct)147 2704 y({)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 76 83 76 82 bop 75 -100 a FD(76)613 b Fp(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)147 45 y FB(int)95 b(class;)47 b(/*)23 b(particle)g(class)g(*/)147 102 y(double)g(d[6];)71 b(/*)23 b(particle)g(coordinates)f(*/)147 158 y(char)71 b(b[7];)g(/*)23 b(some)g(additional)g(information)f(*/)147 214 y(};)75 327 y(struct)h(Partstruct)94 b(particle[1000];)75 440 y(int)429 b(i,)24 b(dest,)f(rank;)75 497 y(MPI_Comm)118 b(comm;)75 666 y(/*)24 b(build)f(datatype)g(describing)f(structure)h (*/)75 779 y(MPI_Datatype)f(Particletype;)75 835 y(MPI_Datatype)g (type[3])h(=)h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 892 y(int)238 b(blocklen[3])23 b(=)g({1,)h(6,)f(7};)75 948 y(MPI_Aint)118 b(disp[3];)75 1118 y(/*)24 b(compute)e (displacements)h(of)g(structure)g(components)f(*/)75 1231 y(MPI_Address\()g(particle,)h(disp\);)75 1287 y(MPI_Address\()f (particle[0].d,)g(disp+1\);)75 1344 y(MPI_Address\()g(particle[0].b,)g (disp+2\);)75 1400 y(for)h(\(i=0;)h(i)f(<3;)h(i++\))f(disp[i])g(-=)h (disp[0];)75 1513 y(MPI_Type_struct\()e(3,)h(blocklen,)g(disp,)g(type,) g(&Particletype\);)147 1626 y(/*)g(If)h(compiler)f(does)g(padding)g(in) g(mysterious)g(ways,)147 1682 y(the)g(following)g(may)g(be)h(safer)f (*/)75 1795 y(MPI_Datatype)f(type1[4])h(=)h({MPI_INT,)e(MPI_DOUBLE,)h (MPI_CHAR,)f(MPI_UB};)75 1852 y(int)238 b(blocklen1[4])22 b(=)i({1,)g(6,)f(7,)h(1};)75 1908 y(MPI_Aint)118 b(disp1[4];)75 2021 y(/*)24 b(compute)e(displacements)h(of)g(structure)g(components)f (*/)75 2134 y(MPI_Address\()g(particle,)h(disp1\);)75 2190 y(MPI_Address\()f(particle[0].d,)g(disp1+1\);)75 2247 y(MPI_Address\()g(particle[0].b,)g(disp1+2\);)75 2303 y(MPI_Address\()g(particle+1,)h(disp1+3\);)75 2360 y(for)g(\(i=0;)h(i)f(<4;)h(i++\))f(disp1[i])g(-=)g(disp1[0];)75 2473 y(/*)h(build)f(datatype)g(describing)f(structure)h(*/)75 2586 y(MPI_Type_struct\()f(4,)h(blocklen1,)g(disp1,)g(type1,)g (&Particletype\);)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 77 84 77 83 bop 75 -100 a Fp(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FD(77)409 45 y FB(/*)24 b(4.1:)266 102 y(send)f(the)h(entire)f (array)g(*/)75 214 y(MPI_Type_commit\()f(&Particletype\);)75 271 y(MPI_Send\()h(particle,)f(1000,)h(Particletype,)g(dest,)g(tag,)g (comm\);)409 440 y(/*)h(4.2:)266 497 y(send)f(only)h(the)f(entries)g (of)g(class)h(zero)f(particles,)266 553 y(preceded)g(by)g(the)h(number) f(of)g(such)h(entries)f(*/)75 666 y(MPI_Datatype)f(Zparticles;)70 b(/*)24 b(datatype)f(describing)f(all)i(particles)791 723 y(with)f(class)g(zero)h(\(needs)f(to)g(be)h(recomputed)791 779 y(if)f(classes)g(change\))g(*/)75 835 y(MPI_Datatype)f(Ztype;)75 948 y(MPI_Aint)118 b(zdisp[1000];)75 1005 y(int)23 b(zblock[1000],)g (j,)g(k;)75 1061 y(int)g(zzblock[2])g(=)h({1,1};)75 1118 y(MPI_Aint)118 b(zzdisp[2];)75 1174 y(MPI_Datatype)22 b(zztype[2];)75 1287 y(/*)i(compute)e(displacements)h(of)g(class)g (zero)h(particles)e(*/)75 1344 y(j)i(=)f(0;)75 1400 y(for\(i=0;)g(i)g (<)h(1000;)f(i++\))123 1456 y(if)g(\(particle[i].class==0\))194 1513 y({)194 1569 y(zdisp[j])g(=)h(i;)194 1626 y(zblock[j])f(=)h(1;)194 1682 y(j++;)194 1739 y(})75 1852 y(/*)g(create)f(datatype)f(for)i (class)f(zero)g(particles)47 b(*/)75 1908 y(MPI_Type_indexed\()22 b(j,)h(zblock,)g(zdisp,)g(Particletype,)f(&Zparticles\);)75 2021 y(/*)i(prepend)e(particle)h(count)g(*/)75 2077 y(MPI_Address\(&j,) f(zzdisp\);)75 2134 y(MPI_Address\(particle,)f(zzdisp+1\);)75 2190 y(zztype[0])i(=)g(MPI_INT;)75 2247 y(zztype[1])g(=)g(Zparticles;) 75 2303 y(MPI_Type_struct\(2,)e(zzblock,)i(zzdisp,)g(zztype,)g (&Ztype\);)75 2416 y(MPI_Type_commit\()f(&Ztype\);)75 2473 y(MPI_Send\()h(MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g(comm\);) 242 2642 y(/*)h(A)f(probably)g(more)g(efficient)g(way)h(of)f(defining)g (Zparticles)f(*/)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 78 85 78 84 bop 75 -100 a FD(78)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FB(/*)24 b(consecutive)e(particles)h(with)g(index)g(zero)g(are)h(handled)f(as)g (one)h(block)f(*/)75 102 y(j=0;)75 158 y(for)g(\(i=0;)h(i)f(<)h(1000;)f (i++\))123 214 y(if)g(\(particle[i].index==0\))170 271 y({)170 327 y(for)h(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d (==)i(0\))h(;)f(k++\);)170 384 y(zdisp[j])g(=)h(i;)170 440 y(zblock[j])f(=)h(k-i;)170 497 y(j++;)170 553 y(i)g(=)g(k;)170 610 y(})75 666 y(MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g (Particletype,)f(&Zparticles\);)457 835 y(/*)h(4.3:)314 892 y(send)g(the)g(first)h(two)f(coordinates)f(of)i(all)f(entries)g(*/) 75 1005 y(MPI_Datatype)f(Allpairs;)142 b(/*)24 b(datatype)f(for)g(all)g (pairs)h(of)f(coordinates)g(*/)75 1118 y(MPI_Aint)g(sizeofentry;)75 1231 y(MPI_Type_extent\()f(Particletype,)g(sizeofentry\);)194 1344 y(/*)i(sizeofentry)e(can)i(also)f(be)h(computed)e(by)i (subtracting)e(the)i(address)266 1400 y(of)f(particle[0])g(from)g(the)h (address)f(of)g(particle[1])f(*/)75 1513 y(MPI_Type_hvector\()g(1000,)h (2,)g(sizeofentry,)g(MPI_REAL,)f(&Allpairs\);)75 1569 y(MPI_Type_commit\()g(&Allpairs\);)75 1626 y(MPI_Send\()h(particle.d,)f (1,)i(Allpairs,)e(dest,)h(tag,)h(comm\);)218 1739 y(/*)g(an)f (alternative)g(solution)f(to)i(4.3)g(*/)75 1852 y(MPI_Datatype)e (Onepair;)71 b(/*)23 b(datatype)g(for)h(one)f(pair)g(of)h(coordinates,) e(with)695 1908 y(the)i(extent)f(of)h(one)f(particle)g(entry)g(*/)75 1965 y(MPI_Aint)g(disp2[3];)75 2021 y(MPI_Datatype)f(type2[3])h(=)h ({MPI_LB,)e(MPI_DOUBLE,)h(MPI_UB};)75 2077 y(int)g(blocklen2[3])g(=)g ({1,)h(2,)f(1};)75 2190 y(MPI_Address\()f(particle,)h(disp2\);)75 2247 y(MPI_Address\()f(particle[0].d,)g(disp2+1\);)75 2303 y(MPI_Address\()g(particle+1,)h(disp2+2\);)75 2360 y(for)g(\(i=0;)h(i<2;)f(i++\))g(disp2[i])g(-=)h(disp2[0];)75 2473 y(MPI_Type_struct\()e(3,)h(blocklen2,)g(disp2,)g(type2,)g (&Onepair\);)75 2529 y(MPI_Type_commit\()f(&Onepair\);)75 2586 y(MPI_Send\()h(particle[0].d,)f(1000,)h(Onepair,)g(dest,)g(tag,)g (comm\);)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 79 86 79 85 bop 75 -100 a Fp(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FD(79)75 45 y Fs(Fifth)18 b(example:)43 b FD(same)17 b(arra)o(y)g(manipulations)i(as)f(in)g(the)g(previous)g(example,)h(but) f(using)h(absolute)75 102 y(addresses)c(in)h(datat)o(yp)q(es.)75 220 y FB(struct)23 b(Partstruct)147 276 y({)147 332 y(int)g(class;)147 389 y(double)g(d[6];)147 445 y(char)g(b[7];)147 502 y(};)75 615 y(struct)g(Partstruct)g(particle[1000];)337 728 y(/*)h(build)f (datatype)g(describing)g(first)g(array)g(entry)g(*/)75 841 y(MPI_Datatype)f(Particletype;)75 897 y(MPI_Datatype)g(type[3])h(=) h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 953 y(int)238 b(block[3])23 b(=)h({1,)f(6,)h(7};)75 1010 y(MPI_Aint)118 b(disp[3];)75 1123 y(MPI_Address\()22 b(particle,)h(disp\);)75 1179 y(MPI_Address\()f(particle[0].d,)g(disp+1\);)75 1236 y(MPI_Address\()g(particle[0].b,)g(disp+2\);)75 1292 y(MPI_Type_struct\()g(3,)h(block,)g(disp,)g(type,)h (&Particletype\);)75 1405 y(/*)g(Particletype)e(describes)g(first)i (array)f(entry)g(--)h(using)f(absolute)147 1462 y(addresses)f(*/)505 1574 y(/*)h(5.1:)361 1631 y(send)h(the)f(entire)g(array)g(*/)75 1744 y(MPI_Type_commit\()f(&Particletype\);)75 1800 y(MPI_Send\()h (MPI_BOTTOM,)f(1000,)h(Particletype,)f(dest,)h(tag,)h(comm\);)481 1970 y(/*)f(5.2:)290 2026 y(send)g(the)h(entries)e(of)i(class)f(zero,) 290 2083 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)e(*/)75 2195 y(MPI_Datatype)g(Zparticles,)h(Ztype;)75 2308 y(MPI_Aint)g (zdisp[1000])75 2365 y(int)g(zblock[1000],)g(i,)g(j,)h(k;)75 2421 y(int)f(zzblock[2])g(=)h({1,1};)75 2478 y(MPI_Datatype)e (zztype[2];)75 2534 y(MPI_Aint)118 b(zzdisp[2];)75 2647 y(j=0;)75 2704 y(for)23 b(\(i=0;)h(i)f(<)h(1000;)f(i++\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 80 87 80 86 bop 75 -100 a FD(80)613 b Fp(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)123 45 y FB(if)23 b(\(particle[i].index==0\))170 102 y({)170 158 y(for)h(\(k=i+1;)f(\(k)g (<)h(1000\)&&\(particle[k].index)d(=)i(0\))h(;)g(k++\);)170 214 y(zdisp[j])f(=)h(i;)170 271 y(zblock[j])f(=)h(k-i;)170 327 y(j++;)170 384 y(i)g(=)g(k;)170 440 y(})75 497 y (MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g(Particletype,)f (&Zparticles\);)75 553 y(/*)i(Zparticles)e(describe)h(particles)f(with) i(class)f(zero,)g(using)147 610 y(their)g(absolute)g(addresses*/)75 723 y(/*)h(prepend)e(particle)h(count)g(*/)75 779 y(MPI_Address\(&j,)f (zzdisp\);)75 835 y(zzdisp[1])h(=)g(MPI_BOTTOM;)75 892 y(zztype[0])g(=)g(MPI_INT;)75 948 y(zztype[1])g(=)g(Zparticles;)75 1005 y(MPI_Type_struct\(2,)e(zzblock,)i(zzdisp,)g(zztype,)g(&Ztype\);) 75 1118 y(MPI_Type_commit\()f(&Ztype\);)75 1174 y(MPI_Send\()h (MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g(comm\);)75 1353 y Fs(Sixth)16 b(example:)43 b FD(handling)17 b(of)e(unions.)75 1462 y FB(union)23 b({)147 1518 y(int)118 b(ival;)147 1574 y(float)70 b(fval;)218 1631 y(})24 b(u[1000])75 1744 y(int)119 b(utype;)75 1857 y(/*)24 b(All)f(entries)g(of)h(u)f (have)h(identical)e(type;)h(variable)147 1913 y(utype)g(keeps)g(track)g (of)h(their)f(current)g(type)g(*/)75 2026 y(MPI_Datatype)70 b(type[2];)75 2083 y(int)286 b(blocklen[2])22 b(=)i({1,1};)75 2139 y(MPI_Aint)166 b(disp[2];)75 2195 y(MPI_Datatype)70 b(mpi_utype[2];)75 2252 y(MPI_Aint)166 b(i,j;)75 2365 y(/*)24 b(compute)e(an)i(MPI)g(datatype)e(for)i(each)f(possible)g (union)g(type;)147 2421 y(assume)g(values)g(are)g(left-aligned)f(in)i (union)f(storage.)g(*/)75 2534 y(MPI_Address\()f(u,)i(&i\);)75 2591 y(MPI_Address\()e(u+1,)i(&j\);)75 2647 y(disp[0])f(=)h(0;)f (disp[1])g(=)h(j-i)75 2704 y(type[1])f(=)h(MPI_UB;)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 81 88 81 87 bop 75 -100 a Fp(3.14.)34 b(NEW)15 b(PR)o(OPOSAL)h(F)o(OR)g(P)l (A)o(CK)f(AND)g(UNP)l(A)o(CK)662 b FD(81)75 102 y FB(type[0])23 b(=)h(MPI_INT;)75 158 y(MPI_Type_struct\(2,)d(blocklen,)i(disp,)g (type,)g(&mpi_utype[0]\);)75 271 y(type[0])g(=)h(MPI_FLOAT;)75 327 y(MPI_Type_struct\(2,)d(blocklen,)i(disp,)g(type,)g (&mpi_utype[1]\);)75 440 y(for\(i=0;)g(i<2;)g(i++\))g (MPI_Type_commit\(&mpi_utype[i])o(\);)75 553 y(/*)h(actual)f (communication)f(*/)75 666 y(MPI_Send\(u,)g(1000,)i(mpi_utype[utype],)d (dest,)i(tag,)h(comm\);)75 816 y Fq(3.14)59 b(New)20 b(p)n(rop)r(osal)g(fo)n(r)g(pack)g(and)f(unpack)1970 874 y Fm(CHANGE)189 920 y Fo(R)n(ationale.)37 b FD(Man)o(y)12 b(existing)h(comm)o(unication)f(libraries)i(pro)o(vide)e(pac)o(k/unpac) o(k)g(functions)h(for)189 976 y(sending)19 b(noncon)o(tiguous)f(data:) 24 b(the)18 b(user)g(explicitly)i(pac)o(ks)e(data)f(in)o(to)h(a)f(con)o (tiguous)h(bu\013er)189 1033 y(b)q(efore)c(sending)h(it,)f(and)g(unpac) o(ks)g(it)g(from)f(a)h(con)o(tiguous)g(bu\013er)f(after)g(receiv)o (eing)j(it.)j(Deriv)o(ed)189 1089 y(datat)o(yp)q(es,)g(whic)o(h)h(are)e (describ)q(ed)j(in)f(Section)g(3.13)e(b)q(elo)o(w,)i(allo)o(w)g(in)g (most)e(cases)h(to)f(a)o(v)o(oid)189 1146 y(explicit)d(pac)o(king)f (and)f(unpac)o(king)h(in)g(most)e(cases:)19 b(the)13 b(user)h(sp)q(eci\014es)h(the)e(la)o(y)o(out)g(of)f(the)i(data)189 1202 y(to)f(b)q(e)h(send)g(or)f(receiv)o(ed,)i(and)f(the)f(comm)o (unication)i(library)f(directly)h(accesses)f(a)f(noncon)o(tigu-)189 1259 y(ous)18 b(bu\013er.)30 b(The)19 b(pac)o(k/unpac)o(k)f(routines)h (are)f(pro)o(vided)h(for)f(compatibilit)o(y)i(with)f(previous)189 1315 y(libraries.)h(Also,)13 b(they)f(pro)o(vide)h(some)e(functionalit) o(y)j(that)d(is)i(not)e(otherwise)i(a)o(v)m(ailable)g(in)g(MPI:)189 1371 y(a)f(message)g(can)g(b)q(e)i(receiv)o(ed)f(in)h(sev)o(eral)e (parts,)g(where)h(the)f(receiv)o(e)i(op)q(eration)f(done)f(on)h(a)f (later)189 1428 y(part)i(ma)o(y)g(dep)q(end)i(on)f(the)g(con)o(ten)o(t) f(of)h(a)f(former)g(part;)g(and)h(outgoing)g(messages)f(ma)o(y)g(b)q(e) i(ex-)189 1484 y(plicitly)22 b(bu\013ered)e(in)h(user)f(supplied)i (space,)f(th)o(us)e(o)o(v)o(eriding)i(the)f(system)f(bu\013ering)h(p)q (olicy)l(.)189 1541 y(Finally)l(,)f(the)e(a)o(v)m(ailabilit)o(y)i(of)e (pac)o(k)f(and)i(unpac)o(k)f(op)q(erations)g(facilitate)h(the)g(dev)o (elopmen)o(t)f(of)189 1597 y(additional)f(comm)o(unication)g(libraries) h(la)o(y)o(ered)e(on)h(top)e(of)h(MPI.)g(\()p Fo(End)g(of)i(r)n (ationale.)p FD(\))75 1757 y Fs(MPI)p 160 1757 14 2 v 16 w(P)l(A)o(CK\(inbuf,)e(incount,)i(int)o(yp)q(e,)f(outbuf,)g (outsize,)g(p)q(osition,)g(dest,)g(comm)m(\))117 1835 y Ft(IN)155 b Fs(inbuf)493 b Ft(input)14 b(bu\013er)h(start)f(\(c)o (hoice\))117 1913 y(IN)155 b Fs(incount)448 b Ft(n)o(um)o(b)q(er)13 b(of)h(input)f(data)h(items)f(\(in)o(teger\))117 1990 y(IN)155 b Fs(int)o(yp)q(e)473 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(input)g(data)f(item)g(\(handle\))117 2067 y(OUT)108 b Fs(outbuf)465 b Ft(output)14 b(bu\013er)h(start)g (\(arra)o(y)e(of)h(c)o(haracters\))117 2145 y(IN)155 b Fs(outsize)457 b Ft(output)14 b(bu\013er)h(size,)f(in)g(c)o(haracter) h(units)f(\(in)o(teger\))117 2222 y(INOUT)62 b Fs(outcount)420 b Ft(curren)o(t)16 b(p)q(osition)d(in)g(bu\013er)i(\(in)o(teger\))117 2300 y(IN)155 b Fs(dest)511 b Ft(destination)14 b(of)f(pac)o(k)o(ed)h (message)g(\(in)o(teger\))117 2377 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(for)j(pac)o(k)o(ed)g(message)g(\(handle\))75 2503 y FB(int)23 b(MPI)p 245 2503 15 2 v 17 w(Pack\(void*)g(inbuf,)g (int)g(incount,)g(MPI)p 1074 2503 V 17 w(Datatype)g(intype,)f(char)i (*outbuf,)393 2560 y(int)g(outsize,)e(int)i(*position,)e(int)i(dest,)f (MPI)p 1375 2560 V 17 w(Comm)g(comm\))75 2647 y(MPI)p 150 2647 V 17 w(PACK\(INBUF,)f(INCOUNT,)h(INTYPE,)g(OUTBUF,)g(OUTSIZE,) f(POSITION,)h(DEST,)g(COMM,)393 2704 y(IERROR\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 82 89 82 88 bop 75 -100 a FD(82)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)170 45 y FB()23 b(INBUF\(*\))170 102 y(CHARACTER*1)g(OUTBUF\(*\))170 158 y(INTEGER)g(INCOUNT,)g(INTYPE,)g(OUTSIZE,)g(POSITION,)f(DEST,)i (COMM,)f(IERROR)166 244 y FD(P)o(ac)o(ks)15 b(the)h(message)f(in)i(the) f(send)g(bu\013er)g(sp)q(eci\014ed)i(b)o(y)d Fs(inbuf,)i(incount,)g (int)o(yp)q(e)g FD(in)o(to)f(the)g(bu\013er)75 301 y(space)i(sp)q (eci\014ed)h(b)o(y)f Fs(outbuf)h FD(and)e Fs(outsize)p FD(.)28 b(The)17 b(input)i(bu\013er)e(can)h(b)q(e)g(an)o(y)f(comm)o (unication)h(bu\013er)75 357 y(allo)o(w)o(ed)f(in)g Fs(MPI)p 377 357 14 2 v 15 w(SEND)p FD(.)g(The)f(output)g(bu\013er)g Fs(outbuf)i FD(is)e(an)g(arra)o(y)f(of)h(c)o(haracters)g(of)f(length)i Fs(outsize)75 414 y FD(\(declared)j(as)e Fs(cha)o(r)g(buf[outsize])j FD(in)e(C)g(and)f Fs(CHARA)o(CTER*1)i(BUF\(OUTSIZE\))f FD(in)g(F)l(ortran\).)29 b(The)75 470 y(input)22 b(v)m(alue)g(of)e Fs(p)q(osition)i FD(is)f(the)g(\014rst)g(lo)q(cation)g(in)h(the)f (output)f(bu\013er)h(to)f(b)q(e)h(used)h(for)e(pac)o(king.)75 527 y Fs(p)q(osition)d FD(is)f(incremen)o(ted)g(b)o(y)g(the)f(size)i (of)e(the)g(pac)o(k)o(ed)h(message,)f(and)g(the)h(output)f(v)m(alue)i (of)e Fs(p)q(osition)75 583 y FD(is)21 b(the)f(\014rst)g(lo)q(cation)i (in)f(the)f(output)h(bu\013er)f(follo)o(wing)h(the)g(lo)q(cations)g(o)q (ccupied)h(b)o(y)e(the)h(pac)o(k)o(ed)75 640 y(message.)k(The)17 b Fs(dest)i FD(and)e Fs(comm)11 b FD(argumen)o(ts)17 b(iden)o(ti\014es)h(the)g(in)o(tended)g(destination)g(of)f(the)g(pac)o (k)o(ed)75 696 y(message)e(\(see)g(b)q(elo)o(w\).)75 800 y Fs(MPI)p 160 800 V 16 w(UNP)l(A)o(CK\(inbuf,)h(insize,)f(p)q (osition,)h(outbuf,)g(outcount,)h(outt)o(yp)q(e,)g(source,)e(comm)m(\)) 117 877 y Ft(IN)155 b Fs(inbuf)493 b Ft(input)14 b(bu\013er)h(start)f (\(arra)o(y)g(of)f(c)o(haracters\))117 952 y(IN)155 b Fs(insize)485 b Ft(size)15 b(of)e(input)h(bu\013er,)g(in)g(c)o (haracter)h(units)f(\(in)o(teger\))117 1026 y(INOUT)62 b Fs(p)q(osition)439 b Ft(curren)o(t)16 b(p)q(osition)d(in)g(bu\013er)i (\(in)o(teger\))117 1101 y(OUT)108 b Fs(outbuf)465 b Ft(output)14 b(bu\013er)h(start)g(\(c)o(hoice\))117 1176 y(IN)155 b Fs(outcount)420 b Ft(n)o(um)o(b)q(er)13 b(of)h(items)f(to)g (b)q(e)i(unpac)o(k)o(ed)f(\(in)o(teger\))117 1250 y(IN)155 b Fs(outt)o(yp)q(e)445 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(output)g (data)g(item)e(\(handle\))117 1325 y(IN)155 b Fs(source)468 b Ft(source)15 b(of)f(pac)o(k)o(ed)g(message)f(\(in)o(teger\))117 1400 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(for)j(pac)o(k)o (ed)g(message)g(\(handle\))75 1524 y FB(int)23 b(MPI)p 245 1524 15 2 v 17 w(Unpack\(char*)f(inbuf,)h(int)h(insize,)f(int)g (*position,)g(void)g(*outbuf,)393 1581 y(int)h(outcount,)e(MPI)p 802 1581 V 17 w(Datatype)h(outtype,)g(int)g(source,)g(MPI)p 1607 1581 V 17 w(Comm)g(comm\))75 1667 y(MPI)p 150 1667 V 17 w(UNPACK\(INBUF,)f(INSIZE,)h(POSITION,)f(OUTBUF,)h(OUTCOUNT,)g (OUTTYPE,)g(SOURCE,)393 1724 y(COMM,)g(IERROR\))170 1780 y(CHARACTER*1)g(INBUF\(*\))170 1837 y()g(OUTBUF\(*\))170 1893 y(INTEGER)g(INSIZE,)g(POSITION,)g(OUTCOUNT,)g(OUTTYPE,)f(SOURCE,)h (COMM,)g(IERROR)166 1979 y FD(Unpac)o(ks)17 b(a)f(message)g(in)o(to)h (the)f(receiv)o(e)i(bu\013er)f(sp)q(eci\014ed)h(b)o(y)f Fs(outbuf,)h(outcount,)g(outt)o(yp)q(e)h FD(from)75 2036 y(the)14 b(bu\013er)g(space)g(sp)q(eci\014ed)i(b)o(y)e Fs(inbuf)i FD(and)e Fs(insize)p FD(.)20 b(The)14 b(output)g(bu\013er)g (can)g(b)q(e)h(an)o(y)e(comm)o(unication)75 2092 y(bu\013er)21 b(allo)o(w)o(ed)h(in)g Fs(MPI)p 523 2092 14 2 v 16 w(RECV)p FD(.)g(The)g(input)g(bu\013er)g Fs(inbuf)g FD(is)g(an)f(arra)o(y)g(of)g (c)o(haracters)f(of)h(length)75 2149 y Fs(outsize)p FD(.)f(\(declared) 14 b(as)f Fs(cha)o(r)g(buf[outsize])i FD(in)f(C)f(and)g Fs(CHARA)o(CTER*1)h(BUF\(OUTSIZE\))g FD(in)g(F)l(ortran\).)75 2205 y(The)h(input)g(v)m(alue)g(of)f Fs(p)q(osition)i FD(is)f(the)f(\014rst)g(lo)q(cation)h(in)g(the)g(output)f(bu\013er)g(o) q(ccupied)i(b)o(y)e(the)h(pac)o(k)o(ed)75 2262 y(message.)34 b Fs(p)q(osition)21 b FD(is)g(incremen)o(ted)g(b)o(y)f(the)g(size)h(of) f(the)g(pac)o(k)o(ed)g(message,)h(so)f(that)f(the)h(output)75 2318 y(v)m(alue)g(of)e Fs(p)q(osition)i FD(is)f(the)g(\014rst)f(lo)q (cation)i(in)f(the)g(output)g(bu\013er)f(after)g(the)h(lo)q(cations)g (o)q(ccupied)i(b)o(y)75 2375 y(the)14 b(message)f(that)f(w)o(as)h (unpac)o(k)o(ed.)20 b(The)13 b Fs(source)h FD(and)g Fs(comm)8 b FD(argumen)o(ts)k(iden)o(tify)j(the)e(source)h(of)f(the)75 2431 y(pac)o(k)o(ed)i(message)g(\(see)g(b)q(elo)o(w\).)189 2534 y Fo(A)n(dvic)n(e)21 b(to)i(users.)81 b FD(Note)21 b(the)h(di\013erence)i(b)q(et)o(w)o(een)e Fs(MPI)p 1277 2534 V 16 w(RECV)g FD(and)h Fs(MPI)p 1608 2534 V 15 w(UNP)l(A)o(CK)p FD(:)f(In)189 2591 y Fs(MPI)p 274 2591 V 15 w(RECV)p FD(,)17 b(the)f Fs(count)h FD(argumen)o(t)e(sp)q(eci\014es)j(the)e (maxim)o(um)f(n)o(um)o(b)q(er)i(of)e(items)h(that)f(can)h(b)q(e)189 2647 y(receiv)o(ed.)35 b(The)21 b(actual)f(n)o(um)o(b)q(er)g(of)g (items)g(receiv)o(ed)h(is)g(determined)g(b)o(y)f(the)g(length)h(of)f (the)189 2704 y(incoming)15 b(message.)20 b(In)15 b Fs(MPI)p 716 2704 V 16 w(UNP)l(A)o(CK)p FD(,)f(the)h Fs(count)h FD(argumen)o(t)e(sp)q(eci\014es)i(the)f(actual)g(n)o(um)o(b)q(er)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 83 90 83 89 bop 75 -100 a Fp(3.14.)34 b(NEW)15 b(PR)o(OPOSAL)h(F)o(OR)g(P)l (A)o(CK)f(AND)g(UNP)l(A)o(CK)662 b FD(83)189 45 y(of)14 b(items)i(that)e(are)h(unpac)o(k)o(ed;)g(the)g(\\size")h(of)e(the)i (corresp)q(onding)g(message)e(is)i(the)f(incremen)o(t)189 102 y(in)20 b Fs(p)q(osition)p FD(.)35 b(The)20 b(reason)f(for)g(this)i (c)o(hange)e(is)i(that)e(the)h(\\incoming)g(message)g(size")g(is)g(not) 189 158 y(predetermined)15 b(since)f(the)g(user)g(decides)h(ho)o(w)e(m) o(uc)o(h)h(to)e(unpac)o(k;)j(nor)e(is)h(it)g(easy)f(to)g(determine)189 214 y(the)d(\\message)g(size")h(from)f(the)g(n)o(um)o(b)q(er)h(of)f (items)h(to)f(b)q(e)h(unpac)o(k)o(ed.)19 b(In)11 b(fact,)g(in)g(a)f (heterogenous)189 271 y(system,)k(this)i(n)o(um)o(b)q(er)f(ma)o(y)g (not)f(b)q(e)i(determined)h(apriori.)j(\()p Fo(End)c(of)g(advic)n(e)g (to)h(users.)p FD(\))166 366 y(T)l(o)f(understand)h(the)f(b)q(eha)o (vior)h(of)e(pac)o(k)h(and)h(unpac)o(k,)f(it)h(is)f(con)o(v)o(enien)o (t)h(to)f(think)h(of)e(the)i(data)75 422 y(part)11 b(of)g(a)g(message)g (as)g(b)q(eing)i(the)e(sequence)i(obtained)f(b)o(y)g(concatenating)f (the)h(successiv)o(e)h(v)m(alues)f(sen)o(t)75 478 y(in)k(that)f (message.)20 b(The)c(pac)o(k)f(op)q(eration)h(stores)f(this)g(sequence) i(in)f(the)g(bu\013er)f(space,)h(as)f(if)h(sending)75 535 y(the)d(message)g(to)g(that)f(bu\013er.)19 b(The)14 b(unpac)o(k)f(op)q(eration)h(retriev)o(es)f(this)h(sequence)g(from)f (bu\013er)g(space,)75 591 y(as)j(if)h(receiv)o(eing)i(a)d(message)g (from)g(that)g(bu\013er.)25 b(\(It)16 b(is)h(helpful)i(to)d(think)i(of) e(in)o(ternal)h(F)l(ortran)f(\014les)75 648 y(or)f Fs(sscanf)h FD(in)g(C,)f(for)f(a)h(similar)i(function.\))166 704 y(A)k(message)f(can)g(b)q(e)i(pac)o(k)o(ed,)f(next)g(sen)o(t)f(with)h (t)o(yp)q(e)g Fv(MPI)p 1228 704 13 2 v 14 w(BYTE)p FD(;)f(the)g (outcome)h(is)g(as)f(if)h(the)75 761 y(message)15 b(w)o(as)g(directly)i (sen)o(t)f(without)f(pac)o(king)h(\014rst.)21 b(Similarly)l(,)d(a)d (message)h(can)f(b)q(e)i(receiv)o(ed)g(with)75 817 y(t)o(yp)q(e)g Fv(MPI)p 256 817 V 14 w(BYTE)p FD(,)f(next)h(unpac)o(k)o(ed;)h(the)f (outcome)f(is)h(as)g(if)g(it)g(w)o(as)f(receiv)o(ed)i(directly)l(,)h (without)d(\014rst)75 874 y(unpac)o(king.)21 b(Th)o(us)75 968 y FB(position)i(=)g(0;)75 1025 y(MPI_PACK\(inbuf,)f(incount,)h (intype,)f(outbuf,)h(outsize,)g(position,)g(dest,)g(tag\);)75 1081 y(MPI_SEND\(outbuf,)f(position,)g(MPI_BYTE,)h(dest,)g(tag,)g (comm\))166 1176 y FD(is)16 b(equiv)m(alen)o(t)h(to)75 1271 y FB(MPI_SEND\(inbuf,)22 b(incount,)h(intype,)f(dest,)i(tag,)f (comm\))75 1365 y(MPI_RECV\(inbuf,)f(insize,)h(MPI_BYTE,)f(source,)h (tag,)h(comm,)f(status\);)75 1422 y(position)g(=)g(0;)75 1478 y(MPI_UNPACK\(inbuf,)f(insize,)g(position,)h(outbuf,)g(outcount,)g (outtype\))166 1573 y FD(is)16 b(equiv)m(alen)o(t)h(to)75 1667 y FB(MPI_RECV\(outbuf,)22 b(outcount,)g(outtype,)h(source,)g(tag,) g(comm,)g(status\))166 1762 y FD(assuming)13 b(that)g(the)g(incoming)i (message)d(con)o(tains)i Fs(outcount)h FD(items)e(of)g(t)o(yp)q(e)g Fs(outt)o(yp)q(e)p FD(.)21 b(\(It)13 b(is)h(the)75 1819 y(user)h(the)h(repsonsibilit)o(y)h(to)e(unpac)o(k)g(the)g(righ)o(t)g(n) o(um)o(b)q(er)h(of)f(data)f(items.\))166 1875 y(The)20 b Fs(comm)14 b FD(and)20 b Fs(dest)h FD(argumen)o(ts)e(in)h(a)g(call)h (to)e Fs(MPI)p 1148 1875 14 2 v 16 w(P)l(A)o(CK)g FD(should)i(matc)o(h) e(the)h(argumen)o(ts)75 1931 y(used)e(to)e(subsequen)o(tly)i(send)g (the)f(pac)o(k)o(ed)g(bu\013er;)h(the)f Fs(comm)11 b FD(and)17 b Fs(source)h FD(argumen)o(ts)e(in)i(a)e(call)j(to)75 1988 y Fs(MPI)p 160 1988 V 16 w(UNP)l(A)o(CK)11 b FD(should)g(matc)o(h) f(the)h Fs(comm)5 b FD(argumen)o(t)10 b(previously)i(used)f(to)f (receiv)o(e)i(the)e(pac)o(k)o(ed)h(bu\013er)75 2044 y(and)k(the)h (source)f(of)g(that)f(bu\013er.)20 b(\(If)15 b(the)g(bu\013er)h(w)o(as) e(receiv)o(ed)i(with)g(an)f(explicit)i Fs(source)f FD(argumen)o(t,)75 2101 y(then)22 b(the)g(same)f(argumen)o(t)g(should)h(b)q(e)g(pro)o (vided)h(to)e Fs(MPI)p 1159 2101 V 15 w(UNP)l(A)o(CK)p FD(;)h(if)g Fv(MPI)p 1509 2101 13 2 v 14 w(ANY)p 1608 2101 V 15 w(SOURCE)e FD(w)o(as)75 2157 y(used,)d(then)g(the)g Fs(source)g FD(rank)g(returned)g(in)h Fs(status)g FD(should)g(b)q(e)f (used.\))25 b(If)17 b(the)g(bu\013er)g(is)g(pac)o(k)o(ed)g(and)75 2214 y(unpac)o(k)o(ed)h(b)o(y)f(the)h(same)f(pro)q(cess,)h(then)g(the)f Fs(comm)12 b FD(and)18 b Fs(dest)h FD(argumen)o(ts)d(of)h(the)h Fs(MPI)p 1662 2214 14 2 v 16 w(P)l(A)o(CK)f FD(call)75 2270 y(should)f(matc)o(h)f(the)g Fs(comm)9 b FD(and)16 b Fs(source)g FD(argumen)o(ts)e(of)h(the)g Fs(MPI)p 1225 2270 V 16 w(UNP)l(A)o(CK)g FD(call.)189 2365 y Fo(R)n(ationale.)38 b FD(The)13 b(need)h(for)e(the)h Fs(comm)7 b FD(and)13 b Fs(source/dest)i FD(argumen)o(ts)d(follo)o(ws)h(from)f(the)h(desire) 189 2421 y(to)g(\(i\))i(supp)q(ort)f(comm)o(unication)h(across)f (heterogenous)h(no)q(des;)f(\(ii\))h(not)f(require)i(messages)e(to)189 2478 y(b)q(e)i(self-t)o(yp)q(ed;)g(and)g(\(iii\))h(allo)o(w)f(data)f (con)o(v)o(ersion)g(to)g(b)q(e)i(done)f(either)g(b)o(y)f(sender)i(or)e (receiv)o(er.)189 2534 y(I.e.,)e(w)o(e)h(wish)h(to)e(con)o(tin)o(ue)i (the)f(general)h(MPI)f(design)h(where)f(b)q(oth)h(sender)f(and)h (receiv)o(er)f(ha)o(v)o(e)189 2591 y(su\016cien)o(t)21 b(information)f(to)g(do)g(data)g(con)o(v)o(ersion,)h(and)g(b)q(oth)f(c) o(hoices)h(are)f(a)o(v)m(ailable)i(to)e(the)189 2647 y(implemen)o(ter,)i(ev)o(en)f(if)g(messages)f(carry)g(no)g (information,)h(in)h(addition)f(to)f(data.)34 b(\()p Fo(End)21 b(of)189 2704 y(r)n(ationale.)p FD(\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 84 91 84 90 bop 75 -100 a FD(84)613 b Fp(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fl(Discussion:)33 b Ft(Rather)13 b(than)g(requiring)f(that)h(the)h Fv(comm)7 b Ft(and)13 b Fv(dest)g Ft(argumen)o(ts)f(of)h Fv(MPI)p 1608 45 13 2 v 14 w(P)m(A)o(CK)e Ft(iden)o(tify)75 95 y(the)j(target)f(of)f(the)i(pac)o(k)o(ed)f(message,)f(one)i(could)e (merely)g(require)i(that)f(the)h(argumen)o(t)d(iden)o(ti\014es)j(a)e (target)i(\\of)75 145 y(the)h(same)e(kind".)18 b(This)c(w)o(ould)f (allo)o(w)g(to)h(pac)o(k)g(a)g(message)f(and)h(send)h(it)f(to)g(sev)o (eral)h(destinations,)f(as)g(long)f(as)75 195 y(all)g(these)i (destinations)f(are)g(\\of)f(the)i(same)e(kind".)k(T)m(o)c(supp)q(ort)i (this)f(extension,)g(it)g(w)o(ould)f(b)q(e)h(useful)g(to)g(ha)o(v)o(e) 75 244 y(an)e(inquiry)f(function)h(that)h(returns)g(the)g(\\kind")e(of) h(a)g(destination,)g(or)g(an)g(enquiry)g(function)g(that)g(c)o(hec)o (ks)i(that)75 294 y(t)o(w)o(o)f(destinations)h(are)h(\\of)e(the)h(same) f(kind".)166 351 y(A)e(further)h(extension)g(is)f(to)g(allo)o(w)e(a)i (\\generic")g(destination,)g(of)g(unkno)o(wn)f(\\kind".)17 b(This)11 b(could)f(sp)q(eci\014ed)75 408 y(b)o(y)h(a)g(constan)o(t)h Fv(dest)g Ft(=)g Fv(MPI)p 530 408 V 14 w(ANY)p 629 408 V 15 w(DEST)p Ft(.)e(A)i(message)f(pac)o(k)o(ed)g(for)g(a)g (destination)h(of)e(unkno)o(wn)h(kind,)g(m)o(ust)f(b)q(e)75 464 y(unpac)o(k)o(ed)i(with)f Fv(source)i Ft(=)f Fv(MPI)p 590 464 V 14 w(ANY)p 689 464 V 15 w(SOURCE)p Ft(,)d(a)i(generic)i (source)g(of)e(unkno)o(wn)g(kind.)17 b(The)11 b(implemen)o(tation)75 520 y(w)o(ould)h(need)h(to)g(use)g(a)f(common)e(in)o(terc)o(hange)j (format)e(\(e.g.,)g(XDR\),)h(for)g(suc)o(h)h(messages.)18 b(The)13 b(last)f(extension)75 577 y(could)18 b(b)q(e)g(in)f(addition)g (to)h(or)f(in)h(place)g(of)f(the)h(previous)g(one.)30 b(It)18 b(w)o(ould)f(b)q(e)h(used)h(in)e(the)i(case)f(where)h(the)75 633 y(destination)14 b(of)g(a)g(message)g(is)g(not)h(kno)o(wn)f(when)g (it)g(is)h(pac)o(k)o(ed,)f(or)g(where)i(the)f(same)e(pac)o(k)o(ed)i (message)f(migh)o(t)75 690 y(b)q(e)h(sen)o(t)f(to)g(sev)o(eral)g (destinations)g(of)g(di\013eren)o(t)h(\\kind".)166 829 y FD(One)23 b(can)f(pac)o(k)g(sev)o(eral)h(messages,)g(in)g (succession,)i(in)e(a)f(bu\013er,)h(then)g(send)g(the)f(resulting)75 886 y(concatenation.)d(The)12 b(outcome)f(is)h(as)g(if)g(the)g (concatenated)g(message)f(w)o(as)g(prepared)h(using)g(a)g(suitable)75 942 y(deriv)o(ed)k(datat)o(yp)q(e.)k(Similarly)l(,)d(one)e(can)g(unpac) o(k)h(a)f(bu\013er)g(in)o(to)g(sev)o(eral)h(messages.)j(Th)o(us)75 1051 y FB(...)75 1108 y(POSITION)k(=)g(0)75 1164 y(CALL)g(MPI_PACK\()g (LEN,)g(1,)h(MPI_INTEGER,)e(OUTBUF,)290 1221 y(1000,)h(POSITION,)g (DEST,)g(COMM,)g(IERR\))75 1277 y(CALL)g(MPI_PACK\()g(A,)h(LEN,)f (MPI_REAL,)f(OUTBUF,)314 1334 y(1000,)h(POSITION,)f(DEST,)i(COMM,)f (IERR\);)75 1390 y(CALL)g(MPI_SEND\()g(OUTBUF,)g(POSITION,)f(MPI_BYTE,) h(DEST,)314 1447 y(TAG,)g(COMM,)g(IERR\))166 1555 y FD(is)16 b(equiv)m(alen)o(t)h(to)75 1651 y FB(...)75 1708 y(BLEN\(1\))23 b(=)h(1)75 1764 y(BLEN\(2\))f(=)h(LEN)75 1821 y(CALL)f(MPI_ADDRESS\()g (LEN,)g(DISP\(1\)\))75 1877 y(CALL)g(MPI_ADDRESS\()g(A,)g(DISP\(2\)\)) 75 1934 y(TYPE\(1\))g(=)h(MPI_INTEGER)75 1990 y(TYPE\(2\))f(=)h (MPI_REAL)75 2047 y(CALL)f(MPI_TYPE_STRUCT\(2,)f(BLEN,)h(DISP,)g(TYPE,) g(NEWTYPE,)g(IERR\))75 2103 y(CALL)g(MPI_COMMIT\(NEWTYPE,)f(IERR\))75 2160 y(CALL)h(MPI_SEND\(MPI_BOTTOM,)e(1,)j(NEWTYPE,)f(DEST,)g(TAG,)g (COMM\))166 2255 y FD(Both)15 b(programs)f(send)i(a)f(message)f(that)h (consists)g(of)g(one)g(in)o(teger)h(\(with)f(v)m(alue)h Fs(LEN)p FD(\),)f(follo)o(w)o(ed)75 2312 y(b)o(y)g Fs(LEN)h FD(reals)f(from)g(arra)o(y)f Fs(A)p FD(.)i(The)f(second)h(program)e(p)q (ossibly)j(a)o(v)o(oids)f(an)f(additional)i(memory-to-)75 2368 y(memory)e(cop)o(y)g(op)q(eration,)g(at)f(the)i(exp)q(ense)g(of)f (additional)h(prepro)q(cessing.)166 2425 y(If)h(the)f(v)m(alue)i(of)e Fs(LEN)g FD(is)h(not)f(kno)o(wn)g(at)g(the)g(receiving)i(side)g(ahead)e (of)g(the)h(receiv)o(e)g(op)q(eration,)75 2482 y(then)f(the)f(message)g (can)g(b)q(e)h(receiv)o(ed)g(b)o(y)f(executing)i(the)e(co)q(de)h(b)q (elo)o(w)75 2591 y FB(...)75 2647 y(CALL)23 b(MPI_RECV\(INBUF,)f(1000,) h(MPI_BYTE,)g(SOURCE,)g(TAG,)g(COMM,)385 2704 y(STATUS,)g(IERR\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 85 92 85 91 bop 75 -100 a Fp(3.14.)34 b(NEW)15 b(PR)o(OPOSAL)h(F)o(OR)g(P)l (A)o(CK)f(AND)g(UNP)l(A)o(CK)662 b FD(85)75 45 y FB(CALL)23 b(MPI_UNPACK\(INBUF,)f(1000,)h(POSITION,)g(LEN,)g(1,)385 102 y(MPI_INTEGER,)f(SOURCE,)h(COMM,)h(IERR\))75 158 y(CALL)f(MPI_UNPACK\(INBUF,)f(1000,)h(POSITION,)g(A,)g(LEN,)385 214 y(MPI_REAL,)g(SOURCE,)g(COMM,)g(IERR\))189 347 y Fo(A)n(dvic)n(e)c(to)h(users.)66 b FD(The)20 b(simplest)h(w)o(a)o(y)d (of)h(receiving)j(an)d(in)o(teger)h Fs(LEN)p FD(,)f(follo)o(w)o(ed)h(b) o(y)f Fs(LEN)189 403 y FD(reals,)14 b(is)h(to)f(send)h(t)o(w)o(o)e (messages,)g(the)i(\014rst)f(with)g(the)h(v)m(alue)g Fs(LEN)p FD(,)f(the)h(second)g(with)f Fs(LEN)h FD(real)189 459 y(n)o(um)o(b)q(ers,)g(as)g(sho)o(wn)f(b)q(elo)o(w)189 623 y FB(...)189 679 y(!)23 b(send)h(side)189 736 y(CALL)f (MPI_SEND\(LEN,)f(1,)i(MPI_INTEGER,)e(DEST,)h(TAG,)g(COMM,)h(IERR\))189 792 y(CALL)f(MPI_SEND\(A,)f(LEN,)i(MPI_REAL,)e(DEST,)h(TAG,)h(COMM,)f (IERR\))189 905 y(...)189 962 y(!)g(receive)g(side)189 1018 y(CALL)g(MPI_RECV\(LEN,)f(1,)i(MPI_INTEGER,)e(SOURCE,)h(TAG,)g (COMM,)g(IERR\))189 1075 y(CALL)g(MPI_RECV\(A,)f(LEN,)i(MPI_REAL,)e (SOURCE,)h(TAG,)g(COMM,)h(IERR\))189 1238 y FD(The)c(\\pac)o(king")g (of)f(sev)o(eral)h(di\013eren)o(t)h(data)e(items)h(in)h(one)f(message,) g(using)h(either)g(explicit)189 1295 y(calls)c(to)e Fs(MPI)p 432 1295 14 2 v 16 w(P)l(A)o(CK)p FD(,)h(or)g(deriv)o(ed)h(datat)o(yp)q (es)e(is)i(only)g(to)e(impro)o(v)o(e)h(p)q(erformance,)g(in)h(systems) 189 1351 y(where)e(comm)o(unication)h(latencies)h(are)d(signi\014can)o (t.)21 b(\()p Fo(End)16 b(of)g(advic)n(e)g(to)h(users.)p FD(\))166 1566 y Fl(Discussion:)51 b Ft(Sev)o(eral)18 b(restrictions)h(w)o(ere)g(suggested)h(ab)q(out)e(sending/receiving)g (of)g(messages)g(that)75 1615 y(where)d(pac)o(k)o(ed)f(together:)166 1672 y(\(1\))j(Messages)h(are)g(alw)o(a)o(ys)e(pac)o(k)o(ed/unpac)o(k)o (ed)h(in)g(sequence,)i(and)e(the)g(starting)g(p)q(oin)o(t)g(for)g (unpac)o(king)75 1722 y(is)e(the)g(starting)f(p)q(oin)o(t)h(for)f(pac)o (king.)20 b(E.g.,)13 b(one)i(cannot)g(pac)o(k)f(separately)i(t)o(w)o(o) e(messages,)g(then)i(concatenate)75 1771 y(them)c(and)g(send)h(the)g (result.)18 b(Or,)13 b(one)f(cannot)h(pac)o(k)f(\014v)o(e)h(messages,)f (and)g(then)h(send)g(a)g(message)f(that)g(consists)75 1821 y(of)h(the)i(last)f(three)h(ones)g(pac)o(k)o(ed.)k(With)13 b(this)h(rule,)g(the)h(only)e(constrain)o(t)i(on)f(message)f(represen)o (tation)j(is)e(that)75 1871 y(it)f(is)g(\\monotone":)j(the)e(represen)o (tation)g(for)f(a)g(pre\014x)h(of)f(the)h(full)e(message)h(is)g(a)g (pre\014x)h(of)f(the)g(represen)o(tation)75 1921 y(for)h(the)g(full)f (message.)19 b(T)m(o)13 b(pic)o(k)h(a)g(fancy)g(example:)j(supp)q(ose)e (that)f(one)h(w)o(an)o(ts)f(to)g(compress)g(messages)g(using)75 1971 y(a)h(Lemp)q(el-Ziv)f(co)q(de)j(b)q(efore)f(they)g(are)g(sen)o(t,) g(and)f(vice-v)o(ersa)i(on)e(the)h(receiv)o(e)h(side.)23 b(This)15 b(restriction)i(w)o(ould)75 2020 y(allo)o(w)12 b(to)i(do)g(so;)g(the)h(pac)o(k/unpac)o(k)f(routines)g(w)o(ould)g(ha)o (v)o(e)g(to)g(store)h(in)o(ternally)e(information)e(on)j(the)g(curren)o (t)75 2070 y(state)g(of)g(the)g(Ziv-Lemp)q(el)e(algorithm.)k(A)d(less)i (fancy)e(example:)k(one)d(migh)o(t)d(w)o(an)o(t)j(to)f(store)i(a)e (general)h(header)75 2120 y(at)g(the)g(start)h(of)e(the)i(message\))166 2176 y(The)g(formal)e(constrain)o(t)i(is)g(that,)g(on)g(the)h(send/pac) o(k)f(side,)h(the)f(initial)e(v)n(alue)i(of)f Fv(p)q(osition)i Ft(matc)o(hes)e(the)75 2226 y(initial)c(bu\013er)k(address)f(used)h(in) d(the)i(send)h(op)q(eration,)e(and)g(that)g Fv(p)q(osition)h Ft(is)f(only)g(mo)q(di\014ed)f(b)o(y)h(the)h(successiv)o(e)75 2276 y(calls)g(to)f(pac)o(k.)18 b(I)13 b(sligh)o(t)f(extension)i(to)f (this,)f(whic)o(h)h(is)g(harmless)f(and)h(simpli\014es)e(implemen)o (tation,)f(is)i(that)h(the)75 2326 y(initial)f(v)n(alue)h(of)g(p)q (osition)h(is)f(zero)i(\(pac)o(king)e(starts)i(at)f(bu\013er)h (start\).)166 2389 y(\(2\))e(Messages)h(are)f(unpac)o(k)o(ed)h(in)e (the)i(same)e(\\c)o(h)o(unks")h(that)g(they)g(where)h(pac)o(k)o(ed.)k (This)13 b(w)o(ould)f(preserv)o(e)75 2445 y(ev)o(en)i(more)e(freedom)h (in)g(the)h(in)o(ternal)f(represen)o(tation)i(of)e(messages,)g(as)g(co) q(de)h(do)q(es)h(not)e(ha)o(v)o(e)g(an)o(ymore)f(to)h(b)q(e)75 2502 y(self-delimiting.)i(But)f(this)g(restriction)h(is)f(more)f (signi\014can)o(t)g(from)f(the)j(user)g(viewp)q(oin)o(t.)166 2647 y FD(The)e(follo)o(wing)g(calls)g(allo)o(w)g(the)f(user)h(to)f (\014nd)h(out)f(ho)o(w)g(m)o(uc)o(h)g(space)h(is)f(needed)i(to)e(pac)o (k/unpac)o(k)75 2704 y(messages)j(and,)g(th)o(us,)f(manage)h(space)g (allo)q(cation)i(for)d(bu\013ers.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 86 93 86 92 bop 75 -100 a FD(86)608 b Fp(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(P)l(A)o(CK)p 291 45 V 16 w(SIZE\(incount,)g(int)o (yp)q(e,)g(dest,)g(comm)n(,)11 b(size\))117 122 y Ft(IN)155 b Fs(incount)448 b Ft(n)o(um)o(b)q(er)13 b(of)h(input)f(data)h(items)f (\(in)o(teger\))117 197 y(IN)155 b Fs(int)o(yp)q(e)473 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(input)g(data)f(item)g (\(handle\))117 273 y(IN)155 b Fs(dest)511 b Ft(destination)14 b(of)f(pac)o(k)o(ed)h(message)g(\(in)o(teger\))117 348 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(for)j(pac)o(k)o(ed)g (message)g(\(handle\))117 423 y(OUT)108 b Fs(size)519 b Ft(size)15 b(of)e(pac)o(k)o(ed)h(message)g(\(in)o(teger\))75 547 y FB(int)23 b(MPI)p 245 547 15 2 v 17 w(Pack)p 358 547 V 17 w(size\(int)g(incount,)f(MPI)p 876 547 V 17 w(Datatype)h(intype,)g(int)g(dest,)g(MPI)p 1609 547 V 17 w(Comm)h(comm,)393 604 y(int*)f(size\))75 690 y(MPI)p 150 690 V 17 w(PACK)p 263 690 V 16 w(SIZE\(INCOUNT,)f(INTYPE,)h(DEST,)h (COMM,)f(SIZE,)g(IERROR\))170 747 y(INTEGER)g(INCOUNT,)g(INTYPE,)g (DEST,)g(COMM,)g(SIZE,)h(IERROR)166 833 y FD(A)11 b(call)h(to)e Fs(MPI)p 426 833 14 2 v 16 w(P)l(A)o(CK)p 557 833 V 16 w(SIZE\(incount,)j(int)o(yp)q(e,)g(dest,)g(comm)m(,)8 b(size\))j FD(returns)g(in)h Fs(size)f FD(the)g(incremen)o(t)75 890 y(in)21 b Fs(p)q(osition)h FD(that)e(is)h(e\013ected)f(b)o(y)h(a)f (call)h(to)f Fs(MPI)p 974 890 V 16 w(P)l(A)o(CK\(inbuf,)i(incount,)h (int)o(yp)q(e,)g(outbuf,)g(outsize,)75 946 y(p)q(osition,)16 b(dest,)g(tag\))p FD(;)75 1050 y Fs(MPI)p 160 1050 V 16 w(UNP)l(A)o(CK)p 354 1050 V 17 w(SIZE\(outcount,)g(outt)o(yp)q(e,)g (source,)g(comm)m(,)11 b(size\))117 1127 y Ft(IN)155 b Fs(outcount)420 b Ft(n)o(um)o(b)q(er)13 b(of)h(output)g(data)f(items) g(\(in)o(teger\))117 1202 y(IN)155 b Fs(outt)o(yp)q(e)445 b Ft(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(output)g(data)g(item)e (\(handle\))117 1277 y(IN)155 b Fs(source)468 b Ft(source)15 b(of)f(pac)o(k)o(ed)g(message)f(\(in)o(teger\))117 1352 y(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(for)j(pac)o(k)o(ed)g (message)g(\(handle\))117 1427 y(OUT)108 b Fs(size)519 b Ft(size)15 b(of)e(pac)o(k)o(ed)h(message)g(\(in)o(teger\))75 1552 y FB(int)23 b(MPI)p 245 1552 15 2 v 17 w(Unpack)p 406 1552 V 17 w(size\(int)f(outount,)h(MPI)p 924 1552 V 17 w(Datatype)g(outtype,)f(int)i(source,)393 1608 y(MPI)p 468 1608 V 17 w(Comm)f(comm,)g(int*)h(size\))75 1695 y(MPI)p 150 1695 V 17 w(PACK)p 263 1695 V 16 w(SIZE\(OUTCOUNT,)e (OUTTYPE,)h(SOURCE,)g(COMM,)g(SIZE,)g(IERROR\))170 1751 y(INTEGER)g(OUTCOUNT,)g(OUTTYPE,)g(SOURCE,)g(COMM,)g(SIZE,)g(IERROR)166 1838 y FD(A)15 b(call)i(to)d Fs(MPI)p 439 1838 14 2 v 16 w(UNP)l(A)o(CK)p 633 1838 V 17 w(SIZE\(outcount,)i(outt)o(yp)q(e,)h (source,)f(comm)m(,)c(size\))j FD(returns)g(in)h Fs(size)g FD(the)75 1894 y(incremen)o(t)i(in)g Fs(p)q(osition)g FD(that)f(is)g(e\013ected)h(b)o(y)f(a)f(call)j(to)d Fs(MPI)p 1157 1894 V 16 w(UNP)l(A)o(CK\(outbuf,)j(outcount,)g(outt)o(yp)q(e,)75 1951 y(inbuf,)d(insize,)f(p)q(osition,)h(dest,)h(tag\))p FD(;)166 2083 y Fl(Discussion:)37 b Ft(This)15 b(de\014nition)f (implies)f(that)h(a)h(pac)o(k)o(ed)g(message)f(tak)o(es)h(the)g(same)f (size,)h(whether)h(it's)75 2133 y(pac)o(k)o(ed)g(\014rst,)f(or)h (second,)g(or)f(third.)22 b(This)15 b(is)g(not)h(true)g(in)f(our)g(h)o (yp)q(othetical)g(example)f(where)i(messages)g(are)75 2183 y(compressed,)g(or)f(in)f(the)i(case)g(where)g(a)f(header)h(is)e (stored)i(when)g(the)g(\014rst)f(message)g(is)g(pac)o(k)o(ed.)22 b(Rather,)15 b(one)75 2233 y(migh)o(t)d(w)o(an)o(t)h(these)j(t)o(w)o(o) d(functions)h(to)g(return)h(an)e(upp)q(er)i(b)q(ound)f(on)g(the)g (amoun)o(t)e(of)i(storage)g(required.)166 2282 y(If)f(w)o(e)i(put)f(in) f(these)i(functions,)f(w)o(e)g(ma)o(y)e(w)o(an)o(t)h(to)h(delete)h(the) g(function)e Fv(MPI)p 1424 2282 13 2 v 14 w(TYPE)p 1546 2282 V 15 w(SIZE)p Ft(.)166 2332 y(If)g(w)o(e)i(supp)q(ort)f(pac)o (king)f(and)h(unpac)o(king,)f(w)o(e)h(ma)o(y)e(w)o(an)o(t)i(to)f (delete)i(the)g(function)75 2382 y Fv(MPI)p 152 2382 V 14 w(USER)p 270 2382 V 14 w(SPECIFIES)p 482 2382 V 13 w(BUFFER)p Ft(.)10 b(If)i(the)h(user)h(w)o(an)o(ts)e(to)g(bu\013er)i (on)e(its)h(o)o(wn,)f(then)h(it)f(will)f(use)i(pac)o(k/unpac)o(k)75 2432 y(calls.)166 2488 y(I)i(cannot)g(resist)h(p)q(oin)o(ting)d(out)i (that)g(at)g(least)g(three)h(di\013eren)o(t)g(groups)f(came)f(out)g (with)h(suggestions)g(for)75 2545 y(pac)o(k/unpac)o(k)i(functions)g (that)g(w)o(ere)h(essen)o(tially)f(iden)o(tical.)27 b(This)17 b(suggests)h(to)f(me)f(that)h(MPI)g(has)h(a)e(go)q(o)q(d)75 2601 y(in)o(ternal)d(coherence,)j(so)e(that)g(there)h(often)f(is)g(one) g(\\righ)o(t")f(w)o(a)o(y)g(of)h(pro)o(viding)e(a)i(new)g(function.) 1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 87 94 87 93 bop 75 357 a Fr(Chapter)34 b(4)75 565 y Fy(Collecti)q(v)m(e)42 b(Comm)m(unication)75 806 y Fq(4.1)59 b(Intro)r(duction)75 909 y FD(Collectiv)o(e)13 b(comm)o(unication)f(is)g(de\014ned)h(to)e(b) q(e)h(comm)o(unication)g(that)f(in)o(v)o(olv)o(es)h(a)f(group)g(of)g (pro)q(cesses.)75 965 y(The)k(functions)h(pro)o(vided)g(b)o(y)g(the)f (MPI)g(collectiv)o(e)i(comm)o(unication)f(include:)143 1061 y FA(\017)23 b FD(Broadcast)14 b(from)g(one)i(mem)o(b)q(er)f(to)g (all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 1157 y FA(\017)23 b FD(Barrier)15 b(sync)o(hronization)h(across)f(all)h(group)f(mem)o(b)q (ers)143 1253 y FA(\017)23 b FD(Gather)14 b(data)h(from)f(all)i(group)f (mem)o(b)q(ers)g(to)g(one)g(mem)o(b)q(er.)143 1349 y FA(\017)23 b FD(Scatter)14 b(data)h(from)f(one)i(mem)o(b)q(er)f(to)g (all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 1445 y FA(\017)23 b FD(Global)15 b(op)q(erations)g(suc)o(h)h(as)e(sum,)h(max,)f(min,)i (etc.,)e(w)o(ere)h(the)g(result)g(is)h(kno)o(wn)e(b)o(y)h(all)h(group) 189 1501 y(mem)o(b)q(ers)e(and)h(a)g(v)m(ariation)g(where)g(the)g (result)g(is)g(kno)o(wn)f(b)o(y)h(only)g(one)g(mem)o(b)q(er.)20 b(The)15 b(abilit)o(y)189 1558 y(to)f(ha)o(v)o(e)h(user)g(de\014ned)i (global)f(op)q(erations.)143 1654 y FA(\017)23 b FD(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g(called)h (parallel)h(pre\014x\).)143 1749 y FA(\017)23 b FD(Broadcast)14 b(from)g(all)j(mem)o(b)q(ers)e(to)f(all)j(mem)o(b)q(ers)e(of)g(a)g (group.)143 1845 y FA(\017)23 b FD(Scatter/Gather)10 b(data)h(from)h(all)g(mem)o(b)q(ers)g(to)g(all)g(mem)o(b)q(ers)g(of)g (a)g(group)f(\(also)h(called)h(complete)189 1902 y(exc)o(hange)i(or)g (all-to-all\).)75 1998 y(While)23 b(v)o(endors)f(ma)o(y)f(optimize)i (some)e(of)h(these)g(collectiv)o(e)h(routines)f(for)g(their)g(arc)o (hitectures,)h(a)75 2054 y(complete)15 b(library)g(of)f(the)h (collectiv)o(e)h(comm)o(unication)f(routines)g(can)g(b)q(e)g(written)f (en)o(tirely)i(using)f(the)75 2111 y(MPI)g(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication)g(functions)g(and)f(a)g(few)g(auxiliary)i(functions.) 166 2168 y(A)h(collectiv)o(e)i(op)q(eration)e(is)g(executed)h(b)o(y)f (ha)o(ving)g(all)h(pro)q(cesses)f(in)h(the)f(group)g(call)h(the)f(com-) 75 2224 y(m)o(unication)f(routine,)f(with)g(matc)o(hing)g(argumen)o (ts.)21 b(The)16 b(syn)o(tax)f(and)h(seman)o(tics)g(of)f(the)h (collectiv)o(e)75 2280 y(op)q(erations)e(are)g(de\014ned)i(to)e(b)q(e)g (consisten)o(t)h(with)f(the)h(syn)o(tax)e(and)i(seman)o(tics)f(of)g (the)g(p)q(oin)o(t-to-p)q(oin)o(t)75 2337 y(op)q(erations.)25 b(Th)o(us)17 b(general)g(datat)o(yp)q(es)f(are)h(allo)o(w)o(ed)g(and)g (m)o(ust)f(matc)o(h)h(b)q(et)o(w)o(een)g(sending)h(and)f(re-)75 2393 y(ceiving)j(pro)q(cesses)g(as)e(sp)q(eci\014ed)j(in)f(the)f(c)o (hapter)f(on)h(p)q(oin)o(t-to-p)q(oin)o(t)h(functions.)31 b(One)20 b(of)e(the)h(k)o(ey)75 2450 y(argumen)o(ts)12 b(is)h(a)g(comm)o(unicator)f(that)g(de\014nes)i(the)f(group)g(of)f (participating)i(pro)q(cesses)f(and)g(pro)o(vides)75 2506 y(a)j(con)o(text)g(for)g(the)g(op)q(eration.)24 b(Sev)o(eral)17 b(collectiv)o(e)h(routines)f(suc)o(h)f(as)g(broadcast)g (and)h(gather)e(ha)o(v)o(e)75 2563 y(a)g(single)i(originating)f(or)f (receiving)i(pro)q(cess.)k(In)16 b(this)g(c)o(hapter)f(suc)o(h)h(pro)q (cesses)g(are)f(called)i(the)e Fo(r)n(o)n(ot)p FD(.)75 2619 y(Some)e(argumen)o(ts)f(in)h(the)g(collectiv)o(e)i(functions)e (are)g(sp)q(eci\014ed)h(as)f(\\signi\014can)o(t)g(only)h(at)e(ro)q (ot".)18 b(These)p 75 2661 720 2 v 127 2688 a Fz(1)144 2704 y Fh(V)m(ersion)c(of)f(Jan)g(4,)f(1994)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 88 95 88 94 bop 75 -100 a FD(88)703 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FD(argumen)o(ts)c(are)h (ignored)g(for)g(all)h(participan)o(ts)f(except)g(the)h(ro)q(ot,)e(and) h(can)g(b)q(e)h(set)e(to)h(an)o(y)f(v)m(alue.)21 b(The)75 102 y(reader)d(is)g(referred)g(to)f(c)o(hapter)g(3)g(for)h(information) f(concerning)i(comm)o(unication)g(bu\013ers,)e(general)75 158 y(datat)o(yp)q(es)11 b(and)g(t)o(yp)q(e)h(matc)o(hing)f(rules;)i (and)f(to)e(c)o(hapter)i(5)f(for)f(information)i(on)f(ho)o(w)g(to)g (de\014ne)h(groups)75 214 y(and)j(create)g(comm)o(unicators.)166 276 y(Collectiv)o(e)i(routines)g(can)f(\(but)g(are)g(not)g(required)h (to\))e(return)i(as)e(so)q(on)h(as)g(their)h(participation)75 332 y(in)i(the)g(collectiv)o(e)h(comm)o(unication)f(is)g(complete.)31 b(The)19 b(completion)g(of)f(a)h(call)g(indicates)h(that)e(the)75 389 y(caller)f(is)f(no)o(w)f(free)h(to)f(access)g(the)h(lo)q(cations)g (in)h(the)e(comm)o(unication)i(bu\013er.)k(It)15 b(do)q(es)h(not)f (indicate)75 445 y(that)k(other)f(pro)q(cesses)i(in)g(the)g(group)e(ha) o(v)o(e)h(started)g(the)g(op)q(eration)h(\(unless)g(otherwise)f (indicated)75 502 y(in)h(the)f(description)h(of)f(the)g(op)q (eration\).)31 b(The)19 b(successful)h(completion)g(of)f(a)g(collectiv) o(e)h(comm)o(uni-)75 558 y(cation)e(call)g(ma)o(y)f(dep)q(end)i(on)e (the)h(execution)g(of)f(a)g(matc)o(hing)h(call)g(at)f(all)h(pro)q (cesses)g(in)g(the)g(group.)75 615 y(Th)o(us,)i(a)f(collectiv)o(e)i (comm)o(unication)f(call)h(ma)o(y)l(,)e(or)g(ma)o(y)f(not,)i(ha)o(v)o (e)f(the)g(e\013ect)h(of)e(sync)o(hronizing)75 671 y(all)g(calling)h (pro)q(cesses.)26 b(Collectiv)o(e)19 b(comm)o(unication)f(calls)g(ma)o (y)f(use)g(the)h(same)f(comm)o(unicators)f(as)75 728 y(p)q(oin)o(t)i(to)f(p)q(oin)o(t)g(comm)o(unication;)i(MPI)e(guaran)o (tees)g(that)f(messages)h(generated)h(on)f(b)q(ehalf)h(of)f(col-)75 784 y(lectiv)o(e)e(comm)o(unication)g(calls)f(will)i(not)d(b)q(e)i (confused)f(with)g(messages)g(generated)f(b)o(y)h(p)q(oin)o(t)g(to)f(p) q(oin)o(t)75 841 y(comm)o(unication.)166 902 y(A)h(more)g(detailed)i (discussion)g(of)e(the)g(correct)g(use)h(of)e(the)i(collectiv)o(e)h (routines)f(can)f(b)q(e)h(found)g(at)75 959 y(the)g(end)h(of)f(this)h (c)o(hapter.)166 1103 y Fl(Discussion:)34 b Ft(The)14 b(collectiv)o(e)g(op)q(erations)h(do)e(not)h(accept)h(a)f(message)g (tag)f(argumen)o(t.)k(The)e(rationale)75 1159 y(for)i(not)g(using)g (tags)g(is)h(that)f(the)h(need)g(for)f(distinguishing)f(collectiv)o(e)h (op)q(erations)h(with)f(the)h(same)e(con)o(text)75 1216 y(seldom)g(arises)i(\(since)g(the)g(op)q(erations)f(are)h(blo)q(c)o (king\);)f(the)h(tag)f(\014eld)g(can)h(b)q(e)f(used)i(b)o(y)e(the)g(p)q (oin)o(t-to-p)q(oin)o(t)75 1272 y(messages)d(that)g(implemen)o(t)d(the) j(collectiv)o(e)g(comm)o(unication.)75 1527 y Fq(4.2)59 b(Communication)18 b(F)n(unctions)75 1639 y FD(The)d(k)o(ey)h(concept)f (of)g(the)g(collectiv)o(e)i(functions)f(is)g(to)f(ha)o(v)o(e)f(a)h (\\group")g(of)g(participating)h(pro)q(cesses.)75 1695 y(The)d(routines)f(do)h(not)e(ha)o(v)o(e)h(a)g(group)g(iden)o(ti\014er) i(as)e(an)g(explicit)j(argumen)o(t.)j(Instead,)13 b(there)f(is)h(a)f (com-)75 1752 y(m)o(unicator)g(argumen)o(t.)18 b(In)12 b(this)g(c)o(hapter)g(a)f(comm)o(unicator)g(can)h(b)q(e)h(though)o(t)e (of)g(as)g(a)h(group)f(iden)o(ti\014er)75 1808 y(link)o(ed)20 b(with)e(a)g(con)o(text.)27 b(\(In)o(ter-comm)o(unicators,)18 b(that)f(is)i(b)q(et)o(w)o(een)f(groups)f(comm)o(unicators,)h(are)75 1865 y(not)d(allo)o(w)o(ed)g(in)h(the)g(collectiv)o(e)h(functions.\))75 2037 y Fq(4.3)59 b(Ba)n(rrier)21 b(synchronization)75 2196 y Fs(MPI)p 160 2196 14 2 v 16 w(BARRIER\()16 b(comm)9 b(\))117 2278 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))75 2408 y FB(int)23 b(MPI)p 245 2408 15 2 v 17 w(Barrier\(MPI)p 526 2408 V 16 w(Comm)g(comm)h(\))75 2499 y(MPI)p 150 2499 V 17 w(BARRIER\(COMM,)e(IERROR\))170 2556 y(INTEGER)h(COMM,)h(IERROR)166 2647 y Fs(MPI)p 251 2647 14 2 v 16 w(BARRIER)15 b FD(blo)q(c)o(ks)g(the)f(caller)h(un)o (til)g(all)f(group)g(mem)o(b)q(ers)g(ha)o(v)o(e)f(called)j(it;)e(the)g (call)h(returns)75 2704 y(at)g(an)o(y)f(pro)q(cess)i(only)g(after)e (all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered)h(the)f(call.) 1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 89 96 89 95 bop 75 -100 a Fp(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FD(89)75 45 y Fq(4.4)59 b(Data)19 b(move)g(functions) 75 148 y FD(Figure)c(4.1)g(illustrates)h(the)f(the)g(di\013eren)o(t)h (collectiv)o(e)h(mo)o(v)o(e)d(functions)i(supp)q(orted)g(b)o(y)f(MPI.) 75 272 y Fn(4.4.1)49 b(Broadcast)90 406 y Fs(MPI)p 175 406 14 2 v 16 w(BCAST\()16 b(bu\013er,)f(count,)h(datat)o(yp)q(e,)g(ro) q(ot,)f(comm)9 b(\))117 484 y Ft(INOUT)62 b Fs(bu\013er)478 b Ft(starting)14 b(address)h(of)f(bu\013er)g(\(c)o(hoice\))117 560 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h (in)e(bu\013er)i(\(in)o(teger\))117 636 y(IN)171 b Fs(datat)o(yp)q(e) 408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)i(\(handle\))117 713 y(IN)171 b Fs(ro)q(ot)492 b Ft(rank)14 b(of)f(broadcast)i(ro)q(ot)f (\(in)o(teger\))117 789 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 914 y FB(int)47 b(MPI)p 269 914 15 2 v 17 w(Bcast\(void*)22 b(buf,)i(int)f(count,)g(MPI)p 1026 914 V 17 w(Datatype)g(datatype,)f(int)i(root,)393 970 y(MPI)p 468 970 V 17 w(Comm)f(comm)h(\))75 1057 y(MPI)p 150 1057 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,)g(COMM,)g (IERROR\))170 1114 y()g(BUFFER\(*\))170 1170 y(INTEGER)g(COUNT,)g (DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)166 1257 y Fs(MPI)p 251 1257 14 2 v 16 w(BCAST)11 b FD(broadcasts)e(a)h(message)g(from)g (the)g(pro)q(cess)h(with)f(rank)21 b Fs(ro)q(ot)10 b FD(to)f(all)j(other)e(pro)q(cesses)75 1314 y(of)18 b(the)g(group.)28 b(It)18 b(is)h(called)h(b)o(y)e(all)h(mem)o(b)q(ers)f(of)g(group)g (using)g(the)h(same)e(argumen)o(ts)h(for)35 b Fs(comm)m(,)75 1370 y(ro)q(ot)16 b FD(and)g(matc)o(hing)g(argumen)o(ts)f(for)31 b Fs(count,)17 b(datat)o(yp)q(e)p FD(.)23 b(On)17 b(return)f(the)g(con) o(ten)o(ts)f(of)g(the)h(bu\013er)g(of)75 1427 y(the)f(pro)q(cess)h (with)f(rank)30 b Fs(ro)q(ot)15 b FD(is)h(con)o(tained)g(in)g(the)f (bu\013er)g(of)g(the)g(calling)i(pro)q(cess.)75 1551 y Fn(4.4.2)49 b(Gather)75 1685 y Fs(MPI)p 160 1685 V 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcount,)h(sendt)o(yp)q(e,)f (recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)m(\))117 1763 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 1839 y(IN)171 b Fs(sendcount)383 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (send)h(bu\013er)f(\(in)o(teger\))117 1915 y(IN)171 b Fs(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er) g(elemen)o(ts)f(\(handle\))117 1992 y(OUT)124 b Fs(recvbuf)434 b Ft(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 2048 y(ro)q(ot\))117 2124 y(IN)171 b Fs(recvcount)389 b Ft(n)o(um)o(b)q(er)11 b(of)g(elemen)o(ts)g(for)g(an)o(y)g(single)g(receiv)o(e)i(\(in)o (teger,)f(sig-)905 2181 y(ni\014can)o(t)i(only)f(at)h(ro)q(ot\))117 2257 y(IN)171 b Fs(recvt)o(yp)q(e)414 b Ft(data)16 b(t)o(yp)q(e)g(of)f (recv)i(bu\013er)g(elemen)o(ts)f(\(signi\014can)o(t)f(only)g(at)905 2313 y(ro)q(ot\))f(\(handle\))117 2390 y(IN)171 b Fs(ro)q(ot)492 b Ft(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 2466 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 2591 y FB(int)23 b(MPI)p 245 2591 15 2 v 17 w(Gather\(void*)f(sendbuf,) h(int)h(sendcount,)e(MPI)p 1217 2591 V 17 w(Datatype)h(sendtype,)393 2647 y(void*)g(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 2647 V 17 w(Datatype)h(recvtype,)f(int)i(root,)393 2704 y(MPI)p 468 2704 V 17 w(Comm)f(comm\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 90 97 90 96 bop 75 -100 a FD(90)703 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 2420 y @beginspecial @setspecial %%BeginDocument: coll-fig1.ps /arrowdict 13 dict def % Local storage for the procedure % ``arrow.'' /arrow % The procedure ``arrow'' adds an { arrowdict begin % arrow shape to the current path. /headlength exch def % It takes seven arguments: the x /halfheadthickness exch 2 div def % and y coordinates of the tail /halfthickness exch 2 div def % (imagine that a line has been /tipy exch def /tipx exch def % drawn down the center of the /taily exch def /tailx exch def % arrow from the tip to the tail, % then x and y lie on this line), % the x and y coordinates of the % tip of the arrow, the thickness % of the arrow in the tail % portion, the thickness of the % arrow at the widest part of the % arrowhead and the length of the % arrowhead. /dx tipx tailx sub def % Compute the differences in x and /dy tipy taily sub def % y for the tip and tail. These /arrowlength dx dx mul dy dy mul add % will be used to compute the sqrt def % length of the arrow and to /angle dy dx atan def % compute the angle of direction % that the arrow is facing with % respect to the current user % coordinate system origin. /base arrowlength headlength sub def % Compute where the base of the % arrowhead will be. /savematrix matrix currentmatrix def % Save the current user coordinate % system. We are using the same % strategy to localize the effect % of transformations as was used % in the program to draw an % ellipse. tailx taily translate % Translate to the starting point % of the tail. angle rotate % Rotate the x-axis to correspond % with the center line of the % arrow. 0 halfthickness neg moveto % Add the arrow shape to the % current path. base halfthickness neg lineto base halfheadthickness neg lineto arrowlength 0 lineto base halfheadthickness lineto base halfthickness lineto 0 halfthickness lineto closepath savematrix setmatrix % Restore the current user % coordinate system. end } def /Box { /height exch def /length exch def length 0 rlineto 0 height rlineto length neg 0 rlineto closepath } def /Gdict 200 dict def /Grid { Gdict begin /ny exch def /nx exch def /dely exch def /delx exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def 0 1 ny { pop x y moveto lenx 0 rlineto stroke /y y dely add def} for /y ypos def /x xpos def 0 1 nx { pop x y moveto 0 leny rlineto stroke /x x delx add def} for end } def /GLdict 300 dict def /GridLabels { GLdict begin /shift exch def /raise exch def /yoff exch def /xoff exch def /p1 exch def /p2 exch def /ny exch def /nx exch def /dely exch def /delx exch def /Darray exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def /dx3 delx 3 div def /dy3 dely 3 div def /ix -1 def /iy ny 1 sub def Darray{ aload pop /Subc exch def /Text exch def /ix ix 1 add def ix nx ge { /ix 0 def /iy iy 1 sub def} if /x xpos delx ix 0.5 add mul add /Helvetica findfont p1 scalefont setfont Text stringwidth pop /Helvetica findfont p2 scalefont setfont Subc stringwidth pop add xoff add 2 div sub shift add def /y ypos dely iy 0.5 add mul add raise add def x y moveto /Helvetica findfont p1 scalefont Text show xoff yoff rmoveto /Helvetica findfont p2 scalefont Subc show } forall end clear } def 2 setlinecap 6.5 72 mul 320 sub 2 div 0 translate 0 150 moveto 20 20 6 6 Grid 0 150 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 225 200 225 12 24 18 arrow stroke 200 195 140 195 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (one-all scatter) dup stringwidth pop 170 exch 2 div sub 242 moveto show (one-all gather) dup stringwidth pop 170 exch 2 div sub 170 moveto show 220 150 moveto 20 20 6 6 Grid 220 150 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(1)] [()()] [()()] [()()] [()()] [()()] [(A)(2)] [()()] [()()] [()()] [()()] [()()] [(A)(3)] [()()] [()()] [()()] [()()] [()()] [(A)(4)] [()()] [()()] [()()] [()()] [()()] [(A)(5)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 0 moveto 20 20 6 6 Grid 0 0 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [(B)(0)] [(B)(1)] [(B)(2)] [(B)(3)] [(B)(4)] [(B)(5)] [(C)(0)] [(C)(1)] [(C)(2)] [(C)(3)] [(C)(4)] [(C)(5)] [(D)(0)] [(D)(1)] [(D)(2)] [(D)(3)] [(D)(4)] [(D)(5)] [(E)(0)] [(E)(1)] [(E)(2)] [(E)(3)] [(E)(4)] [(E)(5)] [(F)(0)] [(F)(1)] [(F)(2)] [(F)(3)] [(F)(4)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 220 0 moveto 20 20 6 6 Grid 220 0 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(1)] [(B)(1)] [(C)(1)] [(D)(1)] [(E)(1)] [(F)(1)] [(A)(2)] [(B)(2)] [(C)(2)] [(D)(2)] [(E)(2)] [(F)(2)] [(A)(3)] [(B)(3)] [(C)(3)] [(D)(3)] [(E)(3)] [(F)(3)] [(A)(4)] [(B)(4)] [(C)(4)] [(D)(4)] [(E)(4)] [(F)(4)] [(A)(5)] [(B)(5)] [(C)(5)] [(D)(5)] [(E)(5)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 60 200 60 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (all-all scatter) dup stringwidth pop 170 exch 2 div sub 77 moveto show 0 300 moveto 20 20 6 6 Grid 0 300 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(B)(0)] [()()] [()()] [()()] [()()] [()()] [(C)(0)] [()()] [()()] [()()] [()()] [()()] [(D)(0)] [()()] [()()] [()()] [()()] [()()] [(E)(0)] [()()] [()()] [()()] [()()] [()()] [(F)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 360 200 360 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (all gather) dup stringwidth pop 170 exch 2 div sub 377 moveto show 220 300 moveto 20 20 6 6 Grid 220 300 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 450 moveto 20 20 6 6 Grid 0 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 510 200 510 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont 0 575 moveto (data) show (data) stringwidth pop 4 add 575 4 add (data) stringwidth pop 33 add 575 4 add 1 4 5 arrow fill (one-all broadcast) dup stringwidth pop 170 exch 2 div sub 527 moveto show gsave 0 570 (processes) stringwidth pop sub translate 90 rotate 0 5 moveto (processes) show -4 8 -33 8 1 4 5 arrow fill grestore 220 450 moveto 20 20 6 6 Grid 220 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels showpage %%EndDocument @endspecial 98 x FD(Figure)f(4.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2574 y(eac)o(h)g(ro)o(w)g(of)g(b)q(o)o(xes)g (represen)o(ts)g(data)g(lo)q(cations)h(in)g(one)g(pro)q(cess.)k(Th)o (us,)15 b(in)h(the)f(one-all)i(broadcast,)75 2631 y(initially)k(just)e (the)f(\014rst)h(pro)q(cess)g(con)o(tains)f(the)h(data)f Fw(A)1085 2638 y Fi(0)1105 2631 y FD(,)h(but)g(after)e(the)i(broadcast) f(all)i(pro)q(cesses)75 2687 y(con)o(tain)15 b(it.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 91 98 91 97 bop 75 -100 a Fp(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FD(91)75 45 y FB(MPI)p 150 45 15 2 v 17 w(GATHER\(SENDBUF,)22 b(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 102 y(ROOT,)g(COMM,)h(IERROR\))170 158 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 214 y(INTEGER)g (SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g(COMM,)g (IERROR)166 335 y FD(Eac)o(h)17 b(pro)q(cess)g(\(including)i(the)e(ro)q (ot)f(pro)q(cess\))h(sends)g(the)g(con)o(ten)o(ts)g(of)f(its)h(send)h (bu\013er)f(to)f(the)75 391 y(ro)q(ot)i(pro)q(cess.)31 b(The)19 b(ro)q(ot)f(pro)q(cess)h(receiv)o(es)h(the)f(messages)f(at)g (stores)g(them)h(in)h(rank)e(order.)31 b(The)75 448 y(outcome)15 b(is)h(as)e(if)i(eac)o(h)f(of)g(the)g FB(n)g FD(pro)q(cesses)h(in)g (the)f(group)g(had)h(executed)g(a)f(call)h(to)189 711 y FB(MPI)p 264 711 V 16 w(Send)p FD(\()p FB(sendbuf)p Fw(;)8 b FB(sendcount)p Fw(;)f FB(se)o(ndtype)p Fw(;)g FB(roo)o(t)p Fw(;)h(:)o(::)o FD(\))p Fw(;)75 940 y FD(and)15 b(the)h(ro)q(ot)e(had)h(executed)i FB(n)e FD(calls)h(to)189 1203 y FB(MPI)p 264 1203 V 16 w(Recv)p FD(\()p FB(recvbuf)p FD([)p FB(i)p FD(])p Fw(;)8 b FB(rec)o(vcount)p Fw(;)f FB(rec)o(vtype)p Fw(;)g FB(i)p Fw(;)g(::)o(:)p FD(\))p Fw(;)75 1432 y FD(where,)189 1695 y FB(recvbuf)p FD([)p FB(i)p FD(])j(=)j FB(recvbuf)d FD(+)g FB(i)g FA(\002)g FB(recvcount)f FA(\002)i FB(extent)p FD(\()p FB(recvtype)p FD(\))p Fw(:)166 1925 y FD(An)18 b(alternativ)o(e)h(description)h(is)e (that)g(the)g FB(n)g FD(messages)g(sen)o(t)g(b)o(y)g(the)g(pro)q (cesses)h(in)g(the)f(group)75 1981 y(are)c(concatenated)h(in)g(rank)f (order,)g(and)h(the)g(resulting)g(message)f(is)h(receiv)o(ed)h(b)o(y)e (the)h(ro)q(ot)f(as)g(if)h(b)o(y)f(a)75 2037 y(call)i(to)f Fs(MPI)p 299 2037 14 2 v 16 w(RECV\(recvbuf,)h(recvcount)g FA(\002)p Fs(n,)g(recvt)o(yp)q(e,)f(...\))p FD(.)166 2128 y(The)g(receiv)o(e)h(bu\013er)g(is)f(ignored)h(for)f(all)h(non-ro) q(ot)f(pro)q(cesses.)166 2218 y(General)21 b(deriv)o(ed)h(datat)o(yp)q (es)f(are)f(allo)o(w)o(ed)i(for)e(b)q(oth)42 b Fs(sendt)o(yp)q(e)24 b FD(and)42 b Fs(recvt)o(yp)q(e)p FD(.)c(The)21 b(t)o(yp)q(e)75 2275 y(signature)15 b(of)30 b Fs(sendcount,)18 b(sendt)o(yp)q(e)g FD(on)d(pro)q(cess)g(i)h(m)o(ust)f(b)q(e)h(equal)g(to)e(the)h(t)o(yp)q (e)h(signature)f(of)30 b Fs(recv-)75 2331 y(count,)14 b(recvt)o(yp)q(e)g FD(at)e(the)h(ro)q(ot.)19 b(Note)12 b(that)g(the)h(amoun)o(t)f(of)h(data)f(sen)o(t)h(m)o(ust)f(b)q(e)i (equal)f(to)g(the)g(amoun)o(t)75 2387 y(receiv)o(ed)20 b(\(pairwise)f(b)q(et)o(w)o(een)g(eac)o(h)f(pro)q(cess)h(and)g(the)g (ro)q(ot\).)28 b Fs(MPI)p 1298 2387 V 16 w(GA)l(THER)20 b FD(and)f(all)g(other)g(data)75 2444 y(mo)o(v)o(emen)o(t)10 b(collectiv)o(e)j(routines)f(mak)o(e)f(this)h(restriction)f(and)h(pro)o (vide)g(no)f(facilit)o(y)h(\(suc)o(h)g(as)e(the)i(status)75 2500 y(argumen)o(t)i(of)h Fs(MPI)p 415 2500 V 16 w(RECV)p FD(\))h(for)e(disco)o(v)o(ering)i(ho)o(w)f(m)o(uc)o(h)g(data)g(w)o(as)f (sen)o(t.)166 2591 y(All)h(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fs(ro)q(ot)p FD(,)e(while)j(on)e (other)g(pro)q(cesses,)75 2647 y(only)19 b(argumen)o(ts)f Fs(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,)d(comm)13 b FD(are)19 b(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 2704 y Fs(ro)q(ot)c FD(and)g Fs(comm)10 b FD(m)o(ust)k(ha)o(v)o(e)h (iden)o(tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 92 99 92 98 bop 75 -100 a FD(92)703 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(GA)l(THERV\()24 b(sendbuf,)j(sendcount,)g(sendt)o(yp)q (e,)g(recvbuf,)e(recvcounts,)h(displs,)g(recvt)o(yp)q(e,)g(ro)q(ot,)75 102 y(comm)m(\))117 179 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 255 y(IN)171 b Fs(sendcount)383 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 330 y(IN)171 b Fs(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 406 y(OUT)124 b Fs(recvbuf)434 b Ft(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o (hoice,)i(signi\014can)o(t)d(only)g(at)905 462 y(ro)q(ot\))117 538 y(IN)171 b Fs(recvcounts)372 b Ft(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(con)o(taining)e(the)i(n)o(um-)905 594 y(b)q(er)20 b(of)e(elemen)o(ts)h(that)f(are)i(receiv)o(ed)g(from)d (eac)o(h)i(pro)q(cess)905 651 y(\(signi\014can)o(t)14 b(only)f(at)h(ro)q(ot\))117 726 y(IN)171 b Fs(displs)469 b Ft(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Ft(sp)q(eci\014es)905 783 y(the)h(displacemen)o (t)e(relativ)o(e)g(to)h Fv(recvbuf)i Ft(at)d(whic)o(h)h(to)g(place)905 839 y(the)h(incoming)e(data)h(from)f(pro)q(cess)k Fb(i)d Ft(\(signi\014can)o(t)g(only)g(at)905 896 y(ro)q(ot\))117 971 y(IN)171 b Fs(recvt)o(yp)q(e)414 b Ft(data)16 b(t)o(yp)q(e)g(of)f (recv)i(bu\013er)g(elemen)o(ts)f(\(signi\014can)o(t)f(only)g(at)905 1028 y(ro)q(ot\))f(\(handle\))117 1103 y(IN)171 b Fs(ro)q(ot)492 b Ft(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 1179 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 1304 y FB(int)23 b(MPI)p 245 1304 15 2 v 17 w(Gatherv\(void*)f (sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 1304 V 17 w(Datatype)h (sendtype,)393 1360 y(void*)g(recvbuf,)g(int)h(*recvcounts,)e(int)h (*displs,)393 1416 y(MPI)p 468 1416 V 17 w(Datatype)g(recvtype,)f(int)i (root,)f(MPI)p 1249 1416 V 17 w(Comm)g(comm\))75 1503 y(MPI)p 150 1503 V 17 w(GATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)g (RECVBUF,)f(RECVCOUNTS,)h(DISPLS,)393 1560 y(RECVTYPE,)g(ROOT,)g(COMM,) g(IERROR\))170 1616 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1673 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g (DISPLS\(*\),)h(RECVTYPE,)g(ROOT,)170 1729 y(COMM,)h(IERROR)166 1816 y Fs(MPI)p 251 1816 14 2 v 16 w(GA)l(THERV)15 b FD(extends)g(the)f(functionalit)o(y)h(of)e Fs(MPI)p 1126 1816 V 16 w(GA)l(THER)i FD(b)o(y)f(allo)o(wing)h(a)f(v)m(arying)g(coun) o(t)75 1872 y(of)e(data)h(from)f(eac)o(h)g(pro)q(cess)i(\(since)f Fs(recvcounts)i FD(is)e(no)o(w)f(an)h(arra)o(y\),)e(and)i(also)g(allo)o (ws)g(more)f(\015exibili)q(t)o(y)75 1929 y(as)j(to)f(where)i(the)f (data)g(is)g(placed)i(on)e(the)g(ro)q(ot)g(\(b)o(y)f(pro)o(viding)i (the)g(new)f(argumen)o(t,)f Fs(displs)p FD(\).)166 1985 y(The)h(outcome)g(is)h(as)f(if)g(eac)o(h)h(pro)q(cess)f(has)g(sen)o(t)g (a)g(message)g(to)f(the)i(ro)q(ot,)189 2080 y FB(MPI)p 264 2080 15 2 v 16 w(Send)p FD(\()p FB(sendbuf)p Fw(;)8 b FB(sendcount)p Fw(;)f FB(se)o(ndtype)p Fw(;)g FB(roo)o(t)p Fw(;)h(:)o(::)o FD(\))p Fw(;)75 2175 y FD(and)15 b(the)h(ro)q(ot)e (executed)i FB(n)f FD(receiv)o(es,)189 2270 y FB(MPI)p 264 2270 V 16 w(Recv)p FD(\()p FB(recvbuf)9 b FD(+)h FB(disp)p FD([)p FB(i)p FD(])f FA(\002)h FB(extent)p FD(\()p FB(recvtype)p FD(\))p Fw(;)e FB(rec)o(vcounts)p FD([)o FB(i)p FD(])p Fw(;)f FB(rec)o(vtype)p Fw(;)g FB(i)p Fw(;)g(::)o(:)p FD(\))p Fw(:)166 2365 y FD(Messages)14 b(are)g(placed)i(in)f(the)g(receiv)o(e)g(bu\013er)g(of)f(the)g(ro)q(ot) g(pro)q(cess)h(in)g(rank)f(order,)g(that)g(is,)h(the)75 2421 y(data)d(sen)o(t)g(from)f(pro)q(cess)i FB(j)f FD(is)h(placed)h(in) f(the)f FB(j)p FD(-th)h(p)q(ortion)f(of)g(the)h(receiv)o(e)g(bu\013er)f Fs(recvbuf)h FD(on)g(pro)q(cess)75 2478 y Fs(ro)q(ot)p FD(.)29 b(The)18 b FB(j)p FD(-th)g(p)q(ortion)h(of)f Fs(recvbuf)h FD(b)q(egins)g(at)f(o\013set)f Fs(displs[j])j FD(elemen)o(ts)f(\(in)g(terms)f(of)f Fs(recvt)o(yp)q(e)p FD(\))75 2534 y(in)o(to)e Fs(recvbuf)p FD(.)166 2591 y(The)i(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fs(sendcount,)j (sendt)o(yp)q(e)g FD(on)e(pro)q(cess)g FB(i)f FD(m)o(ust)g(b)q(e)h (equal)g(to)f(the)75 2647 y(t)o(yp)q(e)h(signature)f(implied)j(b)o(y)e Fs(recvcounts[i],)h(recvt)o(yp)q(e)f FD(at)f(the)h(ro)q(ot)f(\(ho)o(w)o (ev)o(er,)f(the)i(t)o(yp)q(e)f(maps)h(ma)o(y)75 2704 y(b)q(e)f(di\013eren)o(t\).)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 93 100 93 99 bop 75 -100 a Fp(4.4.)29 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1082 b FD(93)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 189 244 m 189 294 l gs 0.50 setgray fill gr gs col-1 s gr n 191.000 286.000 m 189.000 294.000 l 187.000 286.000 l gs 2 setlinejoin col-1 s gr n 289 244 m 259 294 l gs 0.50 setgray fill gr gs col-1 s gr n 264.831 288.169 m 259.000 294.000 l 261.401 286.111 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr $F2psEnd %%EndDocument endTexFig 186 511 a FD(Figure)16 b(4.2:)j(The)c(ro)q(ot)g(pro)q(cess)g(gathers)f (100)h FB(int)p FD(s)f(from)h(eac)o(h)g(pro)q(cess)h(in)g(the)f(group.) 166 644 y(All)g(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fs(ro)q(ot)p FD(,)e(while)j(on)e (other)g(pro)q(cesses,)75 701 y(only)19 b(argumen)o(ts)f Fs(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,)d(comm)13 b FD(are)19 b(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 757 y Fs(ro)q(ot)c FD(and)g Fs(comm)10 b FD(m)o(ust)k(ha)o(v)o(e)h (iden)o(tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)166 814 y(F)l(or)f(b)q(oth)h(functions,)g(the)g(sp)q(eci\014cation)h(of)f (coun)o(t\(s\),)e(t)o(yp)q(e\(s\),)h(\(displacemen)o(ts\),)i(should)f (not)75 870 y(cause)e(an)o(y)g(lo)q(cation)h(on)f(the)h(ro)q(ot)e(to)h (b)q(e)g(written)h(more)e(than)i(once.)k(Suc)o(h)c(a)e(call)j(is)f (erroneous.)166 927 y(W)l(e)f(illustrate)h(the)g(matc)o(hing)f (conditions)h(with)g(the)f(follo)o(wing)h(examples.)75 1048 y Fn(4.4.3)49 b(Examples)14 b(of)j(Usage)g(of)f(MPI)p 768 1048 15 2 v 18 w(GA)l(THER,)h(MPI)p 1091 1048 V 18 w(GA)l(THERV)75 1134 y Fs(Example)c(1)75 1220 y FD(Gather)h(100)h(in)o (ts)g(from)g(ev)o(ery)g(pro)q(cess)g(in)h(group)f(to)g(ro)q(ot.)k(See)d (\014gure)f(4.2.)170 1326 y FB(MPI_Comm)23 b(comm;)170 1382 y(int)h(gsize,sendarray[100];)170 1439 y(int)g(root,)f(*rbuf;)170 1552 y(...)170 1665 y(/*)h(The)f(variable)g(comm)h(is)f(set)h (elsewhere)e(in)i(the)f(program)194 1721 y(*/)170 1778 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 1834 y(rbuf)g(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 1890 y(MPI_Gather\()f (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(100,)f(MPI_INT,)g(root,)g (comm\);)75 2060 y Fs(Example)13 b(2)75 2146 y FD(Do)f(the)h(same)g(as) f(in)i(previous)g(example,)f(but)g(use)h(a)e(deriv)o(ed)i(datat)o(yp)q (e.)19 b(Note)12 b(that)g(the)h(t)o(yp)q(e)g(cannot)75 2202 y(b)q(e)19 b(the)f(en)o(tire)g(set)g(of)f FB(gsize*100)23 b(int)p FD(s)18 b(since)h(t)o(yp)q(e)f(matc)o(hing)g(is)g(de\014ned)i (pairwise)f(b)q(et)o(w)o(een)f(the)75 2259 y(ro)q(ot)c(and)i(eac)o(h)f (pro)q(cess)g(in)h(the)g(gather.)170 2365 y FB(MPI_Comm)23 b(comm;)170 2421 y(int)h(gsize,sendarray[100];)170 2478 y(int)g(root,)f(*rbuf;)170 2534 y(MPI_Datatype)g(rtype;)170 2647 y(...)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 94 101 94 100 bop 75 -100 a FD(94)708 b Fp(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig3.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 189 244 m 204 294 l gs col-1 s gr n 203.617 285.763 m 204.000 294.000 l 199.786 286.912 l gs 2 setlinejoin col-1 s gr n 289 244 m 289 294 l gs col-1 s gr n 291.000 286.000 m 289.000 294.000 l 287.000 286.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 511 a FD(Figure)h(4.3:)22 b(The)16 b(ro)q(ot)g(pro)q(cess)h (gathers)f(100)g FB(int)p FD(s)g(from)f(eac)o(h)i(pro)q(cess)g(in)g (the)g(group,)f(eac)o(h)h(set)f(is)75 568 y(placed)g FB(stride)f FD(in)o(ts)g(apart.)170 694 y FB(/*)24 b(The)f(variable)g (comm)h(is)f(set)h(elsewhere)e(in)i(the)f(program)194 751 y(*/)170 807 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 864 y(MPI_Type_contiguous\()e(100,)h(MPI_INT,)g(&rtype)g(\);)170 920 y(MPI_Type_commit\()f(&rtype)h(\);)170 977 y(rbuf)h(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 1033 y(MPI_Gather\()f (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(1,)f(rtype,)g(root,)g(comm\);) 75 1151 y Fs(Example)13 b(3)75 1237 y FD(No)o(w)g(ha)o(v)o(e)g(eac)o(h) h(pro)q(cess)g(send)g(100)f(in)o(ts)h(to)f(ro)q(ot,)g(but)g(place)i (eac)o(h)f(set)f(\(of)g(100\))f Fo(stride)i FD(in)o(ts)g(apart)f(at)75 1294 y(receiving)18 b(end.)24 b(Use)16 b Fs(MPI)p 547 1294 14 2 v 16 w(GA)l(THERV)i FD(and)e(the)h Fs(displs)g FD(argumen)o(t)f(to)f(ac)o(hiev)o(e)i(this)g(e\013ect.)23 b(Assume)75 1350 y Fw(str)q(ide)13 b FA(\025)g FD(100.)19 b(See)c(\014gure)h(4.3.)170 1445 y FB(MPI_Comm)23 b(comm;)170 1501 y(int)h(gsize,sendarray[100];)170 1558 y(int)g(root,)f(*rbuf,)g (stride;)170 1614 y(int)h(*displs,i,*rcounts;)170 1727 y(...)170 1840 y(MPI_Comm_size\()e(comm,)i(&gsize\);)170 1897 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*stride*siz)o(eof\(int)o (\)\);)170 1953 y(displs)f(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o (nt\)\);)170 2009 y(rcounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)170 2066 y(for)h(\(i=0;)f(i)f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2617 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)166 2704 y Fs(MPI)p 251 2704 14 2 v 16 w(SCA)l(TTER)16 b FD(is)g(the)f(in)o(v)o(erse)h(op)q(eration)f(to)g Fs(MPI)p 1097 2704 V 15 w(GA)l(THER)p FD(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 100 107 100 106 bop 75 -100 a FD(100)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y FD(The)d(ro)q(ot)g(pro)q (cess)g(sends)h(a)f(message)g(to)g(eac)o(h)g(pro)q(cess)g(\(including)j (itself)t(\);)e(eac)o(h)f(pro)q(cess)h(stores)75 102 y(the)j(incoming)h(message)e(in)i(its)f(receiv)o(e)h(bu\013er.)25 b(The)17 b(outcome)f(is)h(as)g(if)g(the)g(ro)q(ot)f(executed)i FB(n)e FD(send)75 158 y(op)q(erations,)189 254 y FB(MPI)p 264 254 15 2 v 16 w(Send)p FD(\()p FB(sendbuf)9 b FD(+)h FB(i)g FA(\002)h FB(sendcount)d FA(\002)j FB(extent)p FD(\()p FB(sendtype)p FD(\))p Fw(;)d FB(s)o(endcoun)o(t)p Fw(;)f FB(sendtype)o Fw(;)h FB(i)o Fw(;)g(:)o(::)p FD(\))o Fw(;)75 350 y FD(and)15 b(eac)o(h)h(pro)q(cess)f(executed)h(a)f(receiv) o(e,)189 446 y FB(MPI)p 264 446 V 16 w(Recv)p FD(\()p FB(recvbuf)p Fw(;)8 b FB(recvcount)p Fw(;)f FB(re)o(cvtype)p Fw(;)g FB(i)p Fw(;)g(::)o(:)p FD(\))o Fw(:)166 541 y FD(An)22 b(alternativ)o(e)f(description)i(is)f(that)f(the)g(ro)q(ot)g (send)h(a)f(message)g(with)h Fs(MPI)p 1594 541 14 2 v 16 w(Send\(sendbuf,)75 598 y(sendcount)p FA(\002)p Fs(n)q(,)j(sendt)o (yp)q(e,)h(...\))p FD(;)c(this)g(message)f(is)h(split)h(in)o(to)e FB(n)g FD(equal)i(segmen)o(ts,)f(and)f(the)h Fw(i)14 b FA(\000)g Fw(th)75 654 y FD(segmen)o(t)h(is)i(sen)o(t)e(to)g(the)h Fw(i)p FD(-th)g(pro)q(cess)g(in)g(the)g(group;)f(and)h(eac)o(h)g(pro)q (cess)g(receiv)o(es)h(this)f(message)f(as)75 711 y(ab)q(o)o(v)o(e.)166 768 y(The)j(t)o(yp)q(e)f(signature)h(asso)q(ciated)g(with)f Fs(sendcount,)k(sendt)o(yp)q(e)f FD(at)d(the)h(ro)q(ot)e(m)o(ust)h(b)q (e)h(equal)h(to)75 824 y(the)f(t)o(yp)q(e)g(signature)g(asso)q(ciated)g (with)g Fs(recvcount,)h(recvt)o(yp)q(e)g FD(at)e(all)i(pro)q(cesses)f (\(ho)o(w)o(ev)o(er,)f(the)h(t)o(yp)q(e)75 880 y(maps)d(ma)o(y)f(b)q(e) i(di\013eren)o(t\).)166 937 y(All)f(argumen)o(ts)f(to)f(the)h(function) h(are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fs(ro)q(ot)p FD(,)e(while)j(on)e(other)g(pro)q(cesses,)75 994 y(only)f(argumen)o(ts) f Fs(recvbuf,)i(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)7 b FD(are)12 b(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fs(ro)q(ot)75 1050 y FD(and)j Fs(comm)10 b FD(m)o(ust)k(ha)o(v)o(e)h (iden)o(tical)i(v)m(alues)g(on)e(all)h(pro)q(cesses.)75 1154 y Fs(MPI)p 160 1154 V 16 w(SCA)l(TTERV\()21 b(sendbuf,)j (sendcounts)q(,)g(displs,)f(sendt)o(yp)q(e,)h(recvbuf,)f(recvcount,)f (recvt)o(yp)q(e,)h(ro)q(ot,)75 1211 y(comm)m(\))117 1289 y Ft(IN)171 b Fs(sendbuf)428 b Ft(address)13 b(of)d(send)i(bu\013er)h (\(c)o(hoice,)f(signi\014can)o(t)e(only)h(at)g(ro)q(ot\))117 1365 y(IN)171 b Fs(sendcounts)366 b Ft(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(sp)q(ecifying)g(the)g(n)o (um-)905 1421 y(b)q(er)15 b(of)e(elemen)o(ts)h(to)g(send)h(to)e(eac)o (h)i(pro)q(cessor)117 1497 y(IN)171 b Fs(displs)469 b Ft(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Ft(sp)q(eci\014es)905 1554 y(the)20 b(displacemen)o(t)e(\(relativ)o(e)h(to)f Fv(sendbuf)j Ft(from)c(whic)o(h)i(to)905 1610 y(tak)o(e)14 b(the)h(outgoing)d(data)i (to)g(pro)q(cess)h Fb(i)117 1686 y Ft(IN)171 b Fs(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 1762 y(OUT)124 b Fs(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1838 y(IN)171 b Fs(recvcount)389 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1914 y(IN)171 b Fs(recvt)o(yp)q(e)414 b Ft(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\))117 1990 y(IN)171 b Fs(ro)q(ot)492 b Ft(rank)14 b(of)f(sending)h(pro)q (cess)i(\(in)o(teger\))117 2066 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 2191 y FB(int)23 b(MPI)p 245 2191 15 2 v 17 w(Scatterv\(void*)f(sendbuf,)h(int)g (*sendcounts,)g(int)g(*displs,)393 2248 y(MPI)p 468 2248 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e(int)i(recvcount,)393 2304 y(MPI)p 468 2304 V 17 w(Datatype)f(recvtype,)f(int)i(root,)f(MPI)p 1249 2304 V 17 w(Comm)g(comm\))75 2391 y(MPI)p 150 2391 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i(DISPLS,)g(SENDTYPE,)f (RECVBUF,)h(RECVCOUNT,)393 2447 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\)) 170 2504 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2560 y(INTEGER)g(SENDCOUNTS\(*\),)f(DISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 2617 y(COMM,)h(IERROR)166 2704 y Fs(MPI)p 251 2704 14 2 v 16 w(SCA)l(TTERV)16 b FD(is)g(the)f(in)o(v)o(erse)h(op)q(eration)f(to)g Fs(MPI)p 1127 2704 V 16 w(GA)l(THERV)p FD(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 101 108 101 107 bop 75 -100 a Fp(4.4.)29 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1059 b FD(101)166 45 y Fs(MPI)p 251 45 14 2 v 16 w(SCA)l(TTERV)24 b FD(extends)f(the)g(functionalit)o(y)g(of)g Fs(MPI)p 1193 45 V 15 w(SCA)l(TTER)h FD(b)o(y)f(allo)o(wing)g(a)f(v)m (arying)75 102 y(coun)o(t)14 b(of)g(data)g(to)g(b)q(e)h(sen)o(t)f(to)f (eac)o(h)i(pro)q(cess)f(\(since)i Fs(sendcounts)h FD(is)e(no)o(w)f(an)g (arra)o(y\),)f(and)h(also)h(allo)o(ws)75 158 y(more)21 b(\015exibili)q(t)o(y)j(as)d(to)g(where)h(the)g(data)f(is)i(tak)o(en)e (from)g(on)h(the)g(ro)q(ot)f(\(b)o(y)g(pro)o(viding)i(the)f(new)75 214 y(argumen)o(t,)14 b Fs(displs)p FD(\).)166 271 y(The)h(outcome)g (is)h(as)f(if)g(the)h(ro)q(ot)e(excuted)i FB(n)f FD(send)h(op)q (erations,)189 364 y FB(MPI)p 264 364 15 2 v 16 w(Send)p FD(\()p FB(sendbuf)9 b FD(+)h FB(displs)p FD([)p FB(i)p FD(])f FA(\002)h FB(extent)p FD(\()p FB(sendtype)p FD(\))p Fw(;)e FB(s)o(endcount)o(s)p FD([)p FB(i)p FD(])o Fw(;)g FB(s)o(endtype)o Fw(;)g FB(i)p Fw(;)f(:)o(::)p FD(\))o Fw(;)75 458 y FD(and)15 b(eac)o(h)h(pro)q(cess)f(executed)h(a)f(receiv) o(e,)189 551 y FB(MPI)p 264 551 V 16 w(Recv)p FD(\()p FB(recvbuf)p Fw(;)8 b FB(recvcount)p Fw(;)f FB(re)o(cvtype)p Fw(;)g FB(i)p Fw(;)g(::)o(:)p FD(\))o Fw(:)166 645 y FD(The)16 b(t)o(yp)q(e)g(signature)g(implied)i(b)o(y)e Fs(sendcount[i],)j(sendt)o(yp)q(e)g FD(at)c(the)h(ro)q(ot)f(m)o(ust)g (b)q(e)i(equal)g(to)e(the)75 701 y(t)o(yp)q(e)f(signature)h(implied)i (b)o(y)d Fs(recvcount,)i(recvt)o(yp)q(e)f FD(at)f(pro)q(cess)g FB(i)g FD(\(ho)o(w)o(ev)o(er,)f(the)i(t)o(yp)q(e)f(maps)h(ma)o(y)e(b)q (e)75 758 y(di\013eren)o(t\).)166 814 y(All)i(argumen)o(ts)f(to)f(the)h (function)h(are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fs(ro)q(ot)p FD(,)e(while)j(on)e(other)g(pro)q(cesses,)75 871 y(only)f(argumen)o(ts)f Fs(recvbuf,)i(recvcount,)g(recvt)o(yp)q(e,) g(ro)q(ot,)e(comm)7 b FD(are)12 b(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fs(ro)q(ot)75 927 y FD(and)j Fs(comm)10 b FD(m)o(ust)k(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e (all)h(pro)q(cesses.)166 984 y(F)l(or)g(b)q(oth)h(functions,)g(the)g (sp)q(eci\014cation)h(of)f(coun)o(t\(s\),)e(t)o(yp)q(e\(s\),)h (\(displacemen)o(ts\),)i(should)f(not)75 1040 y(cause)e(an)o(y)g(lo)q (cation)h(on)f(the)h(ro)q(ot)e(to)h(b)q(e)g(read)h(more)e(than)h(once.) 75 1162 y Fn(4.4.5)49 b(Examples)14 b(of)j(Usage)g(of)f(MPI)p 768 1162 V 18 w(SCA)l(TTER,)h(MPI)p 1115 1162 V 19 w(SCA)l(TTERV)75 1248 y Fs(Example)c(9)75 1333 y FD(The)19 b(rev)o(erse)g(of)g(Example)h (1.)31 b(Scatter)19 b(sets)g(of)f(100)h(in)o(ts)g(from)f(the)h(ro)q(ot) g(to)f(eac)o(h)h(pro)q(cess)h(in)g(the)75 1390 y(group.)g(See)15 b(\014gure)h(4.7.)170 1496 y FB(MPI_Comm)23 b(comm;)170 1552 y(int)h(gsize,*sendbuf;)170 1609 y(int)g(root,)f(rbuf[100];)170 1722 y(...)170 1834 y(/*)h(The)f(variable)g(comm)h(is)f(set)h (elsewhere)e(in)i(the)f(program)194 1891 y(*/)170 1947 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2004 y(sendbuf)f(=)h(\(int)f (*\)malloc\(gsize*100*sizeof\(int)o(\)\);)170 2060 y(MPI_Scatter\()g (sendbuf,)f(100,)i(MPI_INT,)f(rbuf,)g(100,)g(MPI_INT,)g(root,)g (comm\);)75 2230 y Fs(Example)13 b(10)75 2315 y FD(The)21 b(rev)o(erse)f(of)g(Example)i(3.)35 b(The)21 b(ro)q(ot)f(pro)q(cess)h (scatters)e(sets)i(of)f(100)f(in)o(ts)i(to)f(the)h(other)f(pro-)75 2372 y(cesses,)j(but)f(the)f(sets)h(of)f(100)g(are)g Fo(stride)g FD(in)o(ts)h(apart)f(in)h(the)g(sending)h(bu\013er.)38 b(Requires)23 b(use)f(of)75 2428 y Fs(MPI)p 160 2428 14 2 v 16 w(SCA)l(TTERV)p FD(.)16 b(Assume)f Fw(str)q(ide)e FA(\025)g FD(100.)19 b(See)d(\014gure)f(4.8.)170 2534 y FB(MPI_Comm)23 b(comm;)170 2591 y(int)h(gsize,*sendbuf;)170 2647 y(int)g(root,)f(rbuf[100],)g(i,)g(*displs,)g(*scounts;)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 102 109 102 108 bop 75 -100 a FD(102)685 b Fp(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6630800 0 0 33943388 13682606 startTexFig 458 4 a %%BeginDocument: mycoll-fig7.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 373.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 414 m gs 1 -1 scale (sendbuf) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 89 244 l gs col-1 s gr n 91.075 251.981 m 89.000 244.000 l 94.587 250.065 l gs 2 setlinejoin col-1 s gr n 189 299 m 189 244 l gs col-1 s gr n 187.000 252.000 m 189.000 244.000 l 191.000 252.000 l gs 2 setlinejoin col-1 s gr n 249 299 m 289 244 l gs col-1 s gr n 282.677 249.294 m 289.000 244.000 l 285.912 251.646 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 138 522 a FD(Figure)g(4.7:)i(The)e(ro)q(ot)e(pro)q(cess)i(scatters)e (sets)h(of)g(100)f FB(int)p FD(s)h(to)f(eac)o(h)i(pro)q(cess)f(in)h (the)f(group.)458 578 y 16577003 6796570 0 0 33943388 14011514 startTexFig 458 578 a %%BeginDocument: mycoll-fig8.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 378.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 99 419 m gs 1 -1 scale (sendbuf) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 84 244 l gs col-1 s gr n 86.608 251.823 m 84.000 244.000 l 89.982 249.676 l gs 2 setlinejoin col-1 s gr n 204 304 m 189 244 l gs col-1 s gr n 189.000 252.246 m 189.000 244.000 l 192.881 251.276 l gs 2 setlinejoin col-1 s gr n 284 304 m 289 244 l gs col-1 s gr n 286.343 251.806 m 289.000 244.000 l 290.329 252.138 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1107 a FD(Figure)g(4.8:)j(The)d(ro)q(ot)e(pro)q(cess)i(scatters)f (sets)g(of)g(100)f FB(int)p FD(s,)h(mo)o(ving)g(b)o(y)h FB(stride)e FD(in)o(ts)i(from)f(send)h(to)75 1163 y(send)h(in)g(the)f (scatter.)170 1284 y FB(...)170 1397 y(MPI_Comm_size\()22 b(comm,)i(&gsize\);)170 1453 y(sendbuf)f(=)h(\(int)f (*\)malloc\(gsize*stride*sizeof\()o(int\)\);)170 1510 y(displs)g(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o(nt\)\);)170 1566 y(scounts)f(=)h(\(int)f(*\)malloc\(gsize*sizeof\(int\)\);)170 1622 y(for)h(\(i=0;)f(i)g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1745 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 1860 y Fs(MPI)p 251 1860 14 2 v 16 w(ALLGA)l(THER)16 b FD(can)g(b)q(e)h(though)o(t)e(of)g(as)h Fs(MPI)p 1047 1860 V 15 w(GA)l(THER)p FD(,)h(but)f(where)g(all)h(pro)q(cesses)f (receiv)o(e)75 1916 y(the)d(result,)g(instead)g(of)f(just)g(the)h(ro)q (ot.)18 b(The)13 b FB(j)p FD(-th)f(blo)q(c)o(k)h(of)f(data)g(sen)o(t)h (from)e(eac)o(h)i(pro)q(cess)g(is)g(receiv)o(ed)75 1973 y(b)o(y)i(ev)o(ery)g(pro)q(cess)h(and)f(placed)h(in)g(the)g FB(j)p FD(-th)f(blo)q(c)o(k)h(of)f(the)g(bu\013er)g Fs(recvbuf)p FD(.)166 2058 y(The)g(t)o(yp)q(e)g(signature)g(asso)q(ciated)g(with)h Fs(sendcount,)h(sendt)o(yp)q(e)p FD(,)h(at)c(a)h(pro)q(cess)g(m)o(ust)f (b)q(e)i(equal)g(to)75 2115 y(the)f(t)o(yp)q(e)h(signature)f(asso)q (ciated)g(with)h Fs(recvcount,)g(recvt)o(yp)q(e)g FD(at)f(an)o(y)f (other)h(pro)q(cess.)166 2200 y(Th)o(us,)i(the)g(outcome)g(of)g(a)f (call)j(to)d Fs(MPI)p 881 2200 V 16 w(ALLGA)l(THER\(...\))25 b FD(is)17 b(as)g(if)h(all)g(pro)q(cesses)f(executed)h FB(n)75 2256 y FD(calls)e(to)189 2494 y FB(MPI)p 264 2494 15 2 v 16 w(GATHER)p FD(\()p FB(sendbuf)p Fw(;)8 b FB(sendcoun)o(t)p Fw(;)f FB(sendtype)o Fw(;)h FB(r)o(ecvbuf)p Fw(;)f FB(rec)o(vcount)p Fw(;)g FB(rec)o(vtype)p Fw(;)g FB(root)o Fw(;)h FB(c)o(omm)p FD(\))p Fw(;)75 2704 y FD(for)15 b FB(root)d FD(=)h FB(0)p Fw(;)8 b FA(\001)g(\001)g(\001)t Fw(;)g FB(n)h FA(\000)i FB(1)p FD(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 105 112 105 111 bop 75 -100 a Fp(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1054 b FD(105)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(ALLGA)l(THERV\()11 b(sendbuf,)i(sendcount,)h(sendt)o(yp)q(e,)f (recvbuf,)f(recvcounts,)g(displs,)g(recvt)o(yp)q(e,)g(comm)m(\))117 191 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 289 y(IN)171 b Fs(sendcount)383 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 388 y(IN)171 b Fs(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 487 y(OUT)124 b Fs(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 585 y(IN)171 b Fs(recvcounts)372 b Ft(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(con)o(taining)e(the)i(n)o(um-)905 642 y(b)q(er)15 b(of)e(elemen)o(ts)h(that)g(are)g(receiv)o(ed)h(from)e (eac)o(h)h(pro)q(cess)117 740 y(IN)171 b Fs(displs)469 b Ft(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Ft(sp)q(eci\014es)905 797 y(the)f(displacemen)o (t)e(\(relativ)o(e)g(to)h Fv(recvbuf)i Ft(at)d(whic)o(h)h(to)g(place) 905 853 y(the)j(incoming)c(data)j(from)e(pro)q(cess)k Fb(i)117 952 y Ft(IN)171 b Fs(recvt)o(yp)q(e)414 b Ft(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\)) 117 1051 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 1187 y FB(int)23 b(MPI)p 245 1187 15 2 v 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1187 V 16 w(Datatype)g(sendtype,)393 1243 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1300 y(MPI)p 468 1300 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1300 V 17 w(Comm)h(comm\))75 1398 y(MPI)p 150 1398 V 17 w (ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNTS,)f(DISPLS,)393 1454 y(RECVTYPE,)h(COMM,)g(IERROR\))170 1511 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1567 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h (RECVTYPE,)g(COMM,)170 1624 y(IERROR)166 1722 y Fs(MPI)p 251 1722 14 2 v 16 w(ALLGA)l(THERV)11 b FD(can)g(b)q(e)g(though)o(t)e (of)h(as)g Fs(MPI)p 1044 1722 V 16 w(GA)l(THERV)p FD(,)h(but)g(where)f (all)h(pro)q(cesses)g(receiv)o(e)75 1778 y(the)i(result,)g(instead)g (of)f(just)g(the)h(ro)q(ot.)18 b(The)13 b FB(j)p FD(-th)f(blo)q(c)o(k)h (of)f(data)g(sen)o(t)h(from)e(eac)o(h)i(pro)q(cess)g(is)g(receiv)o(ed) 75 1835 y(b)o(y)j(ev)o(ery)f(pro)q(cess)h(and)g(placed)h(in)f(the)g FB(j)p FD(-th)g(blo)q(c)o(k)g(of)f(the)h(bu\013er)g Fs(recvbuf)p FD(.)21 b(These)16 b(blo)q(c)o(ks)h(need)f(not)75 1891 y(all)g(b)q(e)g(the)f(same)g(size.)166 1960 y(The)g(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fs(sendcount,)j(sendt)o(yp)q(e)p FD(,)f(at)d(pro)q(cess)h FB(j)g FD(m)o(ust)f(b)q(e)i(equal)g(to)75 2016 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fs(recvcounts[j],)h(recvt)o(yp)q(e)f FD(at)e(an)o(y)h(other)g(pro)q (cess.)166 2084 y(The)g(outcome)g(is)h(as)f(if)g(all)h(pro)q(cesses)g (executed)g(calls)g(to)189 2237 y FB(MPI)p 264 2237 15 2 v 16 w(GATHERV)p FD(\()p FB(sendbuf)p Fw(;)8 b FB(sendcou)o(nt)p Fw(;)f FB(sendtyp)o(e)p Fw(;)g FB(recvbuf)o Fw(;)h FB(di)o(spls)p Fw(;)f FB(recvc)o(ounts)p Fw(;)g FB(recv)o(type)p Fw(;)189 2414 y FB(root)p Fw(;)h FB(comm)p FD(\))p Fw(;)75 2579 y FD(for)15 b FB(root)d FD(=)h FB(0)p Fw(;)8 b FA(\001)g(\001)g(\001)t Fw(;)g FB(n)h FA(\000)i FB(1)p FD(.)166 2647 y(F)l(or)i(b)q(oth)g Fs(MPI)p 436 2647 14 2 v 16 w(ALLGA)l(THER)i FD(and)e Fs(MPI)p 890 2647 V 16 w(ALLGA)l(THERV)p FD(,)h(all)h(argumen)o(ts)e (on)g(all)h(pro)q(cesses)g(are)75 2704 y(signi\014can)o(t.)21 b(The)15 b(argumen)o(t)g Fs(comm)9 b FD(m)o(ust)15 b(ha)o(v)o(e)f(iden) o(tical)j(v)m(alues)g(on)e(all)h(pro)q(cesses.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 106 113 106 112 bop 75 -100 a FD(106)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fn(4.4.7)49 b(All-to-All)19 b(Scatter/Gather)75 221 y Fs(MPI)p 160 221 14 2 v 16 w(ALL)l(TO)o(ALL\(sendbuf,)d(sendcount,)i(sendt)o(yp)q (e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(comm)m(\))117 320 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 438 y(IN)171 b Fs(sendcount)383 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 557 y(IN)171 b Fs(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 676 y(OUT)124 b Fs(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 795 y(IN)171 b Fs(recvcount)389 b Ft(n)o(um)o(b)q(er)17 b(of)f(elemen)o(ts) h(receiv)o(ed)i(from)c(an)o(y)i(pro)q(cess)i(\(in)o(te-)905 851 y(ger\))117 970 y(IN)171 b Fs(recvt)o(yp)q(e)414 b Ft(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 1089 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 1235 y FB(int)23 b(MPI)p 245 1235 15 2 v 17 w(Alltoall\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1235 V 17 w(Datatype)f(sendtype,)393 1292 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1292 V 17 w(Datatype)h(recvtype,)393 1348 y(MPI)p 468 1348 V 17 w(Comm)g(comm\))75 1456 y(MPI)p 150 1456 V 17 w(ALLTOALL\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,) h(RECVCOUNT,)g(RECVTYPE,)393 1513 y(COMM,)g(IERROR\))170 1569 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1626 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 1734 y Fs(MPI)p 251 1734 14 2 v 16 w(ALL)l(TO)o(ALL)18 b FD(is)h(an)g(extension)h(of)e Fs(MPI)p 962 1734 V 16 w(ALLGA)l(THER)i FD(to)e(the)h(case)g(where)g(eac)o(h)g(pro)q(cess)75 1791 y(sends)d(distinct)h(data)d(to)h(eac)o(h)h(of)f(the)g(receiv)o (ers.)22 b(The)15 b FB(j)p FD(-th)h(blo)q(c)o(k)g(sen)o(t)f(from)g(pro) q(cess)h FB(i)f FD(is)h(receiv)o(ed)75 1847 y(b)o(y)f(pro)q(cess)h FB(j)f FD(and)g(is)h(placed)g(in)g(the)f FB(i)p FD(-th)h(blo)q(c)o(k)f (of)g Fs(recvbuf)p FD(.)166 1925 y(The)g(t)o(yp)q(e)g(signature)g(asso) q(ciated)g(with)h Fs(sendcount,)h(sendt)o(yp)q(e)p FD(,)h(at)c(a)h(pro) q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 1982 y(the)f(t)o(yp)q(e)h (signature)f(asso)q(ciated)g(with)h Fs(recvcount,)g(recvt)o(yp)q(e)g FD(at)f(an)o(y)f(other)h(pro)q(cess.)166 2060 y(The)e(outcome)g(is)h (as)e(if)i(eac)o(h)f(pro)q(cess)g(executed)h(a)f(send)h(to)e(eac)o(h)h (pro)q(cess)h(\(itself)f(included\))j(with)75 2116 y(a)f(call)h(to)189 2319 y FB(MPI)p 264 2319 15 2 v 16 w(Send)p FD(\()p FB(sendbuf)9 b FD(+)h FB(i)g FA(\002)h FB(sendcount)d FA(\002)j FB(extent)p FD(\()p FB(sendtype)p FD(\))p Fw(;)d FB(s)o(endcoun)o(t)p Fw(;)f FB(sendtype)o Fw(;)h FB(i)o Fw(;)g(:)o(::)p FD(\))o Fw(;)75 2501 y FD(and)15 b(a)g(receiv)o(e)h(from)f(ev)o(ery)g(other)g (pro)q(cess)g(with)h(a)f(call)h(to)189 2704 y FB(MPI)p 264 2704 V 16 w(Recv)p FD(\()p FB(recvbuf)9 b FD(+)h FB(i)g FA(\002)h FB(recvcount)d FA(\002)j FB(extent)p FD(\()p FB(recvtype)p FD(\))p Fw(;)d FB(r)o(ecvcoun)o(t)p Fw(;)f FB(i)p Fw(;)h(:)o(::)o FD(\))p Fw(:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 107 114 107 113 bop 75 -100 a Fp(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1054 b FD(107)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V\(sendbuf,)13 b(sendcounts)q(,)h(sdispls,)g (sendt)o(yp)q(e,)g(recvbuf,)e(recvcounts,)i(rdispls,)e(recvt)o(yp)q(e,) 75 102 y(comm)m(\))117 180 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 259 y(IN)171 b Fs(sendcounts)366 b Ft(in)o(teger)20 b(arra)o(y)f(equal) h(to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 315 y(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(to)h(send)h(to)f(eac)o(h)g (pro)q(cessor)117 393 y(IN)171 b Fs(sdispls)452 b Ft(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(j)f Ft(sp)q(eci\014es)905 450 y(the)20 b(displacemen)o(t)e (\(relativ)o(e)h(to)f Fv(sendbuf)j Ft(from)c(whic)o(h)i(to)905 506 y(tak)o(e)14 b(the)h(outgoing)d(data)i(destined)h(for)e(pro)q(cess) j Fb(j)117 585 y Ft(IN)171 b Fs(sendt)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 663 y(OUT)124 b Fs(recvbuf)434 b Ft(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 741 y(IN)171 b Fs(recvcounts)372 b Ft(in)o(teger)20 b(arra)o(y)f(equal)h (to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 798 y(maxim)n(um)d(n)o(um)o(b)q(er)j(of)f(elemen)o(ts)i(that)g(can)f(b)q(e) i(receiv)o(ed)905 854 y(from)12 b(eac)o(h)j(pro)q(cessor)117 933 y(IN)171 b Fs(rdispls)453 b Ft(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Ft(sp)q(eci\014es) 905 989 y(the)f(displacemen)o(t)e(\(relativ)o(e)g(to)h Fv(recvbuf)i Ft(at)d(whic)o(h)h(to)g(place)905 1046 y(the)j(incoming)c (data)j(from)e(pro)q(cess)k Fb(i)117 1124 y Ft(IN)171 b Fs(recvt)o(yp)q(e)414 b Ft(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j (bu\013er)f(elemen)o(ts)e(\(handle\))117 1202 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 1328 y FB(int)23 b(MPI)p 245 1328 15 2 v 17 w(Alltoallv\(void*)f (sendbuf,)h(int)g(*sendcounts,)f(int)i(*sdispls,)393 1385 y(MPI)p 468 1385 V 17 w(Datatype)f(sendtype,)f(void*)i(recvbuf,)e (int)i(*recvcounts,)393 1441 y(int)g(*rdispls,)e(MPI)p 802 1441 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1441 V 17 w(Comm)i(comm\))75 1529 y(MPI)p 150 1529 V 17 w (ALLTOALLV\(SENDBUF,)d(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h(RECVBUF,)g (RECVCOUNTS,)393 1586 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g(IERROR\))170 1642 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1699 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 1755 y(RECVTYPE,)h(COMM,)g (IERROR)166 1843 y Fs(MPI)p 251 1843 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V)12 b FD(adds)g(\015exibili)q(t)o(y)j(to)c Fs(MPI)p 959 1843 V 16 w(ALL)l(TO)o(ALL)h FD(in)h(that)f(the)h(lo)q(cation)g(of) f(data)g(for)g(the)75 1900 y(send)j(is)f(sp)q(eci\014ed)j(b)o(y)d Fs(sdispls)i FD(and)e(the)g(lo)q(cation)h(of)f(the)g(placemen)o(t)h(of) f(the)g(data)g(on)g(the)g(receiv)o(e)h(side)75 1956 y(is)h(sp)q (eci\014ed)h(b)o(y)e Fs(rdispls)p FD(.)166 2014 y(The)i FB(j)p FD(-th)g(blo)q(c)o(k)h(sen)o(t)e(from)g(pro)q(cess)i FB(i)e FD(is)i(receiv)o(ed)g(b)o(y)f(pro)q(cess)g FB(j)g FD(and)g(is)g(placed)h(in)g(the)f FB(i)p FD(-th)75 2071 y(blo)q(c)o(k)f(of)f Fs(recvbuf)p FD(.)20 b(These)c(blo)q(c)o(ks)g (need)g(not)f(all)h(ha)o(v)o(e)f(the)g(same)g(size.)166 2129 y(The)i(t)o(yp)q(e)h(signature)f(asso)q(ciated)h(with)f Fs(sendcount[)q(j],)j(sendt)o(yp)q(e)g FD(at)d(pro)q(cess)g FB(i)g FD(m)o(ust)g(b)q(e)h(equal)75 2185 y(to)d(the)g(t)o(yp)q(e)g (signature)g(asso)q(ciated)h(with)f Fs(recvcount[i],)i(recvt)o(yp)q(e)f FD(at)e(pro)q(cess)i FB(j)p FD(.)166 2243 y(The)f(outcome)g(is)h(as)f (if)g(eac)o(h)h(pro)q(cess)f(sen)o(t)g(a)g(message)g(to)f(eac)o(h)h (other)g(pro)q(cess)h(with,)189 2345 y FB(MPI)p 264 2345 15 2 v 16 w(Send)p FD(\()p FB(sendbuf)9 b FD(+)h FB(displs)p FD([)p FB(i)p FD(])p Fw(;)e FB(sendcount)o(s)p FD([)p FB(i)p FD(])o Fw(;)g FB(s)o(endtype)o Fw(;)g FB(i)p Fw(;)f(:)o(::)p FD(\))o Fw(;)75 2445 y FD(and)15 b(receiv)o(ed)i(a)e(message)f(from)h (ev)o(ery)g(other)g(pro)q(cess)g(with)h(a)f(call)h(to)189 2547 y FB(MPI)p 264 2547 V 16 w(Recv)p FD(\()p FB(recvbuf)9 b FD(+)h FB(displs)p FD([)p FB(i)p FD(])p Fw(;)e FB(recvcount)o(s)p FD([)p FB(i)p FD(])o Fw(;)g FB(r)o(ecvtype)o Fw(;)g FB(i)p Fw(;)f(:)o(::)p FD(\))o Fw(:)166 2647 y FD(F)l(or)21 b(b)q(oth)g Fs(MPI)p 452 2647 14 2 v 16 w(ALL)l(TO)o(ALL)g FD(and)h Fs(MPI)p 885 2647 V 16 w(ALL)l(TO)o(ALL)-5 b(V)p FD(,)20 b(all)j(argumen)o(ts)d(on)i(all)g(pro)q(cesses)g(are)75 2704 y(signi\014can)o(t.)f(The)15 b(argumen)o(t)g Fs(comm)9 b FD(m)o(ust)15 b(ha)o(v)o(e)f(iden)o(tical)j(v)m(alues)g(on)e(all)h (pro)q(cesses.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 108 115 108 114 bop 75 -100 a FD(108)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Fl(Discussion:)32 b Ft(The)13 b(de\014nition)f(of)g(the)h Fv(MPI)p 892 45 13 2 v 14 w(xxxV)g Ft(op)q(erations)g(giv)o(es)f(as)g(m)o(uc)o(h)g (\015exibilit)o(y)f(as)h(one)h(w)o(ould)75 102 y(ac)o(hiev)o(e)e(b)o(y) f(sp)q(ecifying)g Fb(n)g Ft(indep)q(enden)o(t)i(p)q(oin)o(t)e(to)g(p)q (oin)o(t)g(comm)o(unications,)e(with)i(one)h(exception:)17 b(all)9 b(messages)75 158 y(use)15 b(the)f(same)f(datat)o(yp)q(e,)h (and)g(messages)g(are)g(scattered)i(from)c(\(or)i(gathered)h(to\))f (sequen)o(tial)f(storage.)166 393 y Fl(Implemen)o(tati)o(on)k(note:)54 b Ft(Although)18 b(the)h(discussion)g(of)e(collectiv)o(e)i(comm)o (unicati)o(on)d(in)h(terms)i(of)75 449 y(p)q(oin)o(t)e(to)g(p)q(oin)o (t)f(op)q(eration)h(implies)f(that)h(eac)o(h)h(message)e(is)i (transfered)g(directly)g(from)d(sender)k(to)e(receiv)o(er,)75 505 y(implemen)o(tatio)o(ns)11 b(ma)o(y)g(use)j(a)f(tree)h(comm)o (unication)c(pattern,)k(where)g(messages)f(are)h(forw)o(arded)f(b)o(y)g (in)o(terme-)75 562 y(diate)i(no)q(des)h(where)g(they)f(are)h(split)e (\(for)h(scatter\))i(or)d(concatenated)j(\(for)e(gather\),)g(if)f(this) h(is)g(more)f(e\016cien)o(t.)75 917 y Fq(4.5)59 b(Global)20 b(Compute)e(Op)r(erations)75 1043 y FD(The)13 b(functions)g(in)h(this)f (section)h(p)q(erform)e(one)h(of)f(the)h(follo)o(wing)g(op)q(erations)g (across)f(all)i(the)f(mem)o(b)q(ers)75 1100 y(of)i(a)g(group:)189 1245 y(global)g(max)g(on)g(in)o(teger)h(and)f(\015oating)g(p)q(oin)o(t) h(data)e(t)o(yp)q(es)189 1389 y(global)h(min)h(on)g(in)o(teger)f(and)g (\015oating)g(p)q(oin)o(t)h(data)f(t)o(yp)q(es)189 1534 y(global)g(sum)h(on)f(in)o(teger)g(and)h(\015oating)f(p)q(oin)o(t)g (data)g(t)o(yp)q(es)189 1679 y(global)g(pro)q(duct)h(on)f(in)o(teger)h (and)f(\015oating)g(p)q(oin)o(t)h(data)e(t)o(yp)q(es)189 1824 y(global)h(AND)h(on)f(logical)h(and)f(in)o(teger)h(data)e(t)o(yp)q (es)189 1968 y(global)h(OR)h(on)g(logical)g(and)f(in)o(teger)h(data)e (t)o(yp)q(es)189 2113 y(global)h(X)o(OR)h(on)f(logical)i(and)e(in)o (teger)h(data)e(t)o(yp)q(es)189 2258 y(rank)h(of)f(pro)q(cess)i(with)f (maxim)o(um)g(v)m(alue)189 2403 y(rank)g(of)f(pro)q(cess)i(with)f (minim)o(um)i(v)m(alue)189 2547 y(user)e(de\014ned)i(\(asso)q(ciativ)o (e\))d(op)q(eration)189 2692 y(user)h(de\014ned)i(\(asso)q(ciativ)o(e)e (and)g(comm)o(utativ)o(e\))f(op)q(eration)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 109 116 109 115 bop 75 -100 a Fp(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 860 b FD(109)75 45 y Fn(4.5.1)49 b(Reduce)75 179 y Fs(MPI)p 160 179 14 2 v 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat) o(yp)q(e,)h(op,)e(ro)q(ot,)f(comm)m(\))117 257 y Ft(IN)171 b Fs(sendbuf)428 b Ft(address)15 b(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 333 y(OUT)124 b Fs(recvbuf)434 b Ft(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i(signi\014can)o(t)d(only)g (at)905 390 y(ro)q(ot\))117 466 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 542 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)12 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(send)h(bu\013er)h (\(basic)f(t)o(yp)q(es)h(only\))905 598 y(\(handle\))117 674 y(IN)171 b Fs(op)525 b Ft(op)q(eration)14 b(\(state\))117 751 y(IN)171 b Fs(ro)q(ot)492 b Ft(rank)14 b(of)f(ro)q(ot)h(pro)q(cess) i(\(in)o(teger\))117 827 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 952 y FB(int)23 b(MPI)p 245 952 15 2 v 17 w(Reduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)h(count,)393 1008 y(MPI)p 468 1008 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1008 V 17 w(Op)i(op,)f(int)h(root,)f(MPI)p 1505 1008 V 17 w(Comm)g(comm\))75 1095 y(MPI)p 150 1095 V 17 w(REDUCE\(SENDBUF,)f(RECVBUF,)g(COUNT,)h(DATATYPE,)g(OP,)g(ROOT,)h (COMM,)f(IERROR\))170 1152 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1208 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR)166 1295 y FD(Com)o(bines)c(the)f(v)m(alues)i(pro)o(vided)f(in)g(the)g (send)g(bu\013er)f(of)g(eac)o(h)g(pro)q(cess)h(in)g(the)g(group,)f (using)75 1352 y(the)d(op)q(eration)30 b Fs(op)p FD(,)15 b(and)g(returns)g(the)g(com)o(bined)h(v)m(alue)h(in)f(the)f(receiv)o(e) h(bu\013er)f(of)f(the)h(pro)q(cess)h(with)75 1408 y(rank)43 b Fs(ro)q(ot)p FD(.)c(The)22 b(routine)g(is)g(called)h(b)o(y)f(all)g (group)g(mem)o(b)q(ers)f(using)i(the)e(same)h(argumen)o(ts)f(for)75 1464 y Fs(count,)f(datat)o(yp)q(e,)g(op,)f(ro)q(ot)f FD(and)37 b Fs(comm)n FD(.)26 b(Eac)o(h)18 b(pro)q(cess)h(can)f(pro)o (vide)h(one)g(v)m(alue,)h(or)e(a)g(sequence)75 1521 y(of)g(v)m(alues,)h (in)g(whic)o(h)g(case)f(the)g(com)o(bine)h(op)q(eration)f(is)g (executed)h(elemen)o(t-wise)h(on)e(eac)o(h)g(en)o(try)f(of)75 1577 y(the)g(sequence.)26 b(F)l(or)17 b(example,)h(if)f(the)g(op)q (eration)g(is)h Fv(MPI)p 1096 1577 13 2 v 14 w(MAX)f FD(and)g(the)g(send)g(bu\013er)g(con)o(tains)g(t)o(w)o(o)75 1634 y(\015oating)d(p)q(oin)o(t)h(n)o(um)o(b)q(ers,)f(then)h (recvbuf\(1\))f(=)g(global)h(max\(sendbuf\(1\)\))e(and)i(recvbuf\(2\))f (=)g(global)75 1690 y(max\(sendbuf\(2\)\).)23 b(All)18 b(send)f(bu\013ers)f(should)i(de\014ne)g(sequences)f(of)f(equal)i (length)f(of)f(en)o(tries)h(all)g(of)75 1747 y(the)f(same)g(data)f(t)o (yp)q(e,)h(where)g(the)g(t)o(yp)q(e)g(is)g(a)g Fx(basic)h FD(MPI)f(datat)o(yp)q(e)f(and)h(one)g(of)g(those)f(allo)o(w)o(ed)i(for) 75 1803 y(op)q(erands)g(of)33 b Fs(op)16 b FD(\(see)h(b)q(elo)o(w\).)24 b(F)l(or)16 b(all)i(op)q(erations)f(the)f(n)o(um)o(b)q(er)h(of)f (elemen)o(ts)i(in)f(the)g(send)g(bu\013er)75 1860 y(are)e(the)g(same)f (as)h(for)f(the)h(receiv)o(e)h(bu\013ers.)k(F)l(or)14 b(all)i(op)q(erations,)f(the)g(t)o(yp)q(e)g(of)g(elemen)o(ts)g(in)h (the)f(send)75 1916 y(bu\013er)g(are)g(the)g(same)g(as)g(for)g(the)g (receiv)o(e)h(bu\013ers.)166 1973 y(The)f(op)q(eration)g(de\014ned)h(b) o(y)30 b Fs(op)15 b FD(is)g(asso)q(ciativ)o(e)g(and)g(comm)o(utativ)o (e,)f(and)h(the)g(implemen)o(tation)75 2029 y(can)20 b(tak)o(e)f(adv)m(an)o(tage)g(of)g(asso)q(ciativit)o(y)h(and)f(comm)o (utativit)o(y)g(in)i(order)e(to)g(c)o(hange)g(order)h(of)f(ev)m(al-)75 2086 y(uation.)30 b(This)19 b(ma)o(y)e(c)o(hange)i(the)f(result)h(of)f (the)h(reduction,)g(for)f(op)q(erations)h(that)e(are)h(not)g(strictly) 96 b Fm(CHANGE)75 2142 y FD(asso)q(ciativ)o(e)13 b(and)h(comm)o(utativ) o(e,)e(suc)o(h)h(as)g(\015oating)g(p)q(oin)o(t)h(addition.)20 b Fs(MPI)p 1382 2142 14 2 v 16 w(REDUCE)14 b FD(should)g(b)q(e)f(used) 75 2199 y(only)j(when)f(suc)o(h)h(c)o(hanges)f(are)g(acceptable.)166 2338 y Fl(Implemen)o(tati)o(on)f(note:)41 b Ft(It)15 b(is)h(strongly)e(recommended)h(that)g Fv(MPI)p 1344 2338 13 2 v 14 w(REDUCE)f Ft(b)q(e)i(implemen)o(ted)d(so)75 2395 y(that)e(the)h(same)f(result)h(b)q(e)g(obtained)f(whenev)o(er)i (the)f(function)f(is)g(applied)g(on)g(the)h(same)f(argumen)o(ts,)f(app) q(earing)75 2451 y(in)16 b(the)h(same)e(order.)27 b(Note)17 b(that)f(this)g(ma)o(y)f(prev)o(en)o(t)i(optimizations)d(that)j(tak)o (e)f(adv)n(an)o(tage)g(of)f(the)i(ph)o(ysical)75 2508 y(lo)q(cation)c(of)g(pro)q(cessors.)166 2648 y FD(W)l(e)i(list)h(b)q (elo)o(w)g(the)f(supp)q(orted)h(options)f(for)30 b Fs(op)p FD(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 110 117 110 116 bop 75 -100 a FD(110)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)117 45 y Fv(MPI)p 194 45 13 2 v 14 w(MAX)649 b FD(maxim)o(um)117 98 y Fv(MPI)p 194 98 V 14 w(MIN)664 b FD(minim)o(um)117 151 y Fv(MPI)p 194 151 V 14 w(SUM)653 b FD(sum)117 204 y Fv(MPI)p 194 204 V 14 w(PROD)626 b FD(pro)q(duct)117 257 y Fv(MPI)p 194 257 V 14 w(LAND)632 b FD(logical)16 b(and)117 310 y Fv(MPI)p 194 310 V 14 w(BAND)626 b FD(bit-wise)16 b(and)117 364 y Fv(MPI)p 194 364 V 14 w(LOR)661 b FD(logical)16 b(or)117 417 y Fv(MPI)p 194 417 V 14 w(BOR)655 b FD(bit-wise)16 b(or)117 470 y Fv(MPI)p 194 470 V 14 w(LX)o(OR)634 b FD(logical)16 b(xor)117 523 y Fv(MPI)p 194 523 V 14 w(BX)o(OR)628 b FD(bit-wise)16 b(xor)117 576 y Fv(MPI)p 194 576 V 14 w(MAXLOC)569 b FD(maxim)o(um)15 b(v)m(alue)h(and)g(rank)f(of)f(pro)q (cess)i(with)f(it)117 629 y Fv(MPI)p 194 629 V 14 w(MINLOC)584 b FD(minim)o(um)16 b(v)m(alue)g(and)g(rank)f(of)g(pro)q(cess)g(with)h (it)166 712 y(The)25 b Fv(MPI)p 346 712 V 14 w(MINLOC)13 b FD(\()e Fv(MPI)p 634 712 V 14 w(MAXLOC)p FD(\))h(op)q(erations)h (return)g(b)q(oth)g(minim)o(um)h(\(maxim)o(um\))e(v)m(alues)75 768 y(and)17 b(the)h(ranks)f(of)g(pro)q(cesses)g(con)o(taining)h(those) f(v)m(alues.)28 b(The)17 b(p)q(oten)o(tially)i(mixed-t)o(yp)q(e)f (nature)f(of)75 825 y(the)g(output)g(bu\013er)g(is)g(a)g(concern,)g(so) g(MPI)g(treats)f(the)h(bu\013ers)g(uniformly)h(and)f(co)q(erces)g(the)g (ranks)75 881 y(to)j(the)h(same)f(t)o(yp)q(e)h(as)f(the)h(v)m(alues.)37 b(When)j Fv(MPI)p 995 881 V 14 w(MINLOC)21 b FD(or)39 b Fv(MPI)p 1342 881 V 14 w(MAXLOC)20 b FD(are)g(in)o(v)o(ok)o(ed,)i (the)75 938 y(input)e(bu\013er)f(should)i(con)o(tain)e Fw(m)g FD(elemen)o(ts)h(of)f(a)g(data)f(t)o(yp)q(e)i(to)e(whic)o(h)i (the)f(op)q(eration)37 b Fv(MPI)p 1785 938 V 15 w(MIN)75 994 y FD(or)c Fv(MPI)p 226 994 V 14 w(MAX)17 b FD(can)h(b)q(e)g (applied,)h(follo)o(w)o(ed)f(b)o(y)g(space)f(for)g(another)g Fw(m)g FD(elemen)o(ts)i(of)e(the)g(same)g(t)o(yp)q(e.)75 1051 y(In)o(ternally)l(,)h(the)e(function)h(will)h(co)q(erce)f(the)g (rank)f(of)g(the)g(calling)i(pro)q(cess)f(\(an)f(in)o(teger)g(t)o(yp)q (e\))g(to)g(the)75 1107 y(t)o(yp)q(e)g(of)g(the)g(data)f(v)m(alues,)i (and)f(carry)g(these)g(v)m(alues)h(along)f(during)h(the)f(reduction.)23 b(The)17 b(op)q(eration)75 1163 y(returns)f(at)f(the)i(ro)q(ot)e(the)h Fw(m)g FD(minim)o(um)h(\(or)e(maxim)o(um\))h(v)m(alues,)h(follo)o(w)o (ed)f(b)o(y)g(the)h Fw(m)e FD(ranks)h(of)g(the)75 1220 y(pro)q(cesses)k(con)o(taining)g(these)f(v)m(alues.)34 b(T)l(o)19 b(reco)o(v)o(er)f(the)i(ranks)f(of)g(the)g(pro)q(cesses)h (as)f(in)o(tegers,)h(the)75 1276 y(second)c(set)f(of)f(v)m(alues)j(can) e(no)o(w)g(b)q(e)h(copied)g(out)f(of)g(the)g(bu\013er)g(and)h(co)q (erced)g(to)e(in)o(teger)h(v)m(alues.)166 1416 y Fl(Discussion:)54 b Ft(This)18 b(solution)g(will)g(run)h(in)o(to)f(trouble)h(if)f(the)h (v)n(alues)g(are)g(of)f(t)o(yp)q(e)i Fv(MPI)p 1667 1416 V 14 w(CHAR)e Ft(and)75 1472 y(the)e(group)f(con)o(tains)f(more)h(than) g(128)f(pro)q(cesses.)24 b(It)15 b(forces)h(t)o(w)o(o)e(sup)q (er\015uous)j(t)o(yp)q(e)f(con)o(v)o(ersions:)21 b(e.g.,)14 b(MPI)75 1528 y(con)o(v)o(erts)g(ranks)f(to)g(\015oating)f(p)q(oin)o(t) h(and,)f(next,)i(the)f(user)h(con)o(v)o(erts)h(bac)o(k)e(to)f(in)o (teger.)19 b(Besides,)14 b(it's)f(ugly)m(.)j(An)o(y)75 1585 y(b)q(etter)f(suggestions?)166 1724 y FD(The)g(op)q(eration)h (that)e(de\014nes)30 b Fv(MPI)p 801 1724 V 15 w(MAXLOC)14 b FD(is)189 1779 y Fe( )242 1823 y Fw(u)247 1879 y(i)289 1779 y Fe(!)332 1851 y FA(\016)365 1779 y Fe( )418 1823 y Fw(v)420 1879 y(j)463 1779 y Fe(!)508 1851 y FD(=)556 1779 y Fe( )610 1823 y Fw(w)614 1879 y(k)664 1779 y Fe(!)75 1979 y FD(where)189 2066 y Fw(w)f FD(=)g(max\()p Fw(u;)8 b(v)r FD(\))75 2153 y(and)189 2287 y Fw(k)13 b FD(=)274 2188 y Fe(8)274 2226 y(>)274 2238 y(<)274 2313 y(>)274 2325 y(:)332 2230 y Fw(i)194 b FD(if)15 b Fw(u)e(>)g(v)332 2287 y FD(min)q(\()p Fw(i;)8 b(j)s FD(\))39 b(if)15 b Fw(u)e FD(=)g Fw(v)332 2343 y(j)191 b FD(if)15 b Fw(u)e(<)g(v)166 2429 y FD(Note)21 b(that)g(ties)h(are)f(resolv)o(ed)h(in)h(fa)o(v)o(or) d(of)h(the)h(pro)q(cess)g(with)g(lo)o(w)o(er)f(rank)g(and)h(hence)h (this)75 2486 y(op)q(eration)15 b(is)h(asso)q(ciativ)o(e)g(and)f(comm)o (utativ)o(e.)166 2542 y Fv(MPI)p 243 2542 V 14 w(MINLOC)g FD(is)h(de\014ned)g(similarly:)189 2597 y Fe( )242 2641 y Fw(u)247 2697 y(i)289 2597 y Fe(!)332 2669 y FA(\016)365 2597 y Fe( )418 2641 y Fw(v)420 2697 y(j)463 2597 y Fe(!)508 2669 y FD(=)556 2597 y Fe( )610 2641 y Fw(w)614 2697 y(k)664 2597 y Fe(!)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 111 118 111 117 bop 75 -100 a Fp(4.5.)29 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 865 b FD(111)75 45 y(where)189 138 y Fw(w)13 b FD(=)g(min)q(\()p Fw(u;)8 b(v)r FD(\))75 231 y(and)189 370 y Fw(k)13 b FD(=)274 271 y Fe(8)274 309 y(>)274 321 y(<)274 396 y(>)274 408 y(:)332 313 y Fw(i)194 b FD(if)15 b Fw(u)e(<)g(v)332 370 y FD(min)q(\()p Fw(i;)8 b(j)s FD(\))39 b(if)15 b Fw(u)e FD(=)g Fw(v)332 426 y(j)191 b FD(if)15 b Fw(u)e(>)g(v)75 547 y Fs(Op)q(eration)j(/)f(T)l(yp)q(e)i(Com)o(patibilit)o(y)75 633 y FD(All)h(of)d(the)i(options)f(for)g Fs(op)g FD(in)h Fs(MPI)p 715 633 14 2 v 16 w(REDUCE)g FD(do)f(not)g(apply)h(for)f(eac)o (h)g(of)g(the)g(p)q(ossible)i(MPI)e(basic)75 690 y(datat)o(yp)q(es.)j (W)l(e)14 b(en)o(umerate)f(the)h(allo)o(w)o(ed)g(com)o(binations)h (here.)20 b(First,)13 b(de\014ne)i(groups)e(of)h(MPI)f(basic)75 746 y(datat)o(yp)q(es)i(in)h(the)f(follo)o(wing)h(w)o(a)o(y)l(.)75 851 y Fx(C)h(in)o(teger:)22 b FB(MPI)p 400 851 15 2 v 17 w(CHAR,)h(MPI)p 632 851 V 17 w(INT,)g(MPI)p 840 851 V 17 w(LONG,)g(MPI)p 1072 851 V 17 w(SHORT,)189 907 y(MPI)p 264 907 V 16 w(UNSIGNED)p 472 907 V 16 w(SHORT,)h(MPI)p 728 907 V 16 w(UNSIGNED,)f(MPI)p 1055 907 V 17 w(UNSIGNED)p 1264 907 V 16 w(LONG)75 1001 y Fx(F)l(ortran)17 b(in)o(teger:)22 b FB(MPI)p 535 1001 V 17 w(INTEGER)75 1094 y Fx(Floating)d(p)q(oin)o (t:)24 b FB(MPI)p 513 1094 V 16 w(FLOAT,)g(MPI)p 769 1094 V 16 w(DOUBLE,)f(MPI)p 1048 1094 V 17 w(REAL,)189 1151 y(MPI)p 264 1151 V 16 w(DOUBLE)p 424 1151 V 17 w(PRECISION)75 1244 y Fx(Logical:)i FB(MPI)p 349 1244 V 16 w(LOGICAL)75 1337 y Fx(Complex:)d FB(MPI)p 390 1337 V 17 w(COMPLEX)75 1431 y Fx(Byte:)g FB(MPI)p 296 1431 V 17 w(BYTE)166 1536 y FD(No)o(w,)14 b(the)h(v)m(alid)i(datat)o(yp)q(es)e(for)f(eac)o(h)i (option)f(is)h(sp)q(eci\014ed)h(b)q(elo)o(w.)117 1696 y Fv(MPI)p 194 1696 13 2 v 14 w(MAX,)c(MPI)p 400 1696 V 14 w(MIN,)h(MPI)p 592 1696 V 14 w(MAXLOC,)f(MPI)p 878 1696 V 14 w(MINLOC)41 b FD(C)10 b(in)o(teger,)h(F)l(ortran)e(in)o (teger,)i(Floating)f(p)q(oin)o(t)117 1750 y Fv(MPI)p 194 1750 V 14 w(SUM,)j(MPI)p 396 1750 V 14 w(PROD)424 b FD(C)10 b(in)o(teger,)h(F)l(ortran)e(in)o(teger,)i(Floating)f(p)q (oin)o(t,)h(Com-)905 1807 y(plex)117 1861 y Fv(MPI)p 194 1861 V 14 w(LAND,)j(MPI)p 418 1861 V 14 w(LOR,)f(MPI)p 612 1861 V 15 w(LX)o(OR)215 b FD(C)15 b(in)o(teger,)g(Logical)117 1915 y Fv(MPI)p 194 1915 V 14 w(BAND,)e(MPI)p 423 1915 V 14 w(BOR,)g(MPI)p 623 1915 V 14 w(BX)o(OR)199 b FD(C)15 b(in)o(teger,)g(Byte)75 2061 y Fs(Example)e(of)i(Reduce)75 2147 y FD(Eac)o(h)h(pro)q(cess)h(has)g(an)f(arra)o(y)g(of)g(30)g FB(double)p FD(s,)g(in)h(C.)f(F)l(or)g(eac)o(h)h(of)f(the)h(30)f(lo)q (cations,)h(compute)g(the)75 2204 y(v)m(alue)f(and)g(rank)f(of)g(the)g (pro)q(cess)g(con)o(taining)h(the)f(largest)g(v)m(alue.)170 2308 y FB(...)170 2365 y(/*)24 b(each)f(process)g(has)h(an)f(array)g (of)h(30)g(double:)f(a[])194 2421 y(*/)170 2478 y(double)g(a[30];)170 2534 y(double)g(in[60],out[60];)170 2591 y(int)h(i,outranks[30];)170 2704 y(for)g(\(i=0;)f(i<30;)g(++i\))h({)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 112 119 112 118 bop 75 -100 a FD(112)685 b Fp(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)266 45 y FB(in[i])23 b(=)h(a[i];)170 102 y(})170 158 y(MPI_Reduce\()f(in,)g(out,)h(30,)f (MPI_DOUBLE,)f(MPI_MAXLOC,)h(root,)g(comm)g(\);)170 214 y(/*)h(At)g(this)f(point,)g(the)g(answer)g(resides)g(on)h(process)f (root)194 271 y(*/)170 327 y(if)h(\(myrank)f(==)h(root\))f({)266 384 y(/*)g(read)h(ranks)f(out)g(and)h(coerce)f(back)g(to)h(ints)290 440 y(*/)266 497 y(for)f(\(i=0;)h(i<30;)f(++i\))g({)361 553 y(outranks[i])g(=)g(out[30+i];)266 610 y(})170 666 y(})75 790 y Fn(4.5.2)49 b(User-Reduce)75 923 y Fs(MPI)p 160 923 14 2 v 16 w(USER)p 288 923 V 17 w(REDUCE\()16 b(sendbuf,)h(recvbuf,)f(count,)g(datat)o(yp)q(e,)g(function,)g(ro)q (ot,)f(comm)m(\))117 1001 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1077 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)19 b(address)h(of)e (receiv)o(e)i(bu\013er)f(\(c)o(hoice,)h(signi\014can)o(t)905 1133 y(only)13 b(at)h(ro)q(ot\))117 1209 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g (\(in)o(teger\))117 1285 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 1360 y(IN)171 b Fs(function)418 b Ft(user)15 b(de\014ned)g(function)f(\(function\))117 1436 y(IN)171 b Fs(ro)q(ot)492 b Ft(rank)14 b(of)f(ro)q(ot)h(pro)q(cess)i(\(in)o (teger\))117 1512 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))-245 1590 y Fm(CHANGE)75 1637 y FB(int)23 b(MPI)p 245 1637 15 2 v 17 w(User)p 358 1637 V 17 w(reduce\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 1693 y(MPI)p 468 1693 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1693 V 17 w(Uop)i(function,)e(int)i(root,)393 1750 y(MPI)p 468 1750 V 17 w(Comm)f(comm\))75 1836 y(MPI)p 150 1836 V 17 w(USER)p 263 1836 V 16 w(REDUCE\(SENDBUF,)f(RECVBUF,)h (COUNT,)g(DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM,)393 1893 y(IERROR\))170 1949 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2006 y(EXTERNAL)g(FUNCTION)170 2062 y(INTEGER)g(COUNT,)g(DATATYPE,)g (ROOT,)g(COMM,)g(IERROR)166 2149 y FD(Similar)17 b(to)d(the)i(reduce)g (op)q(eration)f(function)h(ab)q(o)o(v)o(e)f(except)h(that)e(a)h(user)h (supplied)h(function)f(is)75 2205 y(used.)k Fs(function)d FD(is)f(a)f(function)h(with)f(three)h(argumen)o(ts.)j(The)c(C)g(t)o(yp) q(e)g(for)g(suc)o(h)g(function)h(is)75 2313 y FB(typedef)23 b(void)g(MPI_Uop\()g(*void,)g(*void,)g(*int\);)166 2421 y FD(If)11 b(the)g(function)h(is)f(passed)g(actual)g(argumen)o(ts)f Fs(\(void)g(*\)invec,)i(\(void)e(*\)inoutvec,)i(len)g FD(then)22 b Fs(*invec)75 2478 y FD(and)31 b Fs(*inoutvec)17 b FD(should)g(b)q(e)f(arra)o(ys)e(with)31 b Fs(*len)16 b FD(v)m(alues.)23 b(The)15 b(t)o(yp)q(e)h(of)f(the)h(elemen)o(ts)g(of) 31 b Fs(*invec)16 b FD(and)75 2534 y(of)27 b Fs(*inoutvec)15 b FD(matc)o(h)f(the)g(t)o(yp)q(e)g(of)f(the)h(elemen)o(ts)h(of)e(the)h (send)h(bu\013ers)e(and)h(the)g(receiv)o(e)h(bu\013er.)20 b(The)75 2591 y(function)f(computes)f(elemen)o(t-wise)h(a)f(comm)o (utativ)o(e)f(and)h(asso)q(ciativ)o(e)g(op)q(eration)g(on)g(eac)o(h)g (pair)g(of)75 2647 y(en)o(tries)i(and)f(returns)h(the)f(result)h(in)39 b Fs(*inoutvec)p FD(.)34 b(A)19 b(pseudo-co)q(de)i(for)38 b Fs(function)21 b FD(is)f(giv)o(en)g(b)q(elo)o(w,)75 2704 y(where)15 b FB(op)g FD(is)h(the)f(comm)o(utativ)o(e)g(and)g(asso) q(ciativ)o(e)h(op)q(eration)f(de\014ned)i(b)o(y)30 b Fs(function)p FD(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 113 120 113 119 bop 75 -100 a Fp(4.5.)29 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 865 b FD(113)361 45 y FB(for\(i=0;)23 b(i)h(<)g(*len;)f(i++\))g({)552 102 y(inoutvec[i])g(op=)g(invec[i])361 158 y(})166 265 y FD(No)15 b(MPI)g(functions)h(ma)o(y)e(b)q(e)i(called)h(inside)g(the)e (user)h(de\014ned)g(function.)166 322 y(The)f(F)l(ortran)f(declaration) i(for)f(it)g(is)75 429 y FB(FUNCTION)23 b(UOP\(INVEC\(*\),)f (INOUTVEC\(*\),)g(LEN\))75 485 y()h(INVEC\(LEN\),)f (INOUTVEC\(LEN\))75 542 y(INTEGER)h(LEN)189 649 y Fo(R)n(ationale.)48 b FD(The)17 b(addition)g(of)g(the)f(third)h(argumen)o(t,)33 b Fs(len)p FD(,)17 b(in)34 b Fs(function)18 b FD(allo)o(ws)f(the)f (system)189 705 y(to)d(a)o(v)o(oid)g(calling)30 b Fs(function)15 b FD(for)e(eac)o(h)h(elemen)o(t)g(in)h(the)f(input)h(bu\013er.)k (Rather,)14 b(the)f(system)h(can)189 762 y(c)o(ho)q(ose)h(to)f(apply)32 b Fs(function)16 b FD(to)f(c)o(h)o(unks)g(of)g(input.)21 b(\()p Fo(End)16 b(of)g(r)n(ationale.)p FD(\))166 952 y Fl(Discussion:)31 b Ft(One)12 b(could)f(tak)o(e)g(adv)n(an)o(tage)g (of)f(the)i(more)e(lenien)o(t)i(C)f(t)o(yping)f(rules)i(and)f(declare)h (the)g(user)75 1008 y(function)i(to)g(b)q(e)h(of)e(t)o(yp)q(e)i Fv(void)f(Uop\(\))p Ft(;)g(suc)o(h)h(declaration)f(w)o(ould)f(not)h (constrain)h(the)f(t)o(yp)q(es)h(of)f(the)h(argumen)o(ts)75 1064 y(and)h(w)o(ould)f(a)o(v)o(oid)g(an)g(additional)f(t)o(yp)q (ecasting)j(within)e(the)i(function)e(b)q(o)q(dy)m(.)24 b(Ho)o(wvw)o(er,)16 b(suc)o(h)h(usage)f(is)g(not)75 1121 y(compatible)c(with)i(C++.)75 1325 y Fs(Example)f(of)i(User-Reduce)75 1411 y FD(Compute)g(the)g(pro)q(duct)h(of)f(an)g(arra)o(y)f(of)g (complex)i(n)o(um)o(b)q(ers,)g(in)g(C.)75 1518 y FB(typedef)23 b(struct)g({)170 1574 y(double)g(real,imag;)75 1631 y(})h(Complex;)75 1744 y(/*)g(the)f(user-defined)f(function)99 1800 y(*/)75 1913 y(void)h(myProd\()g(void)h(*in,)f(void)g(*inout,)g(int)h(*len)f (\))75 1970 y({)170 2026 y(int)h(i;)170 2083 y(Complex)f(c,)h(*invec,)f (*inoutvec;)170 2195 y(invec)h(=)f(\(Complex)g(*\)in;)170 2252 y(inoutvec)g(=)h(\(Complex)f(*\)out;)170 2308 y(for)h(\(i=0;)f(i<) h(*len;)f(++i\))g({)266 2365 y(c.real)g(=)h(inoutvec->real*invec->.r)o (eal)d(-)528 2421 y(inoutvec->imag*invec->imag;)266 2478 y(c.imag)i(=)h(inoutvec->real*invec->im)o(ag)d(+)528 2534 y(inoutvec->imag*invec->real;)266 2591 y(*inoutvec)i(=)g(c;)266 2647 y(invec++;)g(inoutvec++;)170 2704 y(})-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 114 121 114 120 bop 75 -100 a FD(114)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FB(})75 158 y(/*)24 b(and,)f(to)g(call)h(it...)99 214 y(*/)75 271 y(...)170 384 y(/*)g(each)f(process)g(has)h(an)f(array)g(of)h(100)g (Complexes)194 440 y(*/)170 497 y(Complex)f(a[100],)g(answer[100];)170 553 y(MPI_Datatype)g(ctype;)170 666 y(/*)h(explain)f(to)h(MPI)f(how)g (type)h(Complex)f(is)g(defined)194 723 y(*/)170 779 y (MPI_Type_contiguous\()f(2,)h(MPI_DOUBLE,)g(&ctype)g(\);)170 835 y(MPI_Type_commit\()f(&ctype)h(\);)170 948 y(MPI_User_reduce\()f (a,)i(answer,)f(100,)g(ctype,)g(myProd,)g(root,)g(comm)g(\);)170 1061 y(/*)h(At)g(this)f(point,)g(the)g(answer,)g(which)h(consists)e(of) i(100)f(Complexes,)194 1118 y(*)h(resides)f(on)h(process)e(root)194 1174 y(*/)75 1320 y Fs(MPI)p 160 1320 14 2 v 16 w(USER)p 288 1320 V 17 w(REDUCEA\()16 b(sendbuf,)h(recvbuf,)f(count,)g(datat)o (yp)q(e,)g(function,)h(ro)q(ot,)d(comm)m(\))117 1397 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1471 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)19 b(address)h(of)e(receiv)o(e)i(bu\013er)f(\(c)o(hoice,) h(signi\014can)o(t)905 1528 y(only)13 b(at)h(ro)q(ot\))117 1602 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g(\(in)o(teger\))117 1676 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(input)h(bu\013er)h(\(handle\))117 1750 y(IN)171 b Fs(function)418 b Ft(user)15 b(de\014ned)g(function)f (\(function\))117 1824 y(IN)171 b Fs(ro)q(ot)492 b Ft(rank)14 b(of)f(ro)q(ot)h(pro)q(cess)i(\(in)o(teger\))117 1898 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 2023 y FB(int)23 b(MPI)p 245 2023 15 2 v 17 w(User)p 358 2023 V 17 w(reducea\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g (count,)393 2079 y(MPI)p 468 2079 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2079 V 17 w(Uop)i(function,)e(int)i(root,)393 2135 y(MPI)p 468 2135 V 17 w(Comm)f(comm\))75 2222 y(MPI)p 150 2222 V 17 w(USER)p 263 2222 V 16 w(REDUCEA\(SENDBUF,)f(RECVBUF,)h (COUNT,)g(DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM,)393 2278 y(IERROR\))170 2335 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2391 y(EXTERNAL)g(FUNCTION)170 2448 y(INTEGER)g(COUNT,)g(DATATYPE,)g (ROOT,)g(COMM,)g(IERROR)166 2534 y FD(Iden)o(tical)16 b(to)f Fs(MPI)p 493 2534 14 2 v 16 w(USER)p 621 2534 V 17 w(REDUCE)p FD(,)g(except)g(that)g(the)g(op)q(eration)g(de\014ned)h (b)o(y)30 b Fs(function)16 b FD(is)f(not)75 2591 y(required)22 b(to)e(b)q(e)i(comm)o(utativ)o(e,)f(but)g(only)h(asso)q(ciativ)o(e.)37 b(Th)o(us,)22 b(the)f(order)f(of)h(computation)g(can)75 2647 y(b)q(e)d(mo)q(di\014ed)h(only)e(using)h(asso)q(ciativit)o(y)l(.) 27 b(Use)17 b(of)g(this)h(function)g(means)f(that)f(the)i(implemen)o (tation)75 2704 y Fo(c)n(annot)c FD(try)g(and)h(c)o(hange)f(the)h (order)f(of)g(computation)h(of)f(the)h(reduce)g(b)o(y)g(assuming)g (comm)o(utativit)o(y)l(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 115 122 115 121 bop 75 -100 a Fp(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 860 b FD(115)166 45 y Fl(Implemen)o(tati)o(on)12 b(note:)166 102 y Fv(MPI)p 243 102 13 2 v 14 w(USER)p 361 102 V 14 w(REDUCEA)e Ft(and)h Fv(MPI)p 729 102 V 14 w(USER)p 847 102 V 15 w(REDUCE)e Ft(can)j(ha)o(v)o(e)f(iden)o(tical)g(implemen)o (tati)o(ons,)e(if)i(one)g(do)q(es)75 158 y(not)j(wish)g(to)f(tak)o(e)h (adv)n(an)o(tage)f(of)h(comm)o(utati)o(vit)o(y)l(.)75 365 y Fn(4.5.3)49 b(All-Reduce)75 451 y FD(MPI)20 b(includes)i(v)m (arian)o(ts)e(of)f(eac)o(h)h(of)g(the)g(reduce)g(op)q(erations)g(where) g(the)g(result)h(is)f(kno)o(wn)g(to)f(all)75 508 y(pro)q(cesses)f(in)h (the)f(group)g(on)g(return.)28 b(MPI)18 b(requires)g(that)g(all)h(pro)q (cesses)f(participating)h(in)g(an)o(y)e(of)75 564 y(the)e(all-reduce)i (op)q(erations)f(receiv)o(e)g(exactly)g(iden)o(tical)h(results.)75 668 y Fs(MPI)p 160 668 14 2 v 16 w(ALLREDUCE\()e(sendbuf,)i(recvbuf,)f (count,)g(datat)o(yp)q(e,)g(op,)f(comm)m(\))117 746 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 822 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er)g(\(c)o (hoice\))117 897 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 973 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 1049 y(IN)171 b Fs(op)525 b Ft(op)q(eration)14 b(\(state\))117 1125 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 1250 y FB(int)23 b(MPI)p 245 1250 15 2 v 17 w(Allreduce\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 1306 y(MPI)p 468 1306 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1306 V 17 w(Op)i(op,)f(MPI)p 1266 1306 V 17 w(Comm)g(comm\))75 1393 y(MPI)p 150 1393 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g (DATATYPE,)g(OP,)g(COMM,)g(IERROR\))170 1449 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1506 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1593 y FD(Same)19 b(as)h(the)f Fs(MPI)p 519 1593 14 2 v 16 w(REDUCE)h FD(op)q(eration)g(function)g(except)g(that)f (the)h(result)g(app)q(ears)f(in)i(the)75 1649 y(receiv)o(e)16 b(bu\013er)f(of)g(all)h(the)f(group)g(mem)o(b)q(ers.)75 1753 y Fs(MPI)p 160 1753 V 16 w(USER)p 288 1753 V 17 w(ALLREDUCE\()g(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h (function,)f(comm)m(\))117 1831 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1906 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1982 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 2058 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(send)i(bu\013er)g (\(handle\))117 2134 y(IN)171 b Fs(function)418 b Ft(user)15 b(de\014ned)g(function)f(\(function\))117 2210 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 2335 y FB(int)23 b(MPI)p 245 2335 15 2 v 17 w(User)p 358 2335 V 17 w(allreduce\(void*)f(sendbuf,)g(void*)i(recvbuf,)e(int)i (count,)393 2391 y(MPI)p 468 2391 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2391 V 17 w(Uop)i(function,)e(MPI)p 1433 2391 V 17 w(Comm)h(comm\))75 2478 y(MPI)p 150 2478 V 17 w(USER)p 263 2478 V 16 w(ALLREDUCE\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)f (FUNCTION,)h(COMM,)393 2534 y(IERROR\))170 2591 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 2647 y(EXTERNAL)g(FUNCTION)170 2704 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 116 123 116 122 bop 75 -100 a FD(116)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y FD(Same)f(as)g(the)g Fs(MPI)p 506 45 14 2 v 16 w(USER)p 634 45 V 18 w(REDUCE)h FD(op)q(eration)f(function)h(except)g(that)f(the)g(result)h(app)q(ears) f(in)75 102 y(the)g(receiv)o(e)h(bu\013er)g(of)e(all)i(the)g(group)f (mem)o(b)q(ers.)75 206 y Fs(MPI)p 160 206 V 16 w(USER)p 288 206 V 17 w(ALLREDUCEA\()h(sendbuf,)h(recvbuf,)e(count,)h(datat)o (yp)q(e,)h(function,)f(comm)m(\))117 283 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 359 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 435 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 512 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h (of)f(send)i(bu\013er)g(\(handle\))117 588 y(IN)171 b Fs(function)418 b Ft(user)15 b(de\014ned)g(function)f(\(function\))117 664 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 789 y FB(int)23 b(MPI)p 245 789 15 2 v 17 w(User)p 358 789 V 17 w(allreducea\(void*)f(sendbuf,)g(void*)h(recvbuf,)g(int)h (count,)393 845 y(MPI)p 468 845 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 845 V 17 w(Uop)i(function,)e(MPI)p 1433 845 V 17 w(Comm)h(comm\))75 932 y(MPI)p 150 932 V 17 w(USER)p 263 932 V 16 w(ALLREDUCEA\(SENDBUF,)f(RECVBUF,)g(COUNT,)h(DATATYPE,)g (FUNCTION,)g(COMM,)393 988 y(IERROR\))170 1045 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 1101 y(EXTERNAL)g(FUNCTION)170 1158 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)166 1245 y FD(Same)16 b(as)f Fs(MPI)p 429 1245 14 2 v 16 w(USER)p 557 1245 V 18 w(REDUCEA)p FD(,)h(except)g(that)g(the)g(result) g(app)q(ears)g(in)h(the)f(receiv)o(e)h(bu\013er)e(of)75 1301 y(all)h(the)f(group)g(mem)o(b)q(ers.)166 1441 y Fl(Implemen)o(tati)o(on)c(note:)36 b Ft(The)13 b(all-reduce)h(op)q (erations)f(can)g(b)q(e)h(implemen)o(ted)d(as)i(a)g(reduce,)i(follo)o (w)o(ed)75 1497 y(b)o(y)f(a)f(broadcast.)19 b(Ho)o(w)o(ev)o(er,)14 b(a)f(direct)i(implemen)o(tatio)o(n)c(can)j(lead)g(to)g(b)q(etter)h(p)q (erformance.)75 1704 y Fn(4.5.4)49 b(Reduce-Scatter)75 1791 y FD(MPI)15 b(also)f(includes)j(v)m(arian)o(ts)e(of)f(eac)o(h)g (of)h(the)f(reduce)i(op)q(erations)f(where)f(the)h(result)g(is)g (scattered)g(to)75 1847 y(all)h(pro)q(cesses)g(in)g(the)f(group)g(on)g (return.)75 1951 y Fs(MPI)p 160 1951 V 16 w(REDUCE)p 352 1951 V 17 w(SCA)l(TTER\()h(sendbuf,)h(recvbuf,)f(recvcounts,)g (datat)o(yp)q(e,)h(op,)e(comm)m(\))117 2029 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 2105 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\)) 117 2181 y(IN)171 b Fs(recvcounts)372 b Ft(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g(the)h(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g (in)f(re-)905 2238 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 2294 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 2370 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 2446 y(IN)171 b Fs(op)525 b Ft(op)q(eration)14 b(\(state\))117 2522 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 2647 y FB(int)23 b(MPI)p 245 2647 15 2 v 17 w(Reduce)p 406 2647 V 17 w(scatter\(void*)f(sendbuf,)g(void*)i (recvbuf,)e(int)i(*recvcounts,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2704 V 17 w(Op)i(op,)f(MPI)p 1266 2704 V 17 w(Comm)g(comm\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 117 124 117 123 bop 75 -100 a Fp(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 860 b FD(117)75 45 y FB(MPI)p 150 45 15 2 v 17 w(REDUCE)p 311 45 V 16 w(SCATTER\(SENDBUF,)22 b(RECVBUF,)h(RECVCOUNTS,)f (DATATYPE,)h(OP,)g(COMM,)393 102 y(IERROR\))170 158 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 214 y(INTEGER)g(RECVCOUNTS,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR)166 308 y Fs(MPI)p 251 308 14 2 v 16 w(REDUCE)p 443 308 V 17 w(SCA)l(TTER)16 b FD(\014rst)f(do)q(es)h(a)f(comp)q(onen)o(t)o(wise)g(reduction)h(on)g (v)o(ectors)e(pro)o(vided)i(b)o(y)75 364 y(the)h(pro)q(cesses.)24 b(Next,)17 b(the)f(resulting)i(v)o(ector)e(of)g(results)h(is)g(split)g (in)o(to)g FB(n)f FD(disjoin)o(t)i(segmen)o(ts,)e(where)75 421 y FB(n)h FD(is)g(the)g(n)o(um)o(b)q(er)g(of)f(mem)o(b)q(ers)h(in)h (the)f(group;)g(segmen)o(t)f FB(i)h FD(con)o(tains)g Fs(recvcounts[i])h FD(elemen)o(ts.)26 b(The)75 477 y FB(i)p FD(-th)15 b(segmen)o(t)g(is)h(sen)o(t)f(to)f(pro)q(cess)i(with)f (rank)g FB(i)p FD(.)166 624 y Fl(Implemen)o(tati)o(on)10 b(note:)17 b Ft(The)12 b Fv(MPI)p 792 624 13 2 v 15 w(REDUCE)p 970 624 V 13 w(SCA)m(TTER)f Ft(routine)h(is)g(functionally)f(equiv)n (alen)o(t)g(to:)17 b(A)75 680 y Fv(MPI)p 152 680 V 14 w(REDUCE)28 b Ft(op)q(eration)g(function)h(with)f Fv(count)i Ft(equal)f(to)f(the)i(sum)e(of)g Fv(recvcounts[i])j Ft(follo)o(w)o(ed)c (b)o(y)75 736 y Fv(MPI)p 152 736 V 14 w(SCA)m(TTERV)15 b Ft(with)i Fv(sendcounts)h Ft(equal)f(to)f Fv(recvcounts)p Ft(.)28 b(Ho)o(w)o(ev)o(er,)17 b(a)g(direct)g(implemen)o(tatio)o(n)d (ma)o(y)h(run)75 793 y(faster.)75 986 y Fs(MPI)p 160 986 14 2 v 16 w(USER)p 288 986 V 17 w(REDUCE)p 481 986 V 17 w(SCA)l(TTER\()h(sendbuf,)i(recvbuf,)d(recvcnts,)h(t)o(yp)q(e,)g (function,)g(comm)n(\))117 1071 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1160 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1249 y(IN)171 b Fs(recvcnts)418 b Ft(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g(the)h (n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(in)f(re-)905 1306 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 1362 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 1451 y(IN)171 b Fs(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 1541 y(IN)171 b Fs(function)418 b Ft(user)15 b(de\014ned)g(function)f(\(function\))117 1630 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))166 1761 y FD(Same)j(as)g(the)g Fs(MPI)p 503 1761 V 16 w(REDUCE)p 695 1761 V 17 w(SCA)l(TTER)i FD(op)q(eration)e(function)h(except)g (that)e(the)i(user)f(sp)q(eci\014es)75 1818 y(the)h(reduction)h(op)q (eration)g(as)f(in)h Fs(MPI)p 753 1818 V 15 w(USER)p 880 1818 V 18 w(REDUCE)p FD(.)75 1929 y Fs(MPI)p 160 1929 V 16 w(USER)p 288 1929 V 17 w(REDUCE)p 481 1929 V 17 w(SCA)l(TTERA\()h(sendbuf,)g(recvbuf,)e(recvcnts,)i(t)o(yp)q(e,)e (function,)i(comm)m(\))117 2013 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 2102 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 2191 y(IN)171 b Fs(recvcnts)418 b Ft(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g(the)h (n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(in)f(re-)905 2248 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 2304 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 2394 y(IN)171 b Fs(t)o(yp)q(e)491 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 2483 y(IN)171 b Fs(function)418 b Ft(user)15 b(de\014ned)g(function)f(\(function\))117 2572 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))166 2704 y FD(The)k(\\A")g(v)o(ersion)g(of)g Fs(MPI)p 645 2704 V 16 w(USER)p 773 2704 V 17 w(REDUCE)p 966 2704 V 18 w(SCA)l(TTER)p FD(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 118 125 118 124 bop 75 -100 a FD(118)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fn(4.5.5)49 b(Scan)75 183 y Fs(MPI)p 160 183 14 2 v 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h(op,)e(comm)9 b(\))117 262 y Ft(IN)171 b Fs(sendbuf)428 b Ft(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 342 y(OUT)124 b Fs(recvbuf)434 b Ft(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 422 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input) g(bu\013er)g(\(in)o(teger\))117 502 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 582 y(IN)171 b Fs(op)525 b Ft(op)q(eration)14 b(\(state\))117 662 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 789 y FB(int)23 b(MPI)p 245 789 15 2 v 17 w(Scan\(void*)g(sendbuf,)f(void*)i(recvbuf,)e(int)i(count,)393 845 y(MPI)p 468 845 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 845 V 17 w(Op)i(op,)f(MPI)p 1266 845 V 17 w(Comm)g(comm)h(\))75 934 y(MPI)p 150 934 V 17 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 991 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1047 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1136 y Fs(MPI)p 251 1136 14 2 v 16 w(SCAN)f FD(is)g(used)h(to)e(p)q(erform)g(a)h(parallel)h(pre\014x)f(with)g(resp) q(ect)g(to)f(an)h(asso)q(ciativ)o(e)g(and)75 1193 y(comm)o(utativ)o(e)e (reduction)i(op)q(eration)g(on)f(data)f(distributed)j(across)d(the)h (group.)40 b(The)23 b(op)q(eration)75 1249 y(returns)17 b(in)h(the)f(receiv)o(e)i(bu\013er)e(of)g(the)g(pro)q(cess)g(with)h (rank)f FB(i)g FD(the)g(reduction)h(of)f(the)g(v)m(alues)i(in)f(the)75 1305 y(send)k(bu\013ers)f(of)f(pro)q(cesses)i(with)g(ranks)e FB(0,...,i)p FD(.)37 b(The)21 b(t)o(yp)q(e)h(of)e(op)q(erations)i(supp) q(orted,)g(their)75 1362 y(seman)o(tics,)15 b(and)g(the)h(constrain)o (ts)e(on)h(send)h(and)g(receiv)o(e)g(bu\013ers)f(are)g(as)g(for)f Fs(MPI)p 1531 1362 V 16 w(REDUCE)p FD(.)75 1468 y Fs(MPI)p 160 1468 V 16 w(USER)p 288 1468 V 17 w(SCAN\()i(sendbuf,)h(recvbuf,)f (count,)g(datat)o(yp)q(e,)g(function,)g(comm)n(\))117 1548 y Ft(IN)171 b Fs(sendbuf)428 b Ft(address)15 b(of)f(input)f (bu\013er)117 1628 y(OUT)124 b Fs(recvbuf)434 b Ft(address)15 b(of)f(output)g(bu\013er)117 1708 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input)h(and)g(output) g(bu\013er)h(\(in)o(te-)905 1764 y(ger\))117 1844 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)i (\(handle\))117 1924 y(IN)171 b Fs(function)418 b Ft(user)15 b(pro)o(vided)f(function)117 2004 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 2131 y FB(int)23 b(MPI)p 245 2131 15 2 v 17 w(User)p 358 2131 V 17 w(scan\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)h(count,)393 2187 y(MPI)p 468 2187 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2187 V 17 w(Uop)i(function,)e(MPI)p 1433 2187 V 17 w(Comm)h(comm\))75 2276 y(MPI)p 150 2276 V 17 w(USER)p 263 2276 V 16 w(SCAN\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)g (FUNCTION,)g(COMM,)g(IERROR\))170 2332 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2389 y(EXTERNAL)g(FUNCTION)170 2445 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)166 2534 y FD(Same)12 b(as)g(the)h Fs(MPI)p 498 2534 14 2 v 15 w(SCAN)g FD(op)q(eration)g(function)g(except)g(that)f(a)g(user)g (supplied)j(function)e(is)g(used.)75 2591 y Fs(function)k FD(is)f(an)g(asso)q(ciativ)o(e)f(and)h(comm)o(utativ)o(e)f(function)h (with)g(an)g(input)g(v)o(ector,)f(an)g(inout)h(v)o(ector,)75 2647 y(and)g(a)g(length)g(argumen)o(t.)21 b(The)16 b(t)o(yp)q(es)g(of)g (the)g(t)o(w)o(o)e(v)o(ectors)h(and)h(of)g(the)g(returned)g(v)m(alues)h (all)g(agree.)75 2704 y(See)f Fs(MPI)p 241 2704 V 16 w(USER)p 369 2704 V 17 w(REDUCE)g FD(for)e(more)h(details.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 119 126 119 125 bop 75 -100 a Fp(4.6.)34 b(CORRECTNESS)1280 b FD(119)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(USER)p 288 45 V 17 w(SCANA\()16 b(sendbuf,)h(recvbuf,)f(count,)g(datat)o(yp)q(e,)g (function,)h(comm)m(\))117 123 y Ft(IN)171 b Fs(sendbuf)428 b Ft(address)15 b(of)f(input)f(bu\013er)i(\(c)o(hoice\))117 200 y(OUT)124 b Fs(recvbuf)434 b Ft(address)15 b(of)f(output)g (bu\013er)h(\(c)o(hoice\))117 277 y(IN)171 b Fs(count)466 b Ft(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input)h(and)g(output) g(bu\013er)h(\(in)o(te-)905 333 y(ger\))117 410 y(IN)171 b Fs(datat)o(yp)q(e)408 b Ft(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)i (\(handle\))117 486 y(IN)171 b Fs(function)418 b Ft(user)15 b(de\014ned)g(function)f(\(function\))117 563 y(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))75 688 y FB(int)23 b(MPI)p 245 688 15 2 v 17 w(User)p 358 688 V 17 w(scana\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g(count,) 393 745 y(MPI)p 468 745 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 745 V 17 w(Uop)i(function,)e(MPI)p 1433 745 V 17 w(Comm)h(comm\))75 832 y(MPI)p 150 832 V 17 w(USER)p 263 832 V 16 w(SCANA\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)g (FUNCTION,)f(COMM,)i(IERROR\))170 889 y()f(SENDBUF\(*\),)g (RECVBUF\(*\))170 945 y(EXTERNAL)g(FUNCTION)170 1002 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)166 1089 y FD(Same)13 b(as)g Fs(MPI)p 424 1089 14 2 v 16 w(USER)p 552 1089 V 17 w(SCAN)p FD(,)h(except)f(that)g(the)g(user-de\014ned)i (op)q(eration)e(need)h(not)f(b)q(e)h(comm)o(u-)75 1145 y(tativ)o(e.)166 1278 y Fl(Implemen)o(tati)o(on)e(note:)166 1329 y Fv(MPI)p 243 1329 13 2 v 14 w(USER)p 361 1329 V 14 w(SCAN)j Ft(could)g(b)q(e)g(implemen)o(ted)e(as)i Fv(MPI)p 1040 1329 V 14 w(USER)p 1158 1329 V 15 w(SCANA)p Ft(,)e(though)i(for)g(some)f(arc)o(hitectures)75 1379 y(it)g(ma)o(y)d(b)q(e)k(p)q(ossible)f(to)g(arriv)o(e)g(at)f(a)h(faster) g Fv(MPI)p 864 1379 V 15 w(USER)p 983 1379 V 14 w(SCAN)f Ft(b)o(y)h(taking)f(adv)n(an)o(tage)g(of)g(comm)o(utativit)o(y)l(.)75 1609 y Fq(4.6)59 b(Co)n(rrectness)75 1714 y Fn(4.6.1)49 b(Synchronization)18 b(Side-E\013ects)75 1801 y FD(A)13 b(correct)f(program)g(should)i(in)o(v)o(ok)o(e)f(collectiv)o(e)h(comm)o (unications)g(so)e(that)g(deadlo)q(c)o(k)i(will)g(not)f(o)q(ccur,)75 1858 y(whether)f(collectiv)o(e)h(comm)o(unication)g(is)f(sync)o (hronizing)h(or)f(not.)18 b(The)12 b(follo)o(wing)g(examples)h (illustrate)75 1914 y(dangerous)i(use)h(of)e(collectiv)o(e)j(routines.) 166 1972 y(The)e(follo)o(wing)h(example)g(illustrates)h(an)e(erroneous) g(program.)75 2083 y FB(/*)24 b(Example)e(A)i(*/)75 2139 y(switch\(rank\))147 2195 y({)147 2252 y(case)f(0:)g({)h (MPI_Bcast\(&var1,)e(count,)h(type,)g(0,)h(comm\);)385 2308 y(MPI_Send\(&var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)385 2365 y(break;)337 2421 y(})147 2478 y(case)g(1:)g({)h(MPI_Recv\(&var2,) e(count,)h(type,)g(0,)h(tag,)f(comm\);)385 2534 y(MPI_Bcast\(&var1,)f (count,)h(type,)g(0,)h(comm\);)385 2591 y(break;)337 2647 y(})147 2704 y(})-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 120 127 120 126 bop 75 -100 a FD(120)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y FD(Pro)q(cess)g(zero)f (executes)h(a)g(broadcast,)f(follo)o(w)o(ed)h(b)o(y)f(a)h(blo)q(c)o (king)h(send)f(op)q(eration;)g(pro)q(cess)g(one)75 102 y(\014rst)h(executes)h(a)g(matc)o(hing)f(blo)q(c)o(king)i(receiv)o(e,)g (follo)o(w)o(ed)f(b)o(y)g(the)f(matc)o(hing)h(broadcast)f(call.)28 b(This)75 158 y(program)18 b(ma)o(y)h(deadlo)q(c)o(k.)32 b(The)20 b(broadcast)e(call)j(on)e(pro)q(cess)g(zero)g Fo(may)h FD(blo)q(c)o(k)g(un)o(til)g(pro)q(cess)g(one)75 214 y(executes)f(the)g(matc)o(hing)f(broadcast)g(call,)i(so)e(that)f (the)i(send)g(is)g(not)f(executed.)30 b(Pro)q(cess)19 b(one)f(will)75 271 y(de\014nitely)f(blo)q(c)o(k)f(on)f(the)h(receiv)o (e)g(and)f(so,)g(in)h(this)f(case,)g(nev)o(er)g(executes)h(the)f (broadcast.)166 328 y(The)g(follo)o(wing)h(example)g(is)g(correct,)e (but)i(non-deterministic:)75 439 y FB(/*)24 b(Example)e(B)i(*/)75 495 y(switch\(rank\))147 552 y({)170 608 y(case)g(0:)f({)h (MPI_Bcast\(&var1,)e(count,)h(type,)g(0,)h(comm\);)409 665 y(MPI_Send\(&var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)409 721 y(break;)361 778 y(})170 834 y(case)h(1:)f({)h(MPI_Recv\(&var2,)e (count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h(comm\);)409 890 y(MPI_Bcast\(&var1,)f(count,)h(type,)g(0,)h(comm\);)409 947 y(MPI_Recv\(&var2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h (comm\);)409 1003 y(break;)361 1060 y(})170 1116 y(case)h(2:)f({)h (MPI_Send\(&var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)409 1173 y(MPI_Bcast\(&var1,)f(count,)h(type,)g(0,)h(comm\);)409 1229 y(break;)361 1286 y(})170 1342 y(})166 1452 y FD(All)19 b(three)f(pro)q(cesses)g(participate)h(in)g(a)e(broadcast.)28 b(Pro)q(cess)17 b(0)h(sends)g(a)g(message)f(to)h(pro)q(cess)75 1508 y(1)h(after)g(the)h(broadcast,)f(and)h(pro)q(cess)f(2)h(sends)g(a) f(message)g(to)g(pro)q(cess)h(1)f(b)q(efore)g(the)h(broadcast.)75 1565 y(Pro)q(cess)15 b(1)g(receiv)o(es)h(b)q(efore)g(and)f(after)f(the) i(broadcast,)e(with)h(a)g(wildcard)i(source)e(argumen)o(t.)166 1622 y(Tw)o(o)e(p)q(ossible)i(executions,)g(with)e(di\013eren)o(t)h (matc)o(hings)g(of)f(sends)h(and)g(receiv)o(es,)g(are)g(illustrated)75 1679 y(in)i(\014gure)f(4.10.)166 1736 y(Note)h(that)g(the)h(second)g (execution)h(has)f(the)f(p)q(eculiar)j(e\013ect)e(that)f(a)g(send)h (executed)h(after)e(the)75 1792 y(broadcast)i(is)h(receiv)o(ed)g(at)f (another)g(no)q(de)h(b)q(efore)g(the)f(broadcast.)29 b(This)19 b(example)g(illustrates)h(the)75 1849 y(fact)i(that)f(one)i (should)g(not)f(rely)h(on)f(collectiv)o(e)i(comm)o(unication)g (functions)f(to)e(ha)o(v)o(e)h(particular)75 1905 y(sync)o(hronization) d(e\013ects.)29 b(T)l(o)17 b(assume)h(that)g(collectiv)o(e)i(comm)o (unication)f(functions)g(do)f(or)f(do)h(not)75 1962 y(ha)o(v)o(e)d (certain)g(sync)o(hronizing)i(side-e\013ects)f(is)g(non-p)q(ortable.) 166 2095 y Fl(Discussion:)166 2145 y Ft(An)c(alternativ)o(e)g(design)h (is)f(to)g(require)h(that)g(all)e(collectiv)o(e)h(comm)o(unication)d (calls)j(are)g(sync)o(hronizing.)18 b(In)75 2195 y(this)11 b(case,)h(the)g(second)g(program)d(is)i(deterministic)g(and)f(only)h (the)g(\014rst)h(execution)g(ma)o(y)d(o)q(ccur.)18 b(This)11 b(will)e(mak)o(e)75 2245 y(a)14 b(di\013erence)h(only)e(for)h (collectiv)o(e)g(op)q(erations)g(where)h(not)e(all)g(pro)q(cesses)j(b)q (oth)e(send)h(and)f(receiv)o(e)h(\(broadcast,)75 2295 y(reduce,)g(scatter,)g(gather\).)75 2503 y Fn(4.6.2)49 b(Multiple)17 b(Calls)h(to)e(Collective)75 2591 y FD(It)d(is)h(the)g (user's)f(resp)q(onsibilit)o(y)j(to)d(mak)o(e)f(sure)i(that)f(there)g (are)g(no)h(t)o(w)o(o)e(concurren)o(tly)i(executing)g(col-)75 2647 y(lectiv)o(e)h(calls)g(that)f(use)g(the)g(same)g(comm)o(unicator)g (on)g(the)g(same)g(pro)q(cess.)20 b(Since)15 b(all)g(collectiv)o(e)h (com-)75 2704 y(m)o(unication)j(calls)g(are)f(blo)q(c)o(king)h(this)g (restriction)f(only)h(a\013ects)e(m)o(ultithreaded)i(implemen)o (tations.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 121 128 121 127 bop 75 -100 a Fp(4.6.)34 b(CORRECTNESS)1280 b FD(121)383 4 y 18945146 16482269 0 0 29733355 25918013 startTexFig 383 4 a %%BeginDocument: coll-matchings.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -50 0 translate -40.0 436.0 translate 0.900 -0.900 scale 0.500 setlinewidth n 469 134 m 334 134 l gs col-1 s gr n 342.000 136.000 m 334.000 134.000 l 342.000 132.000 l gs 2 setlinejoin col-1 s gr n 154 399 m 284 399 l gs col-1 s gr n 276.000 397.000 m 284.000 399.000 l 276.000 401.000 l gs 2 setlinejoin col-1 s gr n 469 454 m 329 454 l gs col-1 s gr n 337.000 456.000 m 329.000 454.000 l 337.000 452.000 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 279 59 m gs 1 -1 scale (First Execution) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 274 299 m gs 1 -1 scale (Second Execution) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 139 94 m gs 1 -1 scale (0) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 319 94 m gs 1 -1 scale (1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 509 94 m gs 1 -1 scale (2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 194 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 194 m gs 1 -1 scale (send) col-1 show gr n 159 189 m 279 189 l gs col-1 s gr n 271.000 187.000 m 279.000 189.000 l 271.000 191.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 294 139 m gs 1 -1 scale (recv) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 44 94 m gs 1 -1 scale (process:) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 139 m gs 1 -1 scale (send) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 199 184 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 384 129 m gs 1 -1 scale (match) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 429 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 404 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 459 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 459 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 484 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 404 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 379 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 194 394 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 379 449 m gs 1 -1 scale (match) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1146 a FD(Figure)13 b(4.10:)k(A)c(race)f(condition)i(causes)f (non-deterministic)i(matc)o(hing)d(of)g(sends)i(and)e(receiv)o(es.)20 b(One)75 1202 y(cannot)15 b(rely)h(on)f(sync)o(hronization)h(from)e(a)h (broadcast)g(to)f(mak)o(e)h(the)g(program)f(deterministic.)75 1336 y(On)h(the)g(other)f(hand,)h(it)g(is)g(legitimate)h(for)e(one)g (pro)q(cess)h(to)f(start)g(a)g(new)h(collectiv)o(e)h(comm)o(unication) 75 1393 y(call)i(ev)o(en)e(though)h(a)f(previous)h(call)h(that)e(uses)g (the)h(same)f(comm)o(unicator)g(has)h(not)f(y)o(et)g(terminated)75 1449 y(on)f(another)g(pro)q(cess.)20 b(As)15 b(illustrated)i(in)f(the)f (follo)o(wing)h(example:)75 1557 y FB(/*)24 b(Example)e(C)i(*/)99 1614 y(MPI_Bcast\(&var1,)e(count,)h(type,)g(0,)g(comm\);)99 1670 y(MPI_Bcast\(&var2,)f(count,)h(type,)g(1,)g(comm\);)166 1778 y FD(In)17 b(a)e(nonsync)o(hronizing)j(implemen)o(tation)g(of)d (broadcast,)g(pro)q(cess)i(zero)f(ma)o(y)f(start)g(executing)75 1835 y(the)h(second)g(broadcast)f(b)q(efore)h(pro)q(cess)g(one)g (terminated)g(the)g(\014rst)f(broadcast.)21 b(Both)15 b(pro)q(cess)h(zero)75 1891 y(and)h(one)f(ma)o(y)g(terminate)g(their)h (t)o(w)o(o)e(broadcast)g(calls)j(b)q(efore)e(other)g(pro)q(cesses)h(ha) o(v)o(e)f(started)f(their)75 1948 y(calls.)21 b(It)15 b(is)h(the)f(implemen)o(tor's)h(resp)q(onsibilit)o(y)h(to)e(ensure)h (this)f(will)i(not)e(cause)h(an)o(y)e(error.)166 2080 y Fl(Implemen)o(tati)o(on)e(note:)166 2131 y Ft(Assume)e(that)g (broadcast)h(is)f(implemen)o(ted)e(using)i(p)q(oin)o(t-to-p)q(oin)o(t)e (MPI)j(comm)o(unicati)o(on.)j(The)d(follo)o(wing)75 2180 y(t)o(w)o(o)i(rules)i(are)f(satis\014ed:)134 2256 y(1.)22 b(All)13 b(receiv)o(es)i(sp)q(ecify)g(their)f(source)h(explicitly)e (\(no)h(wildcards\).)134 2332 y(2.)22 b(Eac)o(h)12 b(pro)q(cess)i (sends)g(all)d(messages)h(that)h(p)q(ertain)f(to)h(one)f(collectiv)o(e) g(call)g(b)q(efore)h(sending)f(an)o(y)g(message)189 2382 y(that)i(p)q(ertain)g(to)g(a)f(subsequen)o(t)j(collectiv)o(e)e(call.) 166 2458 y(Then)i(messages)g(b)q(elonging)f(to)h(successiv)o(e)i (broadcasts)f(cannot)f(b)q(e)h(confused,)g(as)f(the)h(order)f(of)g(p)q (oin)o(t-)75 2508 y(to-p)q(oin)o(t)d(messages)h(is)g(preserv)o(ed.)20 b(This)14 b(is)f(true,)i(in)e(general,)h(for)f(an)o(y)h(collectiv)o(e)g (library)m(.)166 2647 y FD(A)k(collectiv)o(e)h(comm)o(unication)f(ma)o (y)f(execute)i(in)f(a)g(con)o(text)f(while)i(p)q(oin)o(t-to-p)q(oin)o (t)f(comm)o(uni-)75 2704 y(cations)g(that)g(use)g(the)h(same)f(con)o (text)f(are)h(p)q(ending,)j(or)c(o)q(ccur)i(concurren)o(tly)l(.)30 b(This)19 b(is)g(illustrated)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 122 129 122 128 bop 75 -100 a FD(122)680 b Fp(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FD(in)g(example)g(B)g(ab)q (o)o(v)o(e,)f(the)g(\014rst)g(pro)q(cess)h(ma)o(y)f(receiv)o(e)h(a)f (message)g(sen)o(t)g(with)h(the)f(con)o(text)g(of)g(com-)75 102 y(m)o(unicator)32 b Fs(comm)10 b FD(while)17 b(it)f(is)g(executing) h(a)f(broadcast)f(with)h(the)g(same)g(comm)o(unicator.)22 b(It)15 b(is)i(the)75 158 y(implemen)o(tor's)f(resp)q(onsibilit)o(y)h (to)e(ensure)h(this)f(will)i(not)e(cause)g(an)o(y)g(confusion.)166 291 y Fl(Implemen)o(tati)o(on)21 b(note:)65 b Ft(Assume)22 b(that)f(collectiv)o(e)g(comm)o(unications)d(are)k(implemen)o(ted)d (using)75 340 y(p)q(oin)o(t-to-p)q(oin)o(t)e(MPI)i(comm)o(unication.)29 b(Then,)20 b(in)e(order)i(to)e(a)o(v)o(oid)g(confusion,)h(whenev)o(er)h (a)e(comm)o(unica-)75 390 y(tor)g(is)f(created,)i(a)e(\\hidden)h(comm)o (unicator")c(need)19 b(b)q(e)f(created)h(for)e(collectiv)o(e)g(comm)o (unication.)26 b(A)17 b(direct)75 440 y(implemen)o(tatio)o(n)11 b(of)i(MPI)h(collectiv)o(e)g(comm)o(unicatio)o(n)d(can)j(ac)o(hiev)o(e) g(a)g(similar)d(e\013ect)k(more)e(c)o(heaply)m(,)g(e.g.,)f(b)o(y)75 490 y(using)j(a)g(hidden)h(tag)f(or)g(con)o(text)h(bit)g(to)f(indicate) g(whether)i(the)f(comm)o(unicator)d(is)i(used)h(for)f(p)q(oin)o(t-to-p) q(oin)o(t)75 540 y(or)f(collectiv)o(e)g(comm)o(unicatio)o(n.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 123 130 123 129 bop 75 357 a Fr(Chapter)34 b(5)75 565 y Fy(Groups,)40 b(Con)m(texts,)75 690 y(Comm)m(unicators,)g(and)g(Cac)m(hing)75 931 y Fq(5.1)59 b(Intro)r(duction)75 1034 y FD(W)l(e)18 b(b)q(egin)h(this)g(c)o(hapter)f(with)g(a)g(p)q(ersp)q(ectiv)o(e)i(of)d (wh)o(y)h(MPI)g(has)g(included)j(features)c(that)h(are)f(not)75 1091 y(found)11 b(in)h(existing)f(v)o(endor)g(systems,)g(and)f(wh)o(y)h (these)g(are)f(imp)q(ortan)o(t.)18 b(One)12 b(of)e(the)h(main)g(motiv)m (ations)75 1147 y(for)f(the)h(features)g(describ)q(ed)i(here)f(is)f(to) f(allo)o(w)i(for)e(the)h(systematic)g(dev)o(elopmen)o(t)h(of)e (parallel)j(libraries.)75 1273 y Fn(5.1.1)49 b(Why)16 b(w)o(e)g(need)g(lib)o(ra)o(ries)75 1360 y FD(P)o(arallel)k(libraries)g (are)e(needed)i(to)e(encapsulate)i(the)f(distracting)g(complications)h (asso)q(ciated)f(with)75 1417 y(distributed)f(realizations)f(of)e(k)o (ey)h(algorithms.)23 b(They)16 b(ensure)h(consisten)o(t)f(correctness)g (of)g(suc)o(h)g(pro-)75 1473 y(cedures,)h(and)g(pro)o(vide)g(a)f (\\higher)h(lev)o(el")h(of)e(p)q(ortabilit)o(y)h(than)g(MPI)f(itself)i (can)e(pro)o(vide.)25 b(As)16 b(suc)o(h,)75 1529 y(libraries)24 b(prev)o(en)o(t)e(eac)o(h)h(programmer)e(from)h(rep)q(eating)h(the)g(w) o(ork)f(of)g(de\014ning)i(consisten)o(t)f(data)75 1586 y(structures,)13 b(data)f(la)o(y)o(outs,)h(and)g(metho)q(ds)g(that)g (implemen)o(t)h(k)o(ey)f(algorithms)g(\(suc)o(h)g(as)g(matrix)g(op)q (er-)75 1642 y(ations\).)21 b(Since)c(the)f(b)q(est)g(libraries)i(come) d(with)i(sev)o(eral)e(v)m(ariations)i(on)e(parallel)j(systems)d (\(di\013eren)o(t)75 1699 y(data)d(la)o(y)o(outs,)g(di\013eren)o(t)h (strategies)f(dep)q(enden)o(t)i(on)f(size)g(of)g(system,)f(or)g(t)o(yp) q(e)h(of)f(\015oating)h(p)q(oin)o(t\),)g(this)75 1755 y(to)q(o)i(needs)h(to)e(b)q(e)i(hidden)h(from)d(the)i(user.)75 1881 y Fn(5.1.2)49 b(Why)16 b(existing)h(systems)c(do)k(not)f(supp)q(o) o(rt)f(lib)o(ra)o(ries)75 1968 y FD(It)f(is)g(di\016cult)h(to)e(write)h (libraries)h(with)f(message)g(passing)g(systems)f(that)g(v)o(endors)g (pro)q(duce,)i(b)q(ecause)75 2025 y(they)h(ha)o(v)o(e)g(no)g(w)o(a)o(y) f(for)g(the)h(library)h(to)f(insulate)h(itself)g(e\013ectiv)o(ely)g (from)e(the)i(other)e(p)q(oin)o(t-to-p)q(oin)o(t)75 2081 y(message)j(passing)h(that)e(go)q(es)i(on)f(in)h(the)g(system.)28 b(F)l(or)18 b(instance,)i(message)e(tags)f(are)h(a)g(means)h(for)75 2138 y(a)i(library)i(to)e(hide)i(its)e(messages)g(from)g(the)h(user.)39 b(Ho)o(w)o(ev)o(er,)22 b(man)o(y)f(libraries)i(ma)o(y)e(c)o(ho)q(ose)h (the)75 2194 y(same)16 b(tags,)f(or)h(the)g(v)o(endor's)g(library)h(ma) o(y)e(itself)i(indiscriminately)j(use)c(tags)f(when)i(implemen)o(ting) 75 2251 y(collectiv)o(e)f(op)q(erations)e(\()p Fo(e.g.)p FD(,)g(global)h(sum\);)e(in)j(that)d(case,)h(collectiv)o(e)i(and)f(p)q (oin)o(t-to-p)q(oin)o(t)f(messages)75 2307 y(ma)o(y)g(get)h(mixed)h (up.)166 2364 y(Collectiv)o(e)c(op)q(erations,)g(suc)o(h)g(as)f (reduce,)h(broadcast,)f(and)g(so)g(on,)h(are)f(not)f(de\014ned)j(in)f (all)g(v)o(endor)75 2421 y(systems,)j(and)i(when)f(de\014ned)i(are)d (most)h(often)f(only)i(a)o(v)m(ailable)h(o)o(v)o(er)d(the)h(full)i(set) d(of)h(pro)q(cessors)g(\(or)75 2477 y(pro)q(cesses\))d(allo)q(cated)h (to)e(the)i(user.)19 b(Libraries)14 b(need)g(to)e(w)o(ork)g(on)h (di\013eren)o(t)h(collections)g(of)f(pro)q(cesses,)75 2534 y(and)j(to)g(ha)o(v)o(e)g(a)o(v)m(ailable)h(collectiv)o(e)h(op)q (erations)e(that)g(only)h(impact)f(the)g(pro)q(cesses)h(that)e(care)h (ab)q(out)75 2590 y(the)f(collectiv)o(e)i(op)q(eration.)166 2647 y(A)k(third)g(issue)h(is)f(that)f(libraries)j(don't)d(w)o(an)o(t)g (to)g(describ)q(e)j(p)q(oin)o(t-to-p)q(oin)o(t)e(comm)o(unication)75 2704 y(in)f(terms)e(of)g(lo)o(w-lev)o(el,)i(hardw)o(are-dep)q(enden)o (t)g(names.)30 b(In)20 b(fact,)e(they)h(migh)o(t)f(lik)o(e)i(to)e(use)h (virtual)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 124 131 124 130 bop 75 -100 a FD(124)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FD(names)e(for)f(pro)q(cesses)i(that)e(someho)o(w)g(re\015ect)i (the)f(relationship)i(of)d(pro)q(cesses)i(to)e(one)h(another)g(in)h (the)75 102 y(distributed)20 b(data)e(structure)g(natural)g(to)g(an)g (algorithm.)29 b(Libraries)20 b(\(or)d(program)h(mo)q(dules\))h(that)75 158 y(execute)h(concurren)o(tly)f(on)g(di\013eren)o(t)g(pro)q(cesses)g (ma)o(y)f(w)o(an)o(t)g(to)g(use)h(the)g(same)g(pro)q(cess)g(names)g (for)75 214 y(their)d(in)o(ternal)g(comm)o(unication.)75 337 y Fn(5.1.3)49 b(What)16 b(features)f(a)o(re)g(needed)h(to)g(supp)q (o)o(rt)g(lib)o(ra)o(ries)75 423 y FD(So,)f(w)o(e)g(ha)o(v)o(e)f (established)j(three)f(k)o(ey)f(areas)f(that)h(will)i(supp)q(ort)e(the) g(creation)h(of)e(libraries:)143 517 y FA(\017)23 b FD(Safe)11 b(comm)o(unication)i(space,)f(that)f(guaran)o(tees)g(that)g(a)g (library)i(can)f(comm)o(unicate)g(as)f(it)h(needs)189 574 y(to,)143 668 y FA(\017)23 b FD(Group)14 b(scop)q(e)h(for)g (collectiv)o(e)h(op)q(erations,)f(that)f(allo)o(w)h(libraries)h(to)e(a) o(v)o(oid)h(unnecessarily)i(syn-)189 725 y(c)o(hronizing)f(unin)o(v)o (olv)o(ed)h(pro)q(cessors,)143 819 y FA(\017)23 b FD(Abstract)15 b(pro)q(cess)h(naming)h(to)e(allo)o(w)h(libraries)i(to)d(describ)q(e)j (their)e(comm)o(unication)h(in)g(terms)189 875 y(suitable)f(to)f(their) g(o)o(wn)g(data)g(structures.)75 998 y Fn(5.1.4)49 b(What)16 b(these)f(features)g(a)o(re)g(called)i(in)g(MPI)75 1084 y FD(The)e(features)g(that)g(implemen)o(t)h(these)g(imp)q(ortan)o(t)e (concepts)i(are)f(as)g(follo)o(ws,)g(in)h(MPI:)143 1178 y FA(\017)23 b Fx(Con)o(texts)18 b FD(pro)o(vide)j(the)e(abilit)o(y)i (to)e(ha)o(v)o(e)g(separate)g(safe)h(\\univ)o(erses")g(of)f (message-passing)189 1235 y(in)g(MPI.)g(A)g(con)o(text)f(is)h (something)g(lik)o(e)h(an)f(additional)i(tag)c(that)i(di\013eren)o (tiates)g(messages,)189 1291 y(but)g(the)f(system)h(manages)f(it,)h (and)g(the)g(user)g(nev)o(er)g(manipulates)h(con)o(texts.)30 b(W)l(e)19 b(sa)o(y)f(that)189 1348 y(comm)o(unicators)11 b(\(b)q(elo)o(w\))g(are)g(a)g(sp)q(eci\014c)j(con)o(text)c(of)h(comm)o (unication,)i(when)f(w)o(e)f(are)g(sp)q(eaking)189 1404 y(lo)q(osely)l(.)21 b(The)15 b(use)g(of)g(separate)g(comm)o(unication)g (con)o(texts)g(b)o(y)g(distinct)h(libraries)h(\(or)d(distinct)189 1461 y(library)e(in)o(v)o(o)q(cations\))g(will)h(insulate)g(comm)o (unication)g(in)o(ternal)f(to)g(the)f(library)i(execution)g(from)189 1517 y(external)18 b(comm)o(unication.)29 b(This)18 b(allo)o(ws)h(the)f (in)o(v)o(o)q(cation)g(of)g(the)g(library)g(ev)o(en)h(if)f(there)g(are) 189 1574 y(p)q(ending)c(comm)o(unications,)f(and)f(a)o(v)o(oids)g(the)h (need)g(to)e(sync)o(hronize)j(en)o(try)d(or)h(exit)h(in)o(to)f(library) 189 1630 y(co)q(de.)143 1724 y FA(\017)23 b Fx(Groups)10 b FD(de\014ne)i(an)f(ordered)g(collection)i(of)e(pro)q(cesses,)h(eac)o (h)f(with)g(a)g(rank,)g(and)g(it)g(is)h(this)f(group)189 1781 y(that)k(de\014nes)i(the)f(lo)o(w-lev)o(el)h(names)f(for)g(in)o (ter-pro)q(cess)g(comm)o(unication)h(\(ranks)e(are)h(used)h(for)189 1837 y(sending)j(and)f(receiving\).)34 b(Th)o(us,)20 b(groups)e(de\014ne)j(a)e(scop)q(e)h(for)e(pro)q(cess)i(names)f(in)h(p) q(oin)o(t)g(to)189 1894 y(p)q(oin)o(t)15 b(comm)o(unication.)21 b(In)16 b(addition,)g(groups)f(de\014ne)h(the)f(scop)q(e)h(of)f (collectiv)o(e)i(op)q(erations.)143 1988 y FA(\017)23 b Fx(Comm)o(unicators)15 b FD(\(see)h([14)o(]\))f(encapsulate)i(all)g (of)f(these)g(ideas)h(in)g(an)f(ob)s(ject)f(that)g(pro)o(vides)189 2045 y(the)g(righ)o(t)g(scop)q(e)h(for)e(all)i(comm)o(unication)g(op)q (erations)g(in)g(MPI.)166 2139 y(The)f(curren)o(t)g(practice)h(in)f (man)o(y)g(comm)o(unication)g(libraries)i(is)e(that)g(there)g(is)g(a)g (a)f(unique,)j(pre-)75 2195 y(de\014ned)22 b(comm)o(unication)e(univ)o (erse)h(that)f(includes)i(all)f(pro)q(cesses)g(a)o(v)m(ailable)h(when)e (the)h(library)f(is)75 2252 y(initiated;)25 b(the)c(pro)q(cesses)g(are) f(assigned)i(consecutiv)o(e)f(ranks.)37 b(P)o(articipan)o(ts)20 b(in)i(a)e(p)q(oin)o(t-to-p)q(oin)o(t)75 2308 y(comm)o(unication)h(are) g(iden)o(ti\014ed)h(b)o(y)f(their)g(rank;)i(a)d(collectiv)o(e)j(comm)o (unication)e(\(suc)o(h)g(as)f(broad-)75 2365 y(cast\))c(in)o(v)o(olv)o (es)h(all)g(pro)q(cesses.)25 b(This)17 b(practice)g(can)g(b)q(e)g (follo)o(w)o(ed)g(in)g(MPI)g(b)o(y)f(using)i(the)e(prede\014ned)75 2421 y(comm)o(unicator)e Fs(MPI)p 456 2421 14 2 v 16 w(COMM)p 614 2421 V 17 w(W)o(ORLD)p FD(.)g Fo(Users)h(who)h(ar)n(e)g (satis\014e)n(d)f(with)h(such)h(pr)n(actic)n(e)e(c)n(an)g(plug)i(in)75 2478 y Fs(MPI)p 160 2478 V 16 w(COMM)p 318 2478 V 16 w(W)o(ORLD)e Fo(wher)n(ever)h(a)f(c)n(ommunic)n(ator)h(ar)n(gument)f (is)g(r)n(e)n(quir)n(e)n(d,)g(and)g(c)n(an)g(c)n(onse)n(quently)75 2534 y(disr)n(e)n(gar)n(d)h(the)g(r)n(est)g(of)g(this)g(chapter.)166 2591 y FD(The)f(discussion)h(has)e(dealt)h(so)f(far)f(with)i Fx(in)o(tra-comm)o(unication)p FD(:)22 b(comm)o(unication)15 b(within)g(a)75 2647 y(group.)27 b(MPI)18 b(also)g(supp)q(orts)g Fx(in)o(ter-comm)o(unication)p FD(:)27 b(comm)o(unication)18 b(across)f(groups.)28 b(When)75 2704 y(an)15 b(application)i(is)f (built)h(b)o(y)e(comp)q(osing)h(sev)o(eral)f(parallel)i(mo)q(dules,)f (it)g(is)g(con)o(v)o(enien)o(t)g(to)e(allo)o(w)i(one)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 125 132 125 131 bop 75 -100 a Fp(5.2.)34 b(BASIC)16 b(CONCEPTS)1215 b FD(125)75 45 y(mo)q(dule)16 b(to)f(comm)o(unicate)g(with)g(the)h (other)e(using)i(lo)q(cal)g(ranks)f(within)h(the)g(second)f(mo)q(dule)h (for)f(ad-)75 102 y(dressing.)21 b(This)15 b(is)h(esp)q(ecially)h(con)o (v)o(enien)o(t)f(in)g(a)f(clien)o(t-serv)o(er)h(computing)f(paradigm,)g (where)g(either)75 158 y(clien)o(t)f(or)e(serv)o(er)g(are)g(parallel.) 21 b(The)12 b(supp)q(ort)h(of)f(in)o(ter-comm)o(unication)i(also)e(pro) o(vides)h(a)f(mec)o(hanism)75 214 y(for)g(the)h(extension)g(of)f(MPI)h (to)f(a)g(dynamic)h(mo)q(del)h(where)f(not)f(all)h(pro)q(cesses)g(are)g (preallo)q(cated)g(at)f(ini-)75 271 y(tialization)j(time.)k(In)14 b(suc)o(h)f(a)g(situation,)h(it)f(b)q(ecomes)h(necessary)f(to)g(supp)q (ort)g(comm)o(unication)h(across)75 327 y(\\univ)o(erses.")28 b(In)o(ter-comm)o(unication)19 b(is)f(supp)q(orted)h(b)o(y)e(ob)s (jects)h(called)h Fx(in)o(ter-comm)o(unicators)p FD(.)75 384 y(These)13 b(bind)h(t)o(w)o(o)e(groups)h(together)f(with)h(a)f(con) o(text)h(shared)g(b)o(y)g(b)q(oth)g(groups.)18 b(MPI)13 b(pro)o(vides)h(mec)o(h-)75 440 y(anisms)f(for)e(creating)i(and)f (manipulating)i(in)o(ter-comm)o(unicators.)19 b(They)12 b(are)g(used)h(in)g(p)q(oin)o(t-to-p)q(oin)o(t)75 497 y(comm)o(unication)k(in)g(the)f(same)g(manner)g(as)f(regular)h Fx(in)o(tra-comm)o(unicators)p FD(.)24 b(Users)16 b(who)f(do)h(not)75 553 y(need)g(in)o(ter-comm)o(unication)g(in)g(their)g(applications)h (can)e(safely)h(ignore)f(this)h(extension.)166 610 y(Comm)o(unicators)f (also)h(pro)o(vide)g(a)f(\\cac)o(hing")h(mec)o(hanism)h(that)e(allo)o (ws)h(one)g(to)f(asso)q(ciate)h(new)75 666 y(attributes)g(with)h(comm)o (unicators,)f(on)g(a)g(par)g(with)h(MPI)f(built-in)j(features.)k(This) 17 b(can)f(b)q(e)h(used)g(b)o(y)75 723 y(adv)m(anced)h(users)e(to)g (further)g(adorn)h(comm)o(unicators,)f(and)g(b)o(y)h(MPI)f(to)g (implemen)o(t)i(some)e(comm)o(u-)75 779 y(nicator)h(functions.)25 b(F)l(or)16 b(example,)h(the)g(virtual)h(top)q(ology)e(functions)h (describ)q(ed)i(in)f(Chapter)e(6)g(are)75 836 y(lik)o(ely)h(to)e(b)q(e) g(supp)q(orted)h(this)g(w)o(a)o(y)l(.)75 981 y Fq(5.2)59 b(Basic)19 b(Concepts)75 1085 y Fn(5.2.1)49 b(Groups)75 1172 y FD(A)17 b Fx(group)g FD(is)g(an)g(ordered)g(set)f(of)h(pro)q (cess)g(iden)o(ti\014ers)h(\(henceforth)f(pro)q(cesses\);)g(pro)q(cess) g(iden)o(ti\014ers)75 1228 y(are)11 b(implemen)o(tation-dep)q(enden)o (t)i(ob)s(jects.)18 b(Eac)o(h)11 b(pro)q(cess)g(in)h(a)f(group)f(is)i (asso)q(ciated)f(with)g(an)g(in)o(teger)75 1285 y Fx(rank)p FD(.)19 b(Ranks)13 b(are)g(con)o(tiguous)g(and)h(start)e(from)g(zero.) 19 b(Groups)13 b(are)g(represen)o(ted)h(b)o(y)f(opaque)g Fx(group)75 1341 y(ob)s(jects)p FD(,)18 b(and)g(hence)g(cannot)f(b)q(e) i(directly)f(transferred)f(from)g(one)h(pro)q(cess)f(to)g(another.)27 b(A)17 b(group)75 1398 y(is)f(used)f(within)h(a)f(comm)o(unicator)g(to) f(describ)q(e)j(the)e(participan)o(ts)g(in)h(a)f(comm)o(unication)h (\\univ)o(erse")75 1454 y(and)j(to)e(rank)i(suc)o(h)f(participan)o(ts)h (\(th)o(us)f(giving)h(them)g(unique)h(names)e(within)i(that)e(\\univ)o (erse")g(of)75 1510 y(comm)o(unication\).)166 1650 y Fl(Implemen)o(tati)o(on)e(note:)48 b Ft(A)17 b(group)g(ma)o(y)e(b)q(e)i (represen)o(ted)j(b)o(y)d(a)f(virtual-to-real)g(pro)q(cess-address-)75 1706 y(translation)d(table.)18 b(Eac)o(h)c(comm)o(unicator)d(ob)r(ject) k(w)o(ould)e(ha)o(v)o(e)h(a)f(p)q(oin)o(ter)i(to)e(suc)o(h)i(a)f (table.)75 1913 y Fn(5.2.2)49 b(Contexts)75 2000 y FD(A)15 b Fx(con)o(text)h FD(is)g(the)f(MPI)g(mec)o(hanism)h(for)f (partitioning)h(comm)o(unication)g(space.)21 b(A)15 b(de\014ning)i (prop-)75 2056 y(ert)o(y)d(of)g(a)g(con)o(text)g(is)h(that)e(a)h (message)g(sen)o(t)h(in)g(a)f(con)o(text)g(cannot)g(b)q(e)h(receiv)o (ed)h(in)f(another)f(con)o(text.)75 2112 y(Con)o(texts)20 b(are)h(not)g(explicit)j(MPI)d(ob)s(jects;)j(they)d(app)q(ear)h(only)g (along)f(with)h(groups)f(in)h(comm)o(u-)75 2169 y(nicators.)31 b(Distinct)20 b(comm)o(unicators)e(in)i(the)f(same)f(pro)q(cess)i(ha)o (v)o(e)e(distinct)i(con)o(texts.)31 b(A)19 b(con)o(text)75 2225 y(is)g(essen)o(tially)h(a)e(tag)g(\(or)g(tags\))f(needed)j(to)e (mak)o(e)g(a)g(comm)o(unicator)g(safe)h(for)e(p)q(oin)o(t-to-p)q(oin)o (t)j(and)75 2282 y(MPI-de\014ned)d(collectiv)o(e)g(comm)o(unication.) 166 2421 y Fl(Implemen)o(tati)o(on)11 b(note:)35 b Ft(A)14 b(p)q(ossible)f(implemen)o(tatio)o(n)d(is)j(for)g(a)g(con)o(text)h(to)e (b)q(e)i(a)f(lab)q(el)f(attac)o(hed)i(to)75 2478 y(messages)f(and)f (matc)o(hed)g(on)g(receiv)o(e.)19 b(Eac)o(h)13 b(in)o(tra-comm)o (unicator)d(stores)j(the)h(v)n(alue)e(of)g(that)g(lab)q(el.)17 b(Comm)o(u-)75 2534 y(nicator)f(generating)g(functions)h(use)f(a)g (collectiv)o(e)g(comm)o(unication)d(to)j(agree)h(on)f(a)g(new)g (group-wide)g(unique)75 2591 y(con)o(text.)25 b(In)15 b(in)o(ter-comm)o(unication,)e(it)i(is)h(more)f(con)o(v)o(enien)o(t)h (to)g(use)g(t)o(w)o(o)g(lab)q(els,)f(one)h(used)h(b)o(y)f(group)f(A)h (to)75 2647 y(send)e(and)e(group)h(B)g(to)g(receiv)o(e,)h(and)e(the)i (second)f(used)h(b)o(y)f(group)f(B)h(to)g(send)h(and)e(group)h(A)g(to)f (receiv)o(e.)19 b(Since)75 2704 y(con)o(texts)c(are)f(not)g(explicit)g (ob)r(jects,)g(other)h(implem)o(en)o(tations)c(are)k(p)q(ossible.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 126 133 126 132 bop 75 -100 a FD(126)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fn(5.2.3)49 b(Comm)o(unicato)o(r)o(s)75 154 y FD(Comm)o(unicators) 18 b(bring)h(together)f(the)h(concepts)h(of)e(group)g(and)h(con)o (text.)30 b(F)l(urthermore,)19 b(to)f(sup-)75 210 y(p)q(ort)c(implemen) o(tation-sp)q(eci\014c)j(optimizations,)d(and)g(virtual)h(top)q (ologies,)f(they)g(\\cac)o(he")g(additional)75 267 y(information.)37 b(The)21 b(source)g(and)g(destination)h(of)e(a)g(message)h(is)g(iden)o (ti\014ed)i(b)o(y)e(the)g(rank)f(of)h(that)75 323 y(pro)q(cess)14 b(within)g(the)g(group.)19 b(F)l(or)13 b(collectiv)o(e)i(comm)o (unication,)f(the)f(comm)o(unicator)g(sp)q(eci\014es)j(the)d(set)75 379 y(of)k(pro)q(cesses)h(that)f(participate)i(in)f(the)g(collectiv)o (e)i(op)q(eration)d(and)h(their)g(order,)g(when)g(signi\014can)o(t.)75 436 y(Th)o(us,)c(the)g(comm)o(unicator)g(restricts)f(the)h(\\spatial")g (scop)q(e)h(of)f(comm)o(unication,)g(and)g(pro)o(vides)h(lo)q(cal)75 492 y(pro)q(cess)g(addressing.)166 561 y(Comm)o(unicators)10 b(are)h(represen)o(ted)h(b)o(y)g(opaque)f Fx(comm)o(unicator)i(ob)s (jects)p FD(,)f(and)g(hence)g(cannot)75 617 y(b)q(e)k(directly)g (transferred)f(from)g(one)g(pro)q(cess)g(to)g(another.)75 806 y Fn(5.2.4)49 b(Prede\014ned)15 b(Comm)o(unicato)o(r)o(s)75 914 y FD(An)g(initial)h(comm)o(unicator)27 b Fv(MPI)p 665 914 13 2 v 14 w(COMM)p 809 914 V 15 w(W)o(ORLD)14 b FD(of)g(all)h(pro)q(cesses)g(the)g(lo)q(cal)g(pro)q(cess)g(can)f (comm)o(u-)75 971 y(nicate)i(with)f(after)g(initialization)j(\(itself)e (included\))h(is)f(de\014ned)h(once)e Fs(MPI)p 1403 971 14 2 v 16 w(INIT)f FD(has)h(b)q(een)i(called.)166 1039 y(In)f(a)f(static-pro)q(cess-mo)q(del)h(implemen)o(tation)g(of)f(MPI,)g (all)h(pro)q(cesses)g(that)e(participate)i(in)g(the)75 1095 y(computation)i(are)g(a)o(v)m(ailable)i(after)d(MPI)i(is)f (initialize)q(d.)32 b(F)l(or)17 b(this)i(case,)g Fv(MPI)p 1481 1095 13 2 v 14 w(COMM)p 1625 1095 V 14 w(W)o(ORLD)g FD(is)f(a)75 1152 y(comm)o(unicator)h(of)g(all)i(pro)q(cesses)f(a)o(v)m (ailable)h(for)e(the)g(computation;)i(this)f(comm)o(unicator)g(has)f (the)75 1208 y(same)14 b(v)m(alue)h(in)g(all)g(pro)q(cesses.)20 b(In)15 b(an)f(implemen)o(tation)h(of)f(MPI)g(where)g(pro)q(cesses)h (can)f(dynamically)75 1265 y(join)23 b(an)f(MPI)g(execution,)j(it)d(ma) o(y)g(b)q(e)h(the)f(case)h(that)e(a)h(pro)q(cess)h(starts)e(an)h(MPI)g (computation)75 1321 y(without)d(ha)o(ving)g(access)g(to)f(all)i(other) f(pro)q(cesses.)31 b(In)20 b(suc)o(h)f(situations,)h Fv(MPI)p 1479 1321 V 14 w(COMM)p 1623 1321 V 15 w(W)o(ORLD)f FD(is)g(a)75 1378 y(comm)o(unicator)d(incorp)q(orating)h(all)h(pro)q (cesses)f(with)g(whic)o(h)g(the)g(joining)h(pro)q(cess)f(can)f (immediately)75 1434 y(comm)o(unicate.)33 b(Therefore,)20 b Fv(MPI)p 683 1434 V 14 w(COMM)p 827 1434 V 15 w(W)o(ORLD)f FD(ma)o(y)g(sim)o(ultaneouly)i(ha)o(v)o(e)e(di\013eren)o(t)h(v)m(alues) g(in)75 1491 y(di\013eren)o(t)15 b(pro)q(cesses.)166 1559 y(All)i(MPI)e(implemen)o(tations)i(are)e(required)i(to)d(pro)o (vide)i(the)g Fv(MPI)p 1316 1559 V 14 w(COMM)p 1460 1559 V 15 w(W)o(ORLD)f FD(comm)o(unica-)75 1615 y(tor.)34 b(It)21 b(cannot)f(b)q(e)h(deallo)q(cated)h(during)f(the)f(life)i(of)e (a)g(pro)q(cess.)36 b(The)20 b(group)g(corresp)q(onding)i(to)75 1672 y(this)14 b(comm)o(unicator)g(do)q(es)g(not)g(app)q(ear)g(as)g(a)f (pre-de\014ned)j(constan)o(t,)d(but)h(it)h(ma)o(y)e(b)q(e)i(accessed)f (using)75 1728 y Fs(MPI)p 160 1728 14 2 v 16 w(COMM)p 318 1728 V 16 w(GROUP)e FD(\(see)f(b)q(elo)o(w\).)18 b(MPI)11 b(do)q(es)f(not)h(sp)q(ecify)g(the)g(corresp)q(ondence)h(b)q (et)o(w)o(een)e(the)h(pro-)75 1785 y(cess)g(rank)g(in)g Fv(MPI)p 388 1785 13 2 v 14 w(COMM)p 532 1785 V 15 w(W)o(ORLD)g FD(and)g(its)g(\(mac)o(hine-dep)q(enden)o(t\))h(absolute)g(address.)18 b(Neither)12 b(do)q(es)75 1841 y(MPI)k(sp)q(ecify)i(the)e(function)h (of)e(the)i(host)e(pro)q(cess,)h(if)h(an)o(y)l(.)23 b(An)16 b(inquiry)i(function)f Fs(MPI)p 1636 1841 14 2 v 15 w(GET)p 1739 1841 V 17 w(HOST)75 1897 y FD(is)i(pro)o(vided)g(in)g(Chapter)f(7) g(so)g(that)f(MPI)i(programs)e(can)h(determine)h(if)g(there)g(is)f(a)g (host)g(pro)q(cess,)75 1954 y(and)13 b(\014nd)g(its)g(rank)f(in)h Fv(MPI)p 539 1954 13 2 v 15 w(COMM)p 684 1954 V 14 w(W)o(ORLD)g FD(if)g(it)g(should)g(exist.)20 b(Other)12 b(implemen)o(tation-dep)q (end)q(en)o(t,)75 2010 y(prede\014ned)17 b(comm)o(unicators)e(ma)o(y)f (also)h(b)q(e)h(pro)o(vided.)166 2161 y Fl(Discussion:)29 b Ft(A)10 b(h)o(yp)q(othetical)f(implemen)o(tation)d(for)k(dynamic)e (pro)q(cess)j(spa)o(wning)e(is)g(to)h(add)f(a)g(function)75 2218 y(that)k(spa)o(wns)h Fv(n)f Ft(copies)h(of)f(a)g(program,)e(when)i (suc)o(h)h(a)f(function)g(is)g(in)o(v)o(ok)o(ed)g(b)o(y)f(a)h(single)g (paren)o(t)h(pro)q(cess.)20 b(The)75 2274 y(paren)o(t)d(is)f(returned)i (a)e(new)h(comm)o(unicator)c(for)j(itself)g(and)g(the)h(new)g(c)o (hildren;)g(eac)o(h)g(c)o(hild)f(is)g(started)h(with)75 2331 y Fv(MPI)p 152 2331 V 14 w(COMM)p 296 2331 V 15 w(W)o(ORLD)i Ft(set)g(to)g(that)g(same)f(comm)o(unicator.)30 b(The)19 b(c)o(hildren)g(will)e(\014nd)i(the)h(paren)o(t)f(is)g(their) 75 2387 y(\\host")12 b(when)g(they)g(use)h(the)g(MPI)f(inquiry)f (function)g Fv(MPI)p 1013 2387 V 15 w(GET)p 1109 2387 V 14 w(HOST)p Ft(,)h(whic)o(h)g(indicates)g(the)g(rank)g(of)f(host)h (in)75 2443 y(the)f(comm)o(unicator)d Fv(MPI)p 488 2443 V 14 w(COMM)p 632 2443 V 15 w(W)o(ORLD)p Ft(.)i(An)h(in)o(ter-)f(or)h (in)o(tra-comm)n(unicator)d(including)i(the)h(new)g(c)o(hildren)75 2500 y(and)k(previous)h(pro)q(cesses)i(can)d(no)o(w)g(b)q(e)h(built)f (explicitly)m(,)f(using)h(the)h(paren)o(t)f(as)h(a)f(bridge.)22 b(Generally)m(,)14 b(a)i(new)75 2556 y(group)f(of)f(pro)q(cesses)k(can) e(merge)e(in)o(to)h(an)f(existing)h(group,)g(pro)o(vided)g(these)i(t)o (w)o(o)d(groups)i(ha)o(v)o(e)e(a)h(non-empt)o(y)75 2613 y(in)o(tersection,)f(or)g(b)q(oth)g(ha)o(v)o(e)g(a)g(non-empt)o(y)e(in) o(tersection)j(with)f(a)f(third)h(group.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 127 134 127 133 bop 75 -100 a Fp(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b FD(127)75 45 y Fq(5.3)59 b(Group)20 b(Management)75 147 y FD(This)g(section)h(describ)q(es)g(the)f(manipulation)h(of)e(pro) q(cess)h(groups)g(in)g(MPI.)f(These)h(op)q(erations)g(are)75 203 y(lo)q(cal)c(and)g(their)f(execution)i(do)e(not)g(require)h(in)o (terpro)q(cess)f(comm)o(unication.)75 322 y Fn(5.3.1)49 b(Group)16 b(Accesso)o(rs)75 455 y Fs(MPI)p 160 455 14 2 v 16 w(GROUP)p 328 455 V 18 w(SIZE\(group,)e(size\))117 532 y Ft(IN)155 b Fs(group)479 b Ft(group)14 b(\(handle\))117 604 y(OUT)108 b Fs(size)519 b Ft(n)o(um)o(b)q(er)13 b(of)h(pro)q (cesses)i(in)e(the)g(group)g(\(in)o(teger\))75 728 y FB(int)23 b(MPI)p 245 728 15 2 v 17 w(Group)p 382 728 V 17 w(size\(MPI)p 591 728 V 16 w(Group)g(group,)g(int)g(*size\))75 815 y(MPI)p 150 815 V 17 w(GROUP)p 287 815 V 16 w(SIZE\(GROUP,)g(SIZE,) g(IERROR\))170 871 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 1005 y Fs(MPI)p 160 1005 14 2 v 16 w(GROUP)p 328 1005 V 18 w(RANK\(group,)14 b(rank\))117 1082 y Ft(IN)155 b Fs(group)479 b Ft(group)14 b(\(handle\))117 1154 y(OUT)108 b Fs(rank)504 b Ft(rank)50 b(of)g(the)h(calling)e(pro)q(cess)j(in)e (group,)59 b(or)905 1210 y Fv(MPI)p 982 1210 13 2 v 15 w(UNDEFINED)13 b Ft(if)h(the)h(pro)q(cess)i(is)d(not)h(a)f(mem)o(b)q (er)f(\(in-)905 1267 y(teger\))75 1391 y FB(int)23 b(MPI)p 245 1391 15 2 v 17 w(Group)p 382 1391 V 17 w(rank\(MPI)p 591 1391 V 16 w(Group)g(group,)g(int)g(*rank\))75 1478 y(MPI)p 150 1478 V 17 w(GROUP)p 287 1478 V 16 w(RANK\(GROUP,)g(RANK,)g (IERROR\))170 1534 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)75 1668 y Fs(MPI)p 160 1668 14 2 v 16 w(GROUP)p 328 1668 V 18 w(TRANSLA)l(TE)p 602 1668 V 17 w(RANKS)16 b(\(group1,)e(n,)h (ranks1,)g(group2,)g(ranks2\))117 1745 y Ft(IN)155 b Fs(group1)456 b Ft(group1)14 b(\(handle\))117 1817 y(IN)155 b Fs(n)564 b Ft(n)o(um)o(b)q(er)11 b(of)h(ranks)g(in)23 b Fv(ranks1)13 b Ft(and)f Fv(ranks2)g Ft(arra)o(ys)g(\(in)o(teger\))117 1889 y(IN)155 b Fs(ranks1)464 b Ft(arra)o(y)14 b(of)f(zero)i(or)f(more) f(v)n(alid)f(ranks)i(in)g(group1)117 1961 y(IN)155 b Fs(group2)456 b Ft(group2)14 b(\(handle\))117 2032 y(OUT)108 b Fs(ranks2)464 b Ft(arra)o(y)64 b(of)f(corresp)q(onding)i(ranks)f(in)g (group2,)905 2089 y Fv(MPI)p 982 2089 13 2 v 15 w(UNDEFINED)12 b Ft(when)j(no)e(corresp)q(ondence)k(exists.)75 2213 y FB(int)23 b(MPI)p 245 2213 15 2 v 17 w(Group)p 382 2213 V 17 w(translate)p 615 2213 V 16 w(ranks)g(\(MPI)p 870 2213 V 16 w(Group)h(group1,)f(int)g(n,)h(int)f(*ranks1,)393 2270 y(MPI)p 468 2270 V 17 w(Group)g(group2,)g(int)g(*ranks2\))75 2356 y(MPI)p 150 2356 V 17 w(GROUP)p 287 2356 V 16 w(TRANSLATE)p 519 2356 V 16 w(RANKS\(GROUP1,)f(N,)i(RANKS1,)f(GROUP2,)g(RANKS2,)g (IERROR\))170 2413 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1\(*\),)e(GROUP2,)h (RANKS2\(*\),)g(IERROR)75 2561 y Fn(5.3.2)49 b(Group)16 b(Constructo)o(rs)75 2647 y FD(These)k(functions)g(allo)o(w)g (construction)g(of)g(new)g(groups)f(from)g(existing)h(groups.)33 b(These)20 b(are)g(lo)q(cal)75 2704 y(op)q(erations,)d(and)h(distinct)g (groups)f(ma)o(y)f(b)q(e)i(de\014ned)h(on)e(di\013eren)o(t)g(pro)q (cesses;)h(a)f(pro)q(cess)h(ma)o(y)e(also)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 128 135 128 134 bop 75 -100 a FD(128)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FD(de\014ne)g(a)f(group)g(that)f(do)q(es)i(not)f(include)i (itself.)21 b(Consisten)o(t)14 b(de\014nitions)i(are)d(required)j(when) e(groups)75 102 y(are)h(used)g(as)g(argumen)o(ts)f(in)i(comm)o (unicator-building)i(functions.)i(MPI)15 b(do)q(es)h(not)e(pro)o(vide)i (a)f(mec)o(h-)75 158 y(anism)h(to)g(build)i(a)e(group)g(from)f(scratc)o (h,)h(but)g(only)g(from)g(other,)f(previously)j(de\014ned)f(groups.)23 b(The)75 214 y(base)13 b(group,)g(up)q(on)h(whic)o(h)g(all)g(other)f (groups)g(are)g(de\014ned,)i(is)e(the)h(group)f(asso)q(ciated)g(with)h (the)f(initial)75 271 y(comm)o(unicator)e Fs(MPI)p 453 271 14 2 v 16 w(COMM)p 611 271 V 17 w(W)o(ORLD)g FD(\(accessible)i (through)f(the)g(function)g Fs(MPI)p 1520 271 V 16 w(COMM)p 1678 271 V 16 w(GROUP)p FD(\).)75 377 y Fs(MPI)p 160 377 V 16 w(GROUP)p 328 377 V 18 w(UNION\(group1,)i(group2,)h (newgroup\))117 457 y Ft(IN)155 b Fs(group1)456 b Ft(\014rst)15 b(group)f(\(handle\))117 536 y(IN)155 b Fs(group2)456 b Ft(second)15 b(group)f(\(handle\))117 616 y(OUT)108 b Fs(newgroup)405 b Ft(union)14 b(group)f(\(handle\))75 743 y FB(int)23 b(MPI)p 245 743 15 2 v 17 w(Group)p 382 743 V 17 w(union\(MPI)p 615 743 V 16 w(Group)g(group1,)g(MPI)p 1037 743 V 17 w(Group)g(group2,)g(MPI)p 1460 743 V 16 w(Group)h(*newgroup\))75 832 y(MPI)p 150 832 V 17 w(GROUP)p 287 832 V 16 w(UNION\(GROUP1,)e(GROUP2,)h(NEWGROUP,)g(IERROR\))170 888 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1024 y Fs(MPI)p 160 1024 14 2 v 16 w(GROUP)p 328 1024 V 18 w(INTERSECTION\(group1,)15 b(group2,)f(newgroup\))117 1104 y Ft(IN)155 b Fs(group1)456 b Ft(\014rst)15 b(group)f(\(handle\)) 117 1184 y(IN)155 b Fs(group2)456 b Ft(second)15 b(group)f(\(handle\)) 117 1263 y(OUT)108 b Fs(newgroup)405 b Ft(in)o(tersection)15 b(group)f(\(handle\))75 1390 y FB(int)23 b(MPI)p 245 1390 15 2 v 17 w(Group)p 382 1390 V 17 w(intersection\(MPI)p 783 1390 V 15 w(Group)g(group1,)g(MPI)p 1204 1390 V 17 w(Group)g(group2,)393 1447 y(MPI)p 468 1447 V 17 w(Group)g(*newgroup\)) 75 1535 y(MPI)p 150 1535 V 17 w(GROUP)p 287 1535 V 16 w(INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 1592 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1728 y Fs(MPI)p 160 1728 14 2 v 16 w(GROUP)p 328 1728 V 18 w(DIFFERENCE\(group1,)14 b(group2,)g(newgroup\))117 1807 y Ft(IN)155 b Fs(group1)456 b Ft(\014rst)15 b(group)f(\(handle\)) 117 1887 y(IN)155 b Fs(group2)456 b Ft(second)15 b(group)f(\(handle\)) 117 1967 y(OUT)108 b Fs(newgroup)405 b Ft(di\013erence)16 b(group)e(\(handle\))75 2094 y FB(int)23 b(MPI)p 245 2094 15 2 v 17 w(Group)p 382 2094 V 17 w(difference\(MPI)p 735 2094 V 15 w(Group)g(group1,)g(MPI)p 1156 2094 V 17 w(Group)g(group2,)393 2150 y(MPI)p 468 2150 V 17 w(Group)g(*newgroup\)) 75 2239 y(MPI)p 150 2239 V 17 w(GROUP)p 287 2239 V 16 w(DIFFERENCE\(GROUP1,)f(GROUP2,)h(NEWGROUP,)f(IERROR\))170 2296 y(INTEGER)h(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 2384 y FD(The)15 b(set-lik)o(e)i(op)q(erations)e(are)g(de\014ned)i(as)d (follo)o(ws:)75 2488 y Fx(union)24 b FD(All)19 b(elemen)o(ts)g(of)f (the)h(\014rst)f(group)g(\()p Fs(group1)p FD(\),)g(follo)o(w)o(ed)h(b)o (y)f(all)h(elemen)o(ts)g(of)f(second)h(group)189 2544 y(\()p Fs(group2)p FD(\))14 b(not)h(in)h(\014rst.)75 2647 y Fx(in)o(tersect)23 b FD(all)18 b(elemen)o(ts)h(of)e(the)g (\014rst)h(group)f(that)g(are)g(also)g(in)i(the)f(second)g(group,)f (ordered)h(as)f(in)189 2704 y(\014rst)d(group.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 129 136 129 135 bop 75 -100 a Fp(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b FD(129)75 45 y Fx(di\013erence)22 b FD(all)c(elemen)o(ts)f(of)g(the)g (\014rst)f(group)g(that)g(are)h(not)f(in)h(the)g(second)g(group,)g (ordered)g(as)f(in)189 102 y(the)f(\014rst)g(group.)75 208 y(Note)k(that)g(for)g(these)h(op)q(erations)g(the)f(order)h(of)f (pro)q(cesses)h(in)g(the)g(output)f(group)h(is)g(determined)75 265 y(primarily)g(b)o(y)f(order)g(in)h(the)f(\014rst)g(group)f(\(if)h (p)q(ossible\))i(and)e(then,)h(if)f(necessary)l(,)h(b)o(y)f(order)g(in) h(the)75 321 y(second)c(group.)j(Neither)d(union)h(nor)e(in)o (tersection)h(are)e(comm)o(utativ)o(e,)h(but)g(b)q(oth)g(are)g(asso)q (ciativ)o(e.)75 428 y Fs(MPI)p 160 428 14 2 v 16 w(GROUP)p 328 428 V 18 w(INCL\(group,)e(n,)j(ranks,)f(newgroup\))117 509 y Ft(IN)155 b Fs(group)479 b Ft(group)14 b(\(handle\))117 590 y(IN)155 b Fs(n)564 b Ft(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g (in)f(arra)o(y)h(ranks)g(\(and)g(size)h(of)e Fv(new-)905 647 y(group)p Ft(\))f(\(in)o(teger\))117 728 y(IN)155 b Fs(ranks)487 b Ft(ranks)13 b(of)f(pro)q(cesses)i(in)e Fv(group)h Ft(to)f(app)q(ear)h(in)f Fv(newgroup)i Ft(\(ar-)905 785 y(ra)o(y)g(of)f(in)o(tegers\))117 867 y(OUT)108 b Fs(newgroup)405 b Ft(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v)o (e,)h(in)g(the)h(order)h(de\014ned)f(b)o(y)905 923 y Fv(ranks)j Ft(\(handle\))75 1051 y FB(int)23 b(MPI)p 245 1051 15 2 v 17 w(Group)p 382 1051 V 17 w(incl\(MPI)p 591 1051 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1051 V 17 w(Group)h(*newgroup\))75 1140 y(MPI)p 150 1140 V 17 w(GROUP)p 287 1140 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 1197 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 1287 y FD(The)17 b(function)h Fs(MPI)p 526 1287 14 2 v 16 w(GROUP)p 694 1287 V 18 w(INCL)f FD(creates)g(a)g(group)f Fs(newgroup)i FD(that)f(consists)g(of)g(the)g Fs(n)h FD(pro-)75 1343 y(cesses)e(in)h Fs(group)f FD(with)h(ranks)e Fs(rank[0],)p Fw(:)8 b(:)g(:)n Fs(,)16 b(rank[n-1])p FD(;)g(the)h(pro)q(cess)f(with)g(rank)g Fs(i)g FD(in)h Fs(newgroup)f FD(is)h(the)75 1400 y(pro)q(cess)e(with)f(rank)h Fs(ranks[i])g FD(in)g Fs(group)p FD(.)k(Eac)o(h)c(of)f(the)g Fs(n)h FD(elemen)o(ts)g(of)f Fs(ranks)h FD(m)o(ust)f(b)q(e)h(a)f(v)m (alid)i(rank)e(in)75 1456 y Fs(group)j FD(and)f(all)i(elemen)o(ts)f(m)o (ust)g(b)q(e)g(distinct,)h(or)e(else)h(the)g(program)f(is)h(erroneous.) 24 b(If)17 b Fs(n)g FD(=)32 b(0,)17 b(then)75 1512 y Fs(newgroup)i FD(is)f(empt)o(y)l(.)29 b(This)18 b(function)h(can,)g (for)e(instance,)i(b)q(e)g(used)g(to)e(reorder)h(the)g(elemen)o(ts)h (of)e(a)75 1569 y(group.)75 1676 y Fs(MPI)p 160 1676 V 16 w(GROUP)p 328 1676 V 18 w(EX)o(CL\(group,)d(n,)h(ranks,)g (newgroup\))117 1756 y Ft(IN)155 b Fs(group)479 b Ft(group)14 b(\(handle\))117 1838 y(IN)155 b Fs(n)564 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 1920 y(IN)155 b Fs(ranks)487 b Ft(arra)o(y)15 b(of)f(in)o(teger)i (ranks)f(in)g Fv(group)g Ft(not)g(to)g(app)q(ear)g(in)g Fv(new-)905 1976 y(group)117 2058 y Ft(OUT)108 b Fs(newgroup)405 b Ft(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 2114 y(de\014ned)d(b)o(y)28 b Fv(group)14 b Ft(\(handle\))75 2242 y FB(int)23 b(MPI)p 245 2242 15 2 v 17 w(Group)p 382 2242 V 17 w(excl\(MPI)p 591 2242 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 2242 V 17 w(Group)h(*newgroup\))75 2332 y(MPI)p 150 2332 V 17 w(GROUP)p 287 2332 V 16 w(EXCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 2388 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 2478 y FD(The)12 b(function)h Fs(MPI)p 516 2478 14 2 v 16 w(GROUP)p 684 2478 V 17 w(EX)o(CL)f FD(creates)g(a)f(group)h(of)g(pro)q(cesses)g Fs(newgroup)h FD(that)e(is)h(obtained)75 2534 y(b)o(y)k(deleting)i(from)e Fs(group)g FD(those)g(pro)q(cesses)h(with)f(ranks)g Fs(ranks[0])h(,)p Fw(:)8 b(:)g(:)14 b Fs(ranks[n-1])p FD(.)24 b(The)17 b(ordering)f(of)75 2591 y(pro)q(cesses)f(in)h Fs(newgroup)g FD(is)f(iden)o(tical)i(to)e(the)g(ordering)g(in)h Fs(group)p FD(.)j(Eac)o(h)c(of)g(the)g Fs(n)g FD(elemen)o(ts)h(of)e Fs(ranks)75 2647 y FD(m)o(ust)h(b)q(e)h(a)f(v)m(alid)i(rank)e(in)h Fs(group)f FD(and)h(all)g(elemen)o(ts)g(m)o(ust)f(b)q(e)h(distinct;)g (otherwise,)f(the)g(program)f(is)75 2704 y(erroneous.)20 b(If)15 b Fs(n)h FD(=)28 b(0,)14 b(then)i Fs(newgroup)g FD(is)g(iden)o(tical)h(to)d Fs(group)p FD(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 130 137 130 136 bop 75 -100 a FD(130)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(RANGE)p 494 45 V 17 w(INCL\(group,)f(n,)h(ranges,)g(newgroup\))117 122 y Ft(IN)155 b Fs(group)479 b Ft(group)14 b(\(handle\))117 197 y(IN)155 b Fs(n)564 b Ft(n)o(um)o(b)q(er)13 b(of)h(triplets)g(in)f (arra)o(y)28 b Fv(ranges)14 b Ft(\(in)o(teger\))117 272 y(IN)155 b Fs(ranges)466 b Ft(a)11 b(one-dimensional)e(arra)o(y)i(of)g (in)o(teger)g(triplets,)h(of)f(the)h(form)905 328 y(\(\014rst)i(rank,)e (last)g(rank,)h(stride\))g(indicating)e(ranks)i(in)f Fv(group)905 385 y Ft(or)i(pro)q(cesses)j(to)c(b)q(e)i(included)f(in)f Fv(newgroup)117 460 y Ft(OUT)108 b Fs(newgroup)405 b Ft(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v)o(e,)h(in)g(the)h (order)h(de\014ned)f(b)o(y)905 516 y Fv(ranges)j Ft(\(handle\))75 641 y FB(int)23 b(MPI)p 245 641 15 2 v 17 w(Group)p 382 641 V 17 w(range)p 519 641 V 16 w(incl\(MPI)p 727 641 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 697 y(MPI)p 468 697 V 17 w(Group)g(*newgroup\))75 784 y(MPI)p 150 784 V 17 w(GROUP)p 287 784 V 16 w(RANGE)p 423 784 V 17 w(INCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 840 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 926 y FD(If)31 b Fs(ranges)15 b FD(consist)g(of)g(the)h(triplets)189 1019 y(\()p Fw(f)5 b(ir)q(st)309 1026 y Fi(1)328 1019 y Fw(;)j(l)q(ast)425 1026 y Fi(1)444 1019 y Fw(;)g(str)q(ide)585 1026 y Fi(1)604 1019 y FD(\))p Fw(;)g(:::;)g FD(\()p Fw(f)d(irst)822 1026 y Ff(n)844 1019 y Fw(;)j(l)q(ast)941 1026 y Ff(n)964 1019 y Fw(;)g(str)q(ide)1105 1026 y Ff(n)1127 1019 y FD(\))75 1111 y(then)16 b Fs(newgroup)f FD(consists)h(of)f(the)g (sequence)h(of)f(pro)q(cesses)h(in)g Fs(group)f FD(with)g(ranks)189 1222 y Fw(f)5 b(ir)q(st)291 1229 y Fi(1)311 1222 y Fw(;)j(f)d(ir)q(st) 434 1229 y Fi(1)463 1222 y FD(+)10 b Fw(str)q(ide)628 1229 y Fi(1)648 1222 y Fw(;)e(:::;)g(f)d(ir)q(st)831 1229 y Fi(1)859 1222 y FD(+)904 1162 y Fe(\026)933 1191 y Fw(l)q(ast)1009 1198 y Fi(1)1039 1191 y FA(\000)11 b Fw(f)5 b(ir)q(st)1187 1198 y Fi(1)p 933 1211 274 2 v 1000 1253 a Fw(str)q(ide)1120 1260 y Fi(1)1212 1162 y Fe(\027)1243 1222 y Fw(str)q(ide)1363 1229 y Fi(1)1383 1222 y Fw(;)j(:::)189 1375 y(f)d(ir)q(st)291 1382 y Ff(n)314 1375 y Fw(;)j(f)d(ir)q(st)437 1382 y Ff(n)471 1375 y FD(+)10 b Fw(str)q(ide)636 1382 y Ff(n)659 1375 y Fw(;)e(:::;)g(f)d(ir) q(st)842 1382 y Ff(n)874 1375 y FD(+)919 1315 y Fe(\026)948 1344 y Fw(l)q(ast)1024 1351 y Ff(n)1058 1344 y FA(\000)10 b Fw(f)5 b(ir)q(st)1205 1351 y Ff(n)p 948 1364 282 2 v 1017 1406 a Fw(str)q(ide)1137 1413 y Ff(n)1234 1315 y Fe(\027)1266 1375 y Fw(str)q(ide)1386 1382 y Ff(n)1409 1375 y Fw(:)75 1496 y FD(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q (e)i(a)f(v)m(alid)h(rank)f(in)h Fs(group)f FD(and)g(all)h(computed)f (ranks)g(m)o(ust)f(b)q(e)i(distinct,)75 1553 y(else)18 b(the)e(program)g(is)h(erroneous.)24 b(Note)16 b(that)g(w)o(e)h(ma)o(y) f(ha)o(v)o(e)g Fw(f)5 b(ir)q(st)1288 1560 y Ff(i)1317 1553 y Fw(>)16 b(l)q(ast)1444 1560 y Ff(i)1458 1553 y FD(,)h(and)g Fw(str)q(ide)1698 1560 y Ff(i)1728 1553 y FD(ma)o(y)f(b)q(e)75 1609 y(negativ)o(e,)f(but)g(cannot)g(b)q(e)h (zero.)166 1666 y(The)i(functionalit)o(y)h(of)e(this)i(routine)f(is)g (sp)q(eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i(the) f(arra)o(y)75 1722 y(of)g(ranges)g(to)g(an)h(arra)o(y)e(of)h(the)h (included)i(ranks)d(and)h(passing)g(the)g(resulting)g(arra)o(y)e(of)i (ranks)f(and)75 1779 y(other)e(argumen)o(ts)g(to)g Fs(MPI)p 560 1779 14 2 v 16 w(GROUP)p 728 1779 V 18 w(INCL)p FD(.)g(A)g(call)i (to)e Fs(MPI)p 1148 1779 V 16 w(GROUP)p 1316 1779 V 18 w(INCL)g FD(is)h(equiv)m(alen)o(t)h(to)e(a)h(call)75 1835 y(to)e Fs(MPI)p 216 1835 V 16 w(GROUP)p 384 1835 V 18 w(RANGE)p 550 1835 V 18 w(INCL)g FD(with)i(eac)o(h)f(rank)g Fs(i)g FD(in)h Fs(ranks)f FD(replaced)i(b)o(y)e(the)g(triplet)h FB(\(i,i,1\))e FD(in)75 1891 y(the)g(argumen)o(t)g Fs(ranges)p FD(.)75 1995 y Fs(MPI)p 160 1995 V 16 w(GROUP)p 328 1995 V 18 w(RANGE)p 494 1995 V 17 w(EX)o(CL\(group,)g(n,)g(ranges,)g (newgroup\))117 2072 y Ft(IN)155 b Fs(group)479 b Ft(group)14 b(\(handle\))117 2147 y(IN)155 b Fs(n)564 b Ft(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 2222 y(IN)155 b Fs(ranges)466 b Ft(a)13 b(one-dimensional)d(arra)o(y)j (of)f(in)o(teger)h(triplets)g(of)f(the)i(form)905 2278 y(\(\014rst)j(rank,)f(last)g(rank,)g(stride\),)h(indicating)e(the)i (ranks)f(in)905 2335 y Fv(group)22 b Ft(of)f(pro)q(cesses)j(to)d(b)q(e) h(excluded)g(from)e(the)i(output)905 2391 y(group)14 b Fv(newgroup)p Ft(.)117 2466 y(OUT)108 b Fs(newgroup)405 b Ft(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 2523 y(in)c Fv(group)g Ft(\(handle\))75 2647 y FB(int)23 b(MPI)p 245 2647 15 2 v 17 w(Group)p 382 2647 V 17 w(range)p 519 2647 V 16 w(excl\(MPI)p 727 2647 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 2704 y(MPI)p 468 2704 V 17 w(Group)g(*newgroup\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 131 138 131 137 bop 75 -100 a Fp(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b FD(131)75 45 y FB(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(RANGE)p 423 45 V 17 w(EXCL\(GROUP,)22 b(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 102 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 191 y FD(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q(e)i(a)f(v)m(alid) h(rank)f(in)h Fs(group)f FD(and)g(all)h(computed)f(ranks)g(m)o(ust)f(b) q(e)i(distinct,)75 247 y(or)i(else)h(the)f(program)f(is)i(erroneous.) 166 306 y(The)i(functionalit)o(y)h(of)e(this)i(routine)f(is)g(sp)q (eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i(the)f (arra)o(y)75 363 y(of)g(ranges)f(to)h(an)g(arra)o(y)f(of)h(the)g (excluded)i(ranks)e(and)g(passing)h(the)f(resulting)h(arra)o(y)e(of)h (ranks)g(and)75 419 y(other)c(argumen)o(ts)g(to)g Fs(MPI)p 554 419 14 2 v 16 w(GROUP)p 722 419 V 18 w(EX)o(CL)p FD(.)g(A)h(call)h(to)e Fs(MPI)p 1146 419 V 16 w(GROUP)p 1314 419 V 18 w(EX)o(CL)g FD(is)i(equiv)m(alen)o(t)g(to)e(a)h(call)75 476 y(to)g Fs(MPI)p 216 476 V 15 w(GROUP)p 383 476 V 18 w(RANGE)p 549 476 V 18 w(EX)o(CL)g FD(with)h(eac)o(h)f(rank)g Fs(i)g FD(in)h Fs(ranks)g FD(replaced)g(b)o(y)f(the)h(triplet)g FB(\(i,i,1\))e FD(in)75 532 y(the)h(argumen)o(t)g Fs(ranges)p FD(.)166 591 y(Note:)20 b(the)c(range)g(op)q(erations)f(do)h(not)f (explicitly)k(en)o(umerate)d(ranks,)f(and)h(therefore)f(are)g(more)75 648 y(scalable)k(if)f(implemen)o(ted)h(e\016cien)o(tly)l(.)28 b(Hence,)18 b(w)o(e)f(recommend)h(MPI)f(programmers)f(to)h(use)h(them) 75 704 y(whenenev)o(er)e(p)q(ossible,)h(as)d(high-qualit)o(y)j (implemen)o(tations)g(will)g(tak)o(e)d(adv)m(an)o(tage)h(of)f(this)i (fact.)75 841 y Fn(5.3.3)49 b(Group)16 b(Destructo)o(rs)75 979 y Fs(MPI)p 160 979 V 16 w(GROUP)p 328 979 V 18 w(FREE\(group\))117 1059 y Ft(INOUT)62 b Fs(group)479 b Ft(group)14 b(\(handle\))75 1187 y FB(int)23 b(MPI)p 245 1187 15 2 v 17 w(Group)p 382 1187 V 17 w(free\(MPI)p 591 1187 V 16 w(Group)g(*group\))75 1276 y(MPI)p 150 1276 V 17 w(GROUP)p 287 1276 V 16 w(FREE\(GROUP,)g (IERROR\))170 1332 y(INTEGER)g(GROUP,)g(IERROR)166 1421 y FD(This)12 b(op)q(eration)g(marks)f(a)g(group)g(ob)s(ject)g(for)g (deallo)q(cation.)20 b(The)12 b(handle)h Fs(group)f FD(is)g(set)f(to)g (n)o(ull)i(b)o(y)75 1478 y(the)g(call.)20 b(The)13 b(group)f(ob)s(ject) g(is)h(actually)h(deallo)q(cated)g(only)f(if)g(there)g(are)f(no)g (other)h(activ)o(e)f(references)75 1534 y(to)h(it.)20 b(In)15 b(particular,)f(if)h(the)f(group)f(w)o(as)g(used)i(to)e(create) h(a)g(comm)o(unicator,)f(then)i(freeing)f(the)g(group)95 b Fm(CHANGE)75 1591 y FD(do)q(es)16 b(not)e(in)o(v)m(alidate)j(the)f (comm)o(unicator.)166 1732 y Fl(Implemen)o(tati)o(on)d(note:)37 b Ft(One)14 b(can)h(k)o(eep)g(a)f(reference)i(coun)o(t)f(that)f(is)g (incremen)o(ted)g(for)g(eac)o(h)h(call)e(to)75 1789 y Fv(MPI)p 152 1789 13 2 v 14 w(COMM)p 296 1789 V 15 w(MAKE)e Ft(and)h Fv(MPI)p 594 1789 V 14 w(COMM)p 738 1789 V 15 w(DUP)p Ft(,)e(and)i(and)f(decremen)o(ted)i(for)f(eac)o(h)g(call)f(to)h Fv(MPI)p 1608 1789 V 14 w(GROUP)p 1764 1789 V 13 w(FREE)75 1845 y Ft(or)i Fv(MPI)p 203 1845 V 14 w(COMM)p 347 1845 V 15 w(FREE)p Ft(;)e(the)i(group)g(ob)r(ject)g(is)g(ultimately)d (deallo)q(cated)i(when)i(the)f(reference)i(coun)o(t)e(drops)g(to)75 1902 y(zero.)75 2143 y Fq(5.4)59 b(Communicato)n(r)19 b(Management)75 2250 y FD(This)d(section)h(describ)q(es)g(the)f (manipulation)h(of)f(comm)o(unicators)f(in)i(MPI.)e(Op)q(erations)i (that)e(access)75 2306 y(comm)o(unicators)g(are)g(lo)q(cal)h(and)f (their)h(execution)g(do)q(es)g(not)f(require)h(in)o(terpro)q(cess)g (comm)o(unication.)75 2363 y(Op)q(erations)22 b(that)e(create)h(comm)o (unicators)g(are)g(collectiv)o(e)i(and)e(ma)o(y)f(require)i(in)o (terpro)q(cess)g(com-)75 2419 y(m)o(unication.)28 b(\(High)18 b(qualit)o(y)g(implemen)o(tations)g(will)h(b)q(e)g(able)f(to)f (amortize)g(these)h(o)o(v)o(erheads)f(o)o(v)o(er)75 2476 y(sev)o(eral)e(calls.\))75 2613 y Fn(5.4.1)49 b(Comm)o(unicato)o(r)13 b(Accesso)o(rs)75 2704 y FD(The)i(follo)o(wing)h(are)f(all)h(lo)q(cal)h (op)q(erations.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 132 139 132 138 bop 75 -100 a FD(132)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(COMM)p 318 45 V 16 w(SIZE\(comm)m(,)d (size\))117 123 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 200 y(OUT)108 b Fs(size)519 b Ft(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)g(of)27 b Fv(comm)8 b Ft(\(in)o(teger\))75 326 y FB(int)23 b(MPI)p 245 326 15 2 v 17 w(Comm)p 358 326 V 17 w(size\(MPI)p 567 326 V 16 w(Comm)g(comm,)g(int)h(*size\))75 413 y(MPI)p 150 413 V 17 w(COMM)p 263 413 V 16 w(SIZE\(COMM,)f(SIZE,)g(IERROR\))170 470 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 604 y Fs(MPI)p 160 604 14 2 v 16 w(COMM)p 318 604 V 16 w(RANK\(comm)m(,)12 b(rank\))117 683 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 760 y(OUT)108 b Fs(rank)504 b Ft(rank)12 b(of)f(the)h(calling)e(pro)q(cess)j(in)f(group)f(of)23 b Fv(comm)5 b Ft(\(in)o(teger\))75 885 y FB(int)23 b(MPI)p 245 885 15 2 v 17 w(Comm)p 358 885 V 17 w(rank\(MPI)p 567 885 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 973 y(MPI)p 150 973 V 17 w(COMM)p 263 973 V 16 w(RANK\(COMM,)f(RANK,)g(IERROR\))170 1029 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 1164 y Fs(MPI)p 160 1164 14 2 v 16 w(COMM)p 318 1164 V 16 w(GROUP\(comm)n(,)12 b(group\))117 1242 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 1319 y(OUT)108 b Fs(group)479 b Ft(group)14 b(corresp)q(onding)h(to)f Fv(comm)8 b Ft(\(handle\))75 1445 y FB(int)23 b(MPI)p 245 1445 15 2 v 17 w(Comm)p 358 1445 V 17 w(group\(MPI)p 591 1445 V 16 w(Comm)g(comm,)g(MPI)p 941 1445 V 17 w(Group)g(*group\))75 1532 y(MPI)p 150 1532 V 17 w(COMM)p 263 1532 V 16 w(GROUP\(COMM,)g(GROUP,)g(IERROR\))170 1588 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)-245 1625 y Fm(CHANGE)166 1676 y FD(A)10 b(group)g(accessed)h(via)f Fs(MPI)p 663 1676 14 2 v 16 w(COMM)p 821 1676 V 16 w(GROUP)i FD(m)o(ust)d(not)h(need)h(to)f(b)q(e)h(freed)f(via)g Fs(MPI)p 1651 1676 V 16 w(GROUP)p 1819 1676 V 18 w(FREE)p FD(.)75 1732 y(The)17 b(function)g Fs(MPI)p 434 1732 V 16 w(COMM)p 592 1732 V 16 w(GROUP)h FD(merely)f(pro)o(vides)g(access) f(to)g(the)g(group,)g(not)g(a)g(duplicate)i(of)75 1789 y(the)d(group.)75 1916 y Fn(5.4.2)49 b(Comm)o(unicato)o(r)13 b(Constructo)o(rs)75 2004 y FD(The)i(follo)o(wing)g(are)g(collectiv)o (e)h(functions)f(that)f(are)h(in)o(v)o(ok)o(ed)g(b)o(y)f(all)i(pro)q (cesses)f(in)g(the)g(group)f(asso)q(ci-)75 2060 y(ated)h(with)h Fs(comm)m FD(.)166 2118 y(Note)f(that)g(there)g(is)h(a)g(c)o(hic)o(k)o (en-and-egg)g(asp)q(ect)g(to)f(MPI)g(in)i(that)d(a)h(comm)o(unicator)h (is)g(needed)75 2174 y(to)21 b(create)h(a)f(new)h(comm)o(unicator.)39 b(The)22 b(base)f(comm)o(unicator)h(for)f(all)i(MPI)e(comm)o(unicators) g(is)75 2231 y(prede\014ned)c(outside)f(of)e(MPI,)h(and)h(is)f Fs(MPI)p 839 2231 V 16 w(COMM)p 997 2231 V 17 w(W)o(ORLD)g FD(\(de\014ned)h(ab)q(o)o(v)o(e\).)75 2335 y Fs(MPI)p 160 2335 V 16 w(COMM)p 318 2335 V 16 w(DUP\(comm)m(,)c(new)o(comm)n(\)) 117 2414 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 2491 y(OUT)108 b Fs(new)o(comm)393 b Ft(cop)o(y)14 b(of)f Fv(comm)8 b Ft(\(handle\))75 2616 y FB(int)23 b(MPI)p 245 2616 15 2 v 17 w(Comm)p 358 2616 V 17 w(dup\(MPI)p 543 2616 V 16 w(Comm)g(comm,)h(MPI)p 894 2616 V 16 w(Comm)g(*newcomm\))75 2704 y(MPI)p 150 2704 V 17 w(COMM)p 263 2704 V 16 w(DUP\(COMM,)f(NEWCOMM,)g(IERROR\)) 1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 133 140 133 139 bop 75 -100 a Fp(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b FD(133)170 45 y FB(INTEGER)23 b(COMM,)h(NEWCOMM,) e(IERROR)166 175 y Fs(MPI)p 251 175 14 2 v 16 w(COMM)p 409 175 V 16 w(DUP)15 b FD(Duplicates)i(the)e(existing)h(comm)o (unicator)f Fs(comm)10 b FD(with)15 b(cac)o(hed)h(attributes)75 232 y(copied)j(or)f(not)g(copied)i(\(according)e(to)g(the)g(cop)o (y-callbac)o(k)h(p)q(olicy)h(of)e(eac)o(h)g(individual)k(cac)o(hed)c (at-)75 288 y(tribute\).)35 b(Returns)21 b(in)g Fs(new)o(comm)15 b FD(a)20 b(new)g(comm)o(unicator)g(with)g(the)h(same)f(group,)h(same)e (cac)o(hed)75 344 y(information,)c(but)g(a)g(new)h(con)o(text.)166 527 y Fl(Implemen)o(tati)o(on)g(note:)49 b Ft(One)17 b(need)h(not)f(actually)g(cop)o(y)g(the)g(group)g(information,)e(but)i (only)f(add)75 584 y(a)j(new)h(reference)i(and)d(incremen)o(t)h(the)g (reference)i(coun)o(t.)35 b(Cop)o(y)19 b(on)g(write)h(can)f(b)q(e)h (used)h(for)e(the)h(cac)o(hed)75 640 y(information.)75 870 y Fs(MPI)p 160 870 V 16 w(COMM)p 318 870 V 16 w(MAKE\(comm)m(,)12 b(group,)i(new)o(comm)n(\))117 991 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 1153 y(IN)155 b Fs(group)479 b Ft(Group,)14 b(whic)o(h)h(is)f(a)g(subset)i(of)e(the)h (group)g(of)e Fv(comm)c Ft(\(han-)905 1210 y(dle\))117 1372 y(OUT)108 b Fs(new)o(comm)393 b Ft(new)15 b(comm)o(uni)o(cator)d (\(handle\))75 1540 y FB(int)23 b(MPI)p 245 1540 15 2 v 17 w(Comm)p 358 1540 V 17 w(make\(MPI)p 567 1540 V 16 w(Comm)g(comm,)g(MPI)p 917 1540 V 17 w(Group)g(group,)g(MPI)p 1316 1540 V 17 w(Comm)h(*newcomm\))75 1670 y(MPI)p 150 1670 V 17 w(COMM)p 263 1670 V 16 w(MAKE\(COMM,)f(GROUP,)g(NEWCOMM,)g (IERROR\))170 1726 y(INTEGER)g(COMM,)h(GROUP,)f(NEWCOMM,)f(IERROR)75 1856 y FD(This)14 b(function)g(creates)f(a)g(new)h(comm)o(unicator)f Fs(new)o(comm)8 b FD(with)14 b(comm)o(unication)g(group)f(de\014ned)i (b)o(y)75 1913 y Fs(group)f FD(and)h(a)e(new)i(con)o(text.)k(No)14 b(cac)o(hed)h(information)f(propagates)f(from)h Fs(comm)8 b FD(to)14 b Fs(new)o(comm)m FD(.)j(The)75 1969 y(function)f(returns)f (a)g(n)o(ull)i(handle)f(to)f(pro)q(cesses)g(that)g(are)g(not)f(in)i Fs(group)p FD(.)166 2069 y(The)f(call)h(is)f(erroneous)g(if)g(not)f (all)i Fs(group)f FD(argumen)o(ts)f(ha)o(v)o(e)g(the)h(same)f(v)m (alue,)i(or)e(if)h Fs(group)g FD(is)g(not)75 2126 y(a)j(subset)h(of)f (the)h(group)f(asso)q(ciated)h(with)g Fs(comm)m FD(.)27 b(Note)18 b(that)g(the)g(call)i(is)f(to)f(b)q(e)h(executed)h(b)o(y)e (all)75 2182 y(pro)q(cesses)e(in)g Fs(comm)m FD(,)c(ev)o(en)j(if)h (they)f(do)g(not)g(b)q(elong)h(to)f(the)g(new)g(group.)166 2365 y Fl(Implemen)o(tati)o(on)e(note:)40 b Ft(Since)15 b(all)e(pro)q(cesses)18 b(calling)13 b Fv(MPI)p 1215 2365 13 2 v 14 w(COMM)p 1359 2365 V 15 w(DUP)h Ft(or)g Fv(MPI)p 1600 2365 V 15 w(COMM)p 1745 2365 V 14 w(MAKE)75 2421 y Ft(pro)o(vide)d(the)g(same)f Fv(group)h Ft(argumen)o(t,)f(it)g (is)h(theoretically)g(p)q(ossible)g(to)f(agree)i(on)e(a)h(group-wide)f (unique)h(con)o(text)75 2478 y(with)h(no)h(comm)o(unication.)h(Ho)o(w)o (ev)o(er,)f(lo)q(cal)f(execution)i(of)e(these)i(functions)f(requires)h (use)g(of)e(a)g(larger)h(con)o(text)75 2534 y(name)h(space)i(and)e (reduces)j(error-c)o(hec)o(king.)22 b(Implemen)o(tations)12 b(ma)o(y)h(strik)o(e)i(v)n(arious)g(compromises)e(b)q(et)o(w)o(een)75 2591 y(these)k(con\015icting)f(goals.)23 b(If)16 b(new)g(comm)o (unicators)e(are)i(created)i(without)d(sync)o(hronizing)h(the)h(pro)q (cesses)i(in-)75 2647 y(v)o(olv)o(ed)d(then)i(the)g(comm)o(ucatio)o(n)d (system)h(should)h(b)q(e)h(able)f(to)f(cop)q(e)i(with)f(messages)g (arriving)f(in)h(a)f(con)o(text)75 2704 y(that)e(has)g(not)g(y)o(et)g (b)q(een)h(allo)q(cated)e(at)h(the)h(receiving)f(pro)q(cess.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 134 141 134 140 bop 75 -100 a FD(134)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(COMM)p 318 45 V 16 w(SPLIT\(com)o(m)n (,)c(colo)o(r,)j(k)o(ey)l(,)h(new)o(comm)n(\))117 128 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 215 y(IN)155 b Fs(colo)o(r)495 b Ft(con)o(trol)14 b(of)f(subset)i (assignmen)o(t)e(\(in)o(teger\))117 302 y(IN)155 b Fs(k)o(ey)525 b Ft(con)o(trol)14 b(of)f(rank)h(assigmen)o(t)f(\(in)o(teger\))117 389 y(OUT)108 b Fs(new)o(comm)393 b Ft(new)15 b(comm)o(uni)o(cator)d (\(handle\))75 520 y FB(int)23 b(MPI)p 245 520 15 2 v 17 w(Comm)p 358 520 V 17 w(split\(MPI)p 591 520 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 520 V 16 w(Comm)g(*newcomm\))75 612 y(MPI)p 150 612 V 17 w(COMM)p 263 612 V 16 w(SPLIT\(COMM,)f(COLOR,)g(KEY,)g(NEWCOMM,)g (IERROR\))170 669 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g(NEWCOMM,)g (IERROR)75 761 y FD(This)c(function)f(partitions)g(the)h(group)e(asso)q (ciated)h(with)h Fs(comm)12 b FD(in)o(to)18 b(disjoin)o(t)g(subgroups,) h(one)f(for)75 817 y(eac)o(h)g(v)m(alue)g(of)f Fs(colo)o(r)p FD(.)26 b(Eac)o(h)17 b(subgroup)h(con)o(tains)f(all)i(pro)q(cesses)f (of)f(the)g(same)g(color.)27 b(Within)19 b(eac)o(h)75 874 y(subgroup,)24 b(the)f(pro)q(cesses)g(are)f(rank)o(ed)h(in)g(the)g (order)g(de\014ned)h(b)o(y)e(the)h(v)m(alue)h(of)e(the)h(argumen)o(t)75 930 y Fs(k)o(ey)p FD(,)d(with)g(ties)g(brok)o(en)g(according)g(to)f (their)h(rank)f(in)i(the)f(old)g(group.)33 b(A)19 b(new)h(comm)o (unicator)f(is)75 987 y(created)14 b(for)f(eac)o(h)h(subgroup)g(and)g (returned)g(in)g Fs(new)o(comm)n FD(.)i(A)e(pro)q(cess)g(ma)o(y)f (supply)i(the)f(color)g(v)m(alue)75 1043 y Fs(MPI)p 160 1043 14 2 v 16 w(UNDEFINED)p FD(,)i(in)h(whic)o(h)h(case)f Fs(new)o(comm)11 b FD(returns)16 b(a)g(n)o(ull)i(comm)o(unicator)f (handle.)25 b(This)17 b(is)g(a)75 1100 y(collectiv)o(e)g(call,)f(but)f (eac)o(h)h(pro)q(cess)f(can)g(pro)o(vide)h(di\013eren)o(t)f(v)m(alues)i (for)d Fs(colo)o(r)g FD(and)i Fs(k)o(ey)p FD(.)166 1162 y(A)27 b(call)i(to)d Fs(MPI)p 475 1162 V 16 w(COMM)p 633 1162 V 17 w(MAKE\(com)o(m)m(,)h(group,)j(new)o(comm)n(\))24 b FD(is)j(equiv)m(alen)o(t)i(to)e(a)g(call)i(to)75 1219 y Fs(MPI)p 160 1219 V 16 w(COMM)p 318 1219 V 16 w(SPLIT\(com)o(m)n(,)7 b(colo)o(r,)j(k)o(ey)l(,)h(new)o(comm)m(\))p FD(,)d(where)i(all)h(mem)o (b)q(ers)g(of)e Fs(group)h FD(pro)o(vide)h Fs(colo)o(r)e FD(=)23 b(0)75 1275 y(and)15 b Fs(k)o(ey)f FD(=)29 b(rank)15 b(in)g Fs(group)p FD(,)f(and)h(all)g(pro)q(cesses)g(that)f(are)g(not)g (mem)o(b)q(ers)h(of)f Fs(group)g FD(pro)o(vide)h Fs(colo)o(r)e FD(=)75 1331 y Fs(MPI)p 160 1331 V 16 w(UNDEFINED)p FD(.)k(The)i (function)f Fs(MPI)p 819 1331 V 16 w(COMM)p 977 1331 V 17 w(SPLIT)f FD(allo)o(ws)h(more)f(general)i(partitioning)g(of)e(a)75 1388 y(group)e(in)o(to)g(one)g(or)g(more)g(subgroups)g(with)h(optional) f(reordering.)75 1544 y Fn(5.4.3)49 b(Comm)o(unicato)o(r)13 b(Destructo)o(rs)75 1688 y Fs(MPI)p 160 1688 V 16 w(COMM)p 318 1688 V 16 w(FREE\(comm)n(\))117 1771 y Ft(INOUT)62 b Fs(comm)466 b Ft(comm)o(unicator)11 b(to)j(b)q(e)h(destro)o(y)o(ed)g (\(handle\))75 1902 y FB(int)23 b(MPI)p 245 1902 15 2 v 17 w(Comm)p 358 1902 V 17 w(free\(MPI)p 567 1902 V 16 w(Comm)g(*comm\))75 1994 y(MPI)p 150 1994 V 17 w(COMM)p 263 1994 V 16 w(FREE\(COMM,)g(IERROR\))170 2051 y(INTEGER)g(COMM,)h (IERROR)166 2143 y FD(This)13 b(collectiv)o(e)g(op)q(eration)g(marks)e (the)h(comm)o(unication)h(ob)s(ject)f(for)f(deallo)q(cation.)20 b(The)13 b(handle)75 2199 y(is)i(set)g(to)f(n)o(ull.)21 b(The)15 b(ob)s(ject)f(is)h(actually)g(deallo)q(cated)h(only)g(if)f (there)f(are)h(no)f(other)h(activ)o(e)f(references)75 2256 y(to)h(it.)20 b(The)15 b(asso)q(ciated)h(group)e(ma)o(y)h(also)g (b)q(e)h(deallo)q(cated,)g(if)g(there)f(are)g(no)g(activ)o(e)h (references)f(to)g(it.)166 2394 y Fl(Implemen)o(tati)o(on)g(note:)47 b Ft(A)17 b(reference)i(coun)o(t)e(mec)o(hanism)d(can)j(b)q(e)g(used:) 24 b(the)17 b(reference)i(coun)o(t)e(is)75 2444 y(incremen)o(ted)d(b)o (y)g(eac)o(h)g(call)f(to)g Fv(MPI)p 663 2444 13 2 v 15 w(COMM)p 808 2444 V 14 w(DUP)p Ft(,)g(and)g(decremen)o(ted)i(b)o(y)f (eac)o(h)g(call)f(to)g Fv(MPI)p 1606 2444 V 15 w(COMM)p 1751 2444 V 14 w(FREE)p Ft(.)75 2494 y(The)h(ob)r(ject)h(is)f (ultimately)d(deallo)q(cated)j(when)h(the)f(coun)o(t)g(reac)o(hes)i (zero.)166 2556 y(Though)i(collectiv)o(e,)h(it)f(is)g(an)o(ticipated)g (that)h(this)f(op)q(eration)g(will)f(normally)f(b)q(e)j(implemen)o(ted) d(to)i(b)q(e)75 2613 y(lo)q(cal,)13 b(though)g(the)i(debugging)e(v)o (ersion)h(of)f(an)h(MPI)g(library)f(migh)o(t)f(c)o(ho)q(ose)j(to)e (sync)o(hronize.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 135 142 135 141 bop 75 -100 a Fp(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FD(135)75 45 y Fq(5.5)59 b(Inter-Communication)75 151 y FD(This)18 b(section)h(in)o(tro)q(duces)f(the)g(concept)g(of)f(in)o (ter-comm)o(unication)i(and)f(describ)q(es)i(the)d(p)q(ortions)h(of)75 208 y(MPI)j(that)g(supp)q(ort)g(it.)38 b(It)22 b(describ)q(es)h(supp)q (ort)e(for)g(writing)g(programs)f(that)h(con)o(tain)g(user-lev)o(el)75 264 y(serv)o(ers.)h(It)16 b(also)f(describ)q(es)j(a)e(name)g(service)g (that)g(simpli\014es)i(writing)e(programs)f(con)o(taining)i(in)o(ter-) 75 321 y(comm)o(unication.)166 380 y(All)h(p)q(oin)o(t-to-p)q(oin)o(t)g (comm)o(unication)g(describ)q(ed)h(th)o(us)d(far)h(has)g(in)o(v)o(olv)o (ed)g(comm)o(unication)h(b)q(e-)75 436 y(t)o(w)o(een)c(pro)q(cesses)g (that)f(are)h(mem)o(b)q(ers)g(of)f(the)h(same)g(group.)19 b(This)c(t)o(yp)q(e)f(of)f(comm)o(unication)i(is)f(called)75 493 y(\\in)o(tra-comm)o(unication")h(and)h(the)f(comm)o(unicator)g (used)h(is)f(called)i(an)e(\\in)o(tra-comm)o(unicator.")166 552 y(In)c(mo)q(dular)g(and)f(m)o(ulti-discipl)q(inary)j(applications,) g(di\013eren)o(t)d(pro)q(cess)h(groups)f(execute)h(distinct)75 608 y(mo)q(dules)i(and)f(pro)q(cesses)h(within)g(di\013eren)o(t)f(mo)q (dules)h(comm)o(unicate)f(with)h(one)f(another)f(in)i(a)f(pip)q(eline) 75 664 y(or)j(a)f(more)h(general)h(mo)q(dule)g(graph.)j(In)d(these)f (applications,)i(the)e(most)f(natural)h(w)o(a)o(y)f(for)h(a)g(pro)q (cess)75 721 y(to)g(sp)q(ecify)i(a)e(target)f(pro)q(cess)i(is)g(b)o(y)g (the)g(rank)f(of)g(the)h(target)e(pro)q(cess)i(within)h(the)e(target)g (group.)20 b(In)75 777 y(applications)13 b(that)e(con)o(tain)h(in)o (ternal)h(user-lev)o(el)g(serv)o(ers,)f(eac)o(h)f(serv)o(er)h(ma)o(y)f (b)q(e)h(a)f(pro)q(cess)h(group)g(that)75 834 y(pro)o(vides)17 b(services)f(to)g(one)g(or)g(more)f(clien)o(ts,)i(and)g(eac)o(h)f (clien)o(t)h(ma)o(y)f(b)q(e)g(a)g(pro)q(cess)h(group)e(that)h(uses)75 890 y(the)h(services)h(of)f(one)g(or)g(more)g(serv)o(ers.)25 b(It)17 b(is)h(again)f(most)f(natural)i(to)e(sp)q(ecify)j(the)e(target) f(pro)q(cess)75 947 y(b)o(y)d(rank)g(within)i(the)e(target)f(group)i (in)g(these)f(applications.)21 b(This)14 b(t)o(yp)q(e)g(of)e(comm)o (unication)j(is)e(called)75 1003 y(\\in)o(ter-comm)o(unication")j(and)f (the)g(comm)o(unicator)g(used)h(is)g(called)h(an)e(\\in)o(ter-comm)o (unicator.")166 1062 y(An)23 b(in)o(ter-comm)o(unication)g(op)q (eration)g(is)f(a)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unication)g(b)q (et)o(w)o(een)g(pro-)75 1119 y(cesses)16 b(in)h(di\013eren)o(t)f (groups.)21 b(The)16 b(group)f(con)o(taining)h(a)g(pro)q(cess)g(that)f (initiates)i(an)e(in)o(ter-comm)o(un-)75 1175 y(ication)i(op)q(eration) f(is)g(called)h(the)f(\\lo)q(cal)h(group,")e(that)g(is,)h(the)g(sender) h(in)f(a)g(send)g(and)g(the)g(receiv)o(er)75 1232 y(in)g(a)e(receiv)o (e.)21 b(The)15 b(group)g(con)o(taining)g(the)g(target)f(pro)q(cess)h (is)g(called)i(the)e(\\remote)f(group,")g(that)g(is,)75 1288 y(the)j(receiv)o(er)i(in)f(a)f(send)h(and)f(the)h(sender)g(in)g(a) f(receiv)o(e.)27 b(As)17 b(in)i(in)o(tra-comm)o(unication,)f(the)f (target)75 1344 y(pro)q(cess)12 b(is)g(sp)q(eci\014ed)i(using)e(a)f Fs(\(comm)m(unicato)o(r,)e(rank\))j FD(pair.)19 b(Unlik)o(e)13 b(in)o(tra-comm)o(unication,)g(the)e(rank)75 1401 y(is)16 b(relativ)o(e)f(to)g(a)g(second,)g(remote)g(group.)166 1460 y(Lo)q(cal)g(and)g(remote)e(groups)h(of)g(an)g(in)o(ter-comm)o (unicator)h(are)f(not)f(required)j(to)d(b)q(e)i(disjoin)o(t.)21 b(An)75 1516 y(in)o(tra-comm)o(unicator)14 b(can)h(b)q(e)h(seen)f(as)f (a)g(particular)i(case)e(of)h(an)f(in)o(ter-comm)o(unicator,)h(where)g (lo)q(cal)75 1573 y(and)h(remote)g(groups)f(happ)q(en)j(to)d(b)q(e)i (iden)o(tical.)24 b(The)16 b(seman)o(tics)h(of)e(accessor)h(function)h (are)f(de\014ned)75 1629 y(so)k(as)g(to)g(b)q(e)h(consisten)o(t)g(with) g(this)g(in)o(terpretation.)36 b(Here)20 b(is)h(a)g(summary)f(of)g(the) g(prop)q(erties)h(of)75 1686 y(in)o(ter-comm)o(unication)16 b(and)g(in)o(ter-comm)o(unicators:)143 1789 y FA(\017)23 b FD(The)c(syn)o(tax)g(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)g(is)g(the)f(same)g(for)g(b)q(oth)h(in)o(ter-)f(and)h(in)o (tra-)189 1846 y(comm)o(unication.)26 b(The)17 b(same)g(comm)o (unicator)g(can)g(b)q(e)g(used)h(b)q(oth)f(for)g(send)g(and)h(for)e (receiv)o(e)189 1902 y(op)q(erations.)143 2006 y FA(\017)23 b FD(A)14 b(target)g(pro)q(cess)h(is)g(addressed)h(b)o(y)f(its)g(rank)f (in)i(the)f(remote)f(group,)g(b)q(oth)h(for)f(sends)i(and)f(for)189 2063 y(receiv)o(es.)143 2166 y FA(\017)23 b FD(Comm)o(unications)12 b(using)i(an)e(in)o(ter-comm)o(unicator)h(are)f(guaran)o(teed)g(not)g (to)g(con\015ict)h(with)g(an)o(y)189 2223 y(comm)o(unications)i(that)g (use)h(a)e(di\013eren)o(t)i(comm)o(unicator.)143 2327 y FA(\017)23 b FD(An)15 b(in)o(ter-comm)o(unicator)g(cannot)g(b)q(e)h (used)g(for)f(collectiv)o(e)i(comm)o(unication.)143 2430 y FA(\017)23 b FD(A)15 b(comm)o(unicator)g(will)i(pro)o(vide)e(either)h (in)o(tra-)f(or)g(in)o(ter-comm)o(unication,)h(nev)o(er)f(b)q(oth.)75 2534 y(The)i(routine)f Fs(MPI)p 412 2534 14 2 v 16 w(COMM)p 570 2534 V 17 w(TEST)p 701 2534 V 16 w(INTER)h FD(ma)o(y)e(b)q(e)i (used)g(to)f(determine)h(if)g(a)f(comm)o(unicator)g(is)h(an)75 2591 y(in)o(ter-)i(or)e(in)o(tra-comm)o(unicator.)29 b(In)o(ter-comm)o(unicators)18 b(can)g(b)q(e)h(used)g(as)f(argumen)o (ts)f(to)g(some)h(of)75 2647 y(the)f(other)f(comm)o(unicator)h(access)g (routines.)25 b(In)o(ter-comm)o(unicators)17 b(cannot)f(b)q(e)i(used)f (as)g(input)h(to)75 2704 y(an)o(y)d(of)g(the)g(constructor)f(routines)i (for)f(in)o(tra-comm)o(unicators.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 136 143 136 142 bop 75 -100 a FD(136)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)166 45 y Fl(Implemen)o(tati)o(on)e(note:)38 b Ft(Comm)o(unicators)12 b(can)i(b)q(e)h(represen)o(ted)j(in)c(eac)o(h)h(pro)q(cess)h(b)o(y)e(a) g(tuple)h(con-)75 95 y(sisting)f(of:)75 180 y Fl(group)75 269 y(send)p 173 269 15 2 v 16 w(con)o(text)75 358 y(receiv)o(e)p 223 358 V 16 w(con)o(text)75 446 y(source)75 531 y Ft(F)m(or)f(in)o (ter-comm)o(unicators,)e Fl(group)g Ft(describ)q(es)16 b(the)e(remote)f(group,)g(and)g Fl(source)f Ft(is)h(the)h(rank)f(of)g (the)h(pro)q(cess)75 581 y(in)j(the)i(lo)q(cal)e(group.)30 b(F)m(or)17 b(in)o(tra-comm)o(unicators,)f Fl(group)g Ft(is)i(the)g(comm)o(unicator)d(group)j(\(remote=lo)q(cal\),)75 631 y Fl(source)h Ft(is)h(the)g(rank)g(of)f(the)i(pro)q(cess)h(in)d (this)h(group,)h(and)f Fl(send)h(con)o(text)e Ft(and)h Fl(receiv)o(e)h(con)o(text)e Ft(are)75 681 y(iden)o(tical.)e(A)d(group) g(is)g(represen)o(ted)i(b)o(y)e(a)g(rank-to-absolute-address)g (translation)g(table.)166 734 y(The)e(in)o(ter-comm)o(unicator)c (cannot)j(b)q(e)h(discussed)h(sensibly)e(without)g(considering)h(pro)q (cesses)h(in)e(b)q(oth)g(the)75 784 y(lo)q(cal)k(and)g(remote)h (groups.)23 b(Imagine)14 b(a)i(pro)q(cess)h Fl(P)e Ft(in)h(group)f Fj(P)s Ft(,)h(whic)o(h)g(has)f(an)h(in)o(ter-comm)o(unicator)d Fl(C)1835 790 y Fa(P)1863 784 y Ft(,)75 834 y(and)j(a)h(pro)q(cess)h Fl(Q)e Ft(in)g(group)h Fj(Q)p Ft(,)f(whic)o(h)g(has)h(an)f(in)o (ter-comm)o(unicator)e Fl(C)1293 840 y Fa(Q)1323 834 y Ft(.)26 b(\(Note)17 b(that)f Fj(P)k Ft(and)c Fj(Q)h Ft(do)f(not)75 884 y(ha)o(v)o(e)e(to)f(b)q(e)i(distinct.\))j(Then)145 969 y Fj(\017)23 b Fl(C)223 975 y Fa(P)251 969 y Fl(.group)13 b Ft(describ)q(es)j(the)e(group)g Fj(Q)g Ft(and)f Fl(C)927 975 y Fa(Q)957 969 y Fl(.group)g Ft(describ)q(es)j(the)e(group)g Fj(P)s Ft(.)145 1058 y Fj(\017)23 b Fl(C)223 1064 y Fa(P)251 1058 y Fl(.send)p 362 1058 V 16 w(con)o(text)15 b(=)h Ft(C)629 1064 y Fa(Q)658 1058 y Fl(.receiv)o(e)p 819 1058 V 16 w(con)o(text)c Ft(and)i(the)h(con)o(text)f(is)g(unique)g(in)f Fj(Q)p Ft(;)189 1107 y Fl(C)223 1113 y Fa(P)251 1107 y Fl(.receiv)o(e)p 412 1107 V 16 w(con)o(text)i(=)32 b(C)699 1113 y Fa(Q)728 1107 y Fl(.send)p 839 1107 V 17 w(con)o(text)12 b Ft(and)i(this)g(con)o(text)g(is)g(unique)g(in)f Fj(P)s Ft(.)145 1196 y Fj(\017)23 b Fl(C)223 1202 y Fa(P)251 1196 y Fl(.source)13 b Ft(is)h(rank)g(of)f Fl(P)h Ft(in)f Fj(P)k Ft(and)d Fl(C)851 1202 y Fa(Q)881 1196 y Fl(.source)f Ft(is)g(rank)h(of)f Fl(Q)h Ft(in)g Fj(Q)p Ft(.)166 1281 y(Assume)g(that)h Fl(P)f Ft(sends)i(a)f(message)f(to)g Fl(Q)h Ft(using)f(the)i(in)o(ter-comm)o(unicator.)h(Then)e Fl(P)f Ft(uses)i(the)g Fl(group)75 1331 y Ft(table)e(to)g(\014nd)g(the) g(absolute)g(address)h(of)e Fl(Q)p Ft(;)h Fl(source)e Ft(and)i Fl(send)p 1129 1331 V 16 w(con)o(text)e Ft(are)j(app)q(ended)g (to)e(the)i(message.)166 1384 y(Assume)d(that)g Fl(Q)g Ft(p)q(osts)h(a)f(receiv)o(e)i(with)e(an)g(explicit)g(source)h(argumen) o(t)e(using)h(the)h(in)o(ter-comm)o(unicator.)75 1434 y(Then)19 b Fl(Q)f Ft(matc)o(hes)h Fl(receiv)o(e)p 556 1434 V 16 w(con)o(text)e Ft(to)h(the)h(message)g(con)o(text)g(and)f (source)i(argumen)o(t)e(to)g(the)h(message)75 1484 y(source.)166 1544 y(The)14 b(same)f(algorithm)e(is)j(appropriate)g(for)g(in)o (tra-comm)n(unicators)e(as)i(w)o(ell.)75 1768 y Fn(5.5.1)49 b(Comm)o(unicato)o(r)13 b(Accesso)o(rs)75 1908 y Fs(MPI)p 160 1908 14 2 v 16 w(COMM)p 318 1908 V 16 w(TEST)p 448 1908 V 17 w(INTER\(comm)m(,)f(\015ag\))117 1989 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))117 2071 y(OUT)108 b Fs(\015ag)518 b Ft(\(logical\))75 2199 y FB(int)23 b(MPI)p 245 2199 15 2 v 17 w(Comm)p 358 2199 V 17 w(test)p 471 2199 V 16 w(inter\(MPI)p 703 2199 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 2289 y(MPI)p 150 2289 V 17 w(COMM)p 263 2289 V 16 w(TEST)p 375 2289 V 17 w(INTER\(COMM,)f(FLAG,)i(IERROR\))170 2345 y(INTEGER)f(COMM,)h (IERROR)170 2402 y(LOGICAL)f(FLAG)75 2492 y FD(This)d(lo)q(cal)g (routine)g(allo)o(ws)f(the)g(calling)i(pro)q(cess)f(to)e(determine)i (if)g(a)f(comm)o(unicator)g(is)g(an)h(in)o(ter-)75 2548 y(comm)o(unicator)15 b(or)g(an)g(in)o(tra-comm)o(unicator.)k(It)c (returns)g Fv(true)i FD(if)e(it)h(is)g(an)f(in)o(ter-comm)o(unicator.) 166 2608 y(When)j(an)f(in)o(ter-comm)o(unicator)h(is)g(used)g(as)f(an)g (input)i(argumen)o(t)d(to)h(the)h(comm)o(unicator)f(ac-)75 2665 y(cessors)e(describ)q(ed)i(under)f(in)o(tra-comm)o(unication,)f (the)h(follo)o(wing)g(table)f(describ)q(es)i(b)q(eha)o(vior.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 137 144 137 143 bop 75 -100 a Fp(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FD(137)p 270 5 1410 2 v 269 62 2 57 v 652 45 a Fs(MPI)p 737 45 14 2 v 16 w(COMM)p 895 45 V 17 w(*)14 b FD(F)l(unction)j(Beha)o (vior)p 1679 62 2 57 v 269 118 V 655 101 a(\(in)f(In)o(ter-Comm)o (unication)f(Mo)q(de\))p 1679 118 V 270 120 1410 2 v 270 130 V 269 186 2 57 v 295 169 a Fs(MPI)p 380 169 14 2 v 16 w(COMM)p 538 169 V 16 w(SIZE)p 729 186 2 57 v 111 w FD(returns)g(the)g(size)h(of)f(the)g(remote)g(group.)p 1679 186 V 269 243 V 295 226 a Fs(MPI)p 380 226 14 2 v 16 w(COMM)p 538 226 V 16 w(GROUP)p 729 243 2 57 v 52 w FD(returns)g(the)g(remote)g(group.)p 1679 243 V 269 299 V 295 282 a Fs(MPI)p 380 282 14 2 v 16 w(COMM)p 538 282 V 16 w(RANK)p 729 299 2 57 v 81 w FD(returns)g(the)g(rank)g(in)h (the)f(lo)q(cal)i(group)p 1679 299 V 270 301 1410 2 v 75 411 a Fn(5.5.2)49 b(Inter-Comm)n(unicato)o(r)13 b(Constructo)o(rs)h (and)j(Destructo)o(rs)75 575 y Fl(Discussion:)31 b Ft(Supp)q(ose)11 b(that)g(a)f(pro)q(cess)i(is)f(in)f(the)h(in)o(tersection)h(of)e(the)h (t)o(w)o(o)f(groups)h(that)f(are)h(b)q(ound)g(together)96 b Fm(CHANGE)75 625 y Ft(in)15 b(an)g(in)o(ter-comm)o(unicator.)20 b(The)15 b(pro)q(cess)i(has)f(t)o(w)o(o)f(di\013eren)o(t)h(in)o (ter-comm)o(unicator)d(ob)r(jects,)j(one)f(for)g(eac)o(h)75 675 y(of)f(its)g(\\roles.")k(It)d(needs)g(to)f(execute)i(t)o(wice)f (eac)o(h)f(call)g(that)g(manipulates)e(the)j(in)o(ter-comm)o(unicator,) c(one)j(for)75 724 y(eac)o(h)g(of)g(its)g(\\roles.")166 775 y(There)h(are)f(three)h(p)q(ossible)g(w)o(a)o(ys)e(of)g(handling)g (this)h(problem:)113 868 y(\(1\))23 b(Ov)o(erlap)15 b(of)g(groups)h (that)f(are)h(b)q(ound)g(in)o(to)f(an)g(in)o(ter-comm)o(unicator)e(is)i (prohibited.)23 b(\(If)16 b(a)f(pro)q(cess)i(is)189 918 y(m)o(ultithreaded,)11 b(and)h(MPI)h(calls)f(blo)q(c)o(k)g(only)f(a)h (thread,)h(rather)h(than)e(a)g(pro)q(cess,)i(then)f(\\dual)f(mem)o(b)q (er-)189 967 y(ship")i(w)o(ould)g(b)q(e)h(allo)o(w)o(ed.)k(It)c(is)g (then)g(the)g(user's)h(resp)q(onsibilit)o(y)e(to)h(mak)o(e)e(sure)j (that)f(calls)f(on)g(b)q(ehalf)189 1017 y(of)i(the)h(t)o(w)o(o)f (\\roles")g(of)g(a)g(pro)q(cess)i(are)f(executed)i(b)o(y)d(t)o(w)o(o)g (indep)q(enden)o(t)i(threads.\))27 b(Op)q(erations)17 b(that)189 1067 y(manipulate)11 b(in)o(ter-comm)o(unicators)h(are)i (collectiv)o(e)g(within)f(the)i(union)e(of)g(the)i(t)o(w)o(o)e(groups.) 113 1147 y(\(2\))23 b(All)11 b(op)q(erations)h(that)g(manipulate)e(in)o (ter-comm)o(unicators)g(are)j(executed)h(in)d(t)o(w)o(o)h(phases:)18 b(one)13 b(call)e(that)189 1197 y(initiates)j(the)h(op)q(eration)f(and) h(one)g(call)f(that)g(terminates)h(the)g(op)q(eration.)20 b(Eac)o(h)15 b(of)f(these)i(t)o(w)o(o)e(calls)g(is)189 1247 y(collectiv)o(e)h(within)g(eac)o(h)h(of)f(the)h(t)o(w)o(o)e (subgroup.)23 b(The)16 b(second)h(call)e(returns)i(pro)o(vided)e(that)g (the)h(other)189 1296 y(group)e(has)h(executed)h(the)f(\014rst)g(call.) 20 b(Th)o(us,)14 b(a)g(pro)q(cess)i(in)f(the)g(in)o(tersection)g(can)g (execute)h(the)f(calls)f(in)189 1346 y(the)d(sequence:)18 b(init1,)10 b(init2,)f(teminate1,)h(terminate2.)16 b(The)11 b(a)o(v)n(ailabili)o(t)o(y)d(of)h(t)o(w)o(o-phase)i(op)q(erations)g (also)189 1396 y(giv)o(es)16 b(more)f(\015exibilit)o(y)f(in)i(a)o(v)o (oiding)e(deadlo)q(c)o(k)i(situations)g(due)h(to)f(a)f(di\013eren)o(t)j (order)e(of)g(calls)g(within)189 1446 y(di\013eren)o(t)e(subgroups.)113 1526 y(\(3\))23 b(All)8 b(calls)h(that)g(manipulate)e(in)o(ter-comm)o (unicators)g(tak)o(e)j(t)o(w)o(o)f(argumen)o(ts:)14 b(1st)c(and)f(2nd)g (in)o(ter-comm)o(unicator.)189 1576 y(A)16 b(pro)q(cess)h(that)f(is)g (in)f(the)h(in)o(tersection)h(of)e(b)q(oth)h(groups)g(pro)o(vide)g(b)q (oth)g(argumen)o(ts;)f(the)i(other)f(pro-)189 1625 y(cesses)i(pro)o (vide)e(only)f(one)h(non-n)o(ull)f(argumen)o(t)g(\(and)h(all)e(pro)q (cesses)19 b(pro)o(vide)d(the)g(same)g(argumen)o(t)e(in)189 1675 y(the)g(same)f(p)q(osition\).)166 1774 y(Option)18 b(\(3\))g(seems)h(tedious)g(\(esp)q(ecially)f(if)g(most)f(of)h(the)g (time)f(the)i(groups)g(do)f(not)g(in)o(tersect\).)34 b(The)75 1831 y(curren)o(t)20 b(draft)f(assumes)f(option)g(\(2\),)i (but)f(w)o(e)g(should)f(discuss)i(option)e(\(1\))h(seriously)m(,)g (whic)o(h)g(w)o(ould)f(allo)o(w)75 1887 y(supp)q(ort)d(for)e(blo)q(c)o (king)g(in)o(ter-comm)o(unicator)e(calls.)166 2028 y FD(The)20 b(functions)g Fs(MPI)p 549 2028 14 2 v 16 w(COMM)p 707 2028 V 16 w(DUP)p FD(,)g Fs(MPI)p 931 2028 V 16 w(COMM)p 1089 2028 V 16 w(MAKE)g FD(and)f Fs(MPI)p 1428 2028 V 16 w(COMM)p 1586 2028 V 17 w(SPLIT)g FD(do)g(not)75 2084 y(apply)d(to)f(in)o(ter-comm)o(unicators.)k(In)d(addition,)g(the)f (functions)h Fs(MPI)p 1303 2084 V 16 w(INTERCOMM)p 1593 2084 V 17 w(ST)l(ART)75 2140 y FD(and)e Fs(MPI)p 247 2140 V 16 w(INTERCOMM)p 537 2140 V 17 w(FINISH)f FD(can)h(b)q(e)h(used) f(to)g(bind)h(t)o(w)o(o)d(in)o(tra-comm)o(unicators)i(in)o(to)g(an)g (in)o(ter-)75 2197 y(comm)o(unicator;)26 b(the)e(function)g Fs(MPI)p 754 2197 V 16 w(INTERCOMM)p 1044 2197 V 16 w(MERGE)g FD(with)g Fs(MPI)p 1431 2197 V 16 w(INTERCOMM)p 1721 2197 V 16 w(FINISH)75 2253 y FD(create)15 b(an)g(in)o(tra-comm)o (unicator)g(b)o(y)g(merging)g(the)h(t)o(w)o(o)e(groups)g(of)h(an)g(in)o (ter-comm)o(unicator.)166 2394 y Fl(Discussion:)58 b Ft(With)20 b(option)f(\(1\),)i(w)o(e)f(can)g(supp)q(ort)h Fv(MPI)p 1179 2394 13 2 v 14 w(COMM)p 1323 2394 V 15 w(DUP)p Ft(.)d(Otherwise,)23 b(w)o(e)d(need)h(to)95 b Fm(CHANGE)75 2450 y Ft(discuss)15 b(the)g(p)q(ossible)f(need)h(for)e (an)h(async)o(hronous)g(call)f Fv(MPI)p 1069 2450 V 15 w(INTERCOMM)p 1335 2450 V 14 w(DUP)p Ft(.)166 2591 y FD(Construction)d(of)h(an)f(in)o(ter-comm)o(unicator)h(from)f(t)o(w)o (o)f(in)o(tra-comm)o(unicators)h(requires)h(separate)75 2647 y(collectiv)o(e)17 b(op)q(erations)f(in)g(the)f(lo)q(cal)i(group)e (and)h(in)g(the)f(remote)g(group,)g(b)q(oth)h(for)e(the)i(starting)f (and)75 2704 y(the)k(\014nishing)i(op)q(erations,)e(as)g(w)o(ell)g(as)g (a)f(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)f(b)q(et)o(w)o(een)h (a)e(pro)q(cess)h(in)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 138 145 138 144 bop 75 -100 a FD(138)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FD(the)g(lo)q(cal)i(group)e(and)g(a)g(pro)q(cess)h(in)g(the)f (remote)g(group.)20 b(The)c(explicit)h(sync)o(hronization)f(can)g (cause)75 102 y(deadlo)q(c)o(k)e(in)g(mo)q(dular)g(programs)e(with)i (cyclic)h(comm)o(unication)f(graphs,)f(ev)o(en)h(if)g(calls)g(are)f (executed)75 158 y(in)j(the)f(same)g(order)g(within)i(eac)o(h)e(group.) 166 335 y Fl(Discussion:)33 b Ft(W)m(e)12 b(ha)o(v)o(e)h(failed)f(to)g (pro)o(vide)h(su\016cien)o(t)g(conditions)g(for)f(when)h(the)h (creation)f(of)f(an)h(in)o(ter-)75 391 y(comm)o(unicator)d(is)j(guaran) o(teed)h(not)f(to)g(deadlo)q(c)o(k.)18 b(W)m(e)13 b(do)g(not)g(wish)g (to)g(hide)h(this)f(fact)g(from)f(the)h(reader.)19 b(W)m(e)75 447 y(will)12 b(con)o(tin)o(ue)i(studying)g(this)g(issue)h(during)e (the)i(public-commen)o(t)c(p)q(erio)q(d.)166 624 y Fs(MPI)p 251 624 14 2 v 16 w(INTERCOMM)p 541 624 V 16 w(ST)l(ART)21 b FD(and)e Fs(MPI)p 894 624 V 16 w(INTERCOMM)p 1184 624 V 17 w(FINISH)g FD(are)f(b)q(oth)i(collectiv)o(e)h(op)q(era-)75 680 y(tions)14 b(in)g(their)g(lo)q(cal)h(group,)e(but)h(the)g(call)g (within)h(eac)o(h)f(group)f(need)i(not)e(b)q(e)h(sync)o(hronized)h (with)f(the)75 737 y(call)i(in)g(the)g(other)e(group.)166 831 y(The)g(in)o(ter-comm)o(unicator)h(ob)s(jects)f(are)g(destro)o(y)o (ed)f(in)j(the)e(same)g(w)o(a)o(y)f(as)h(in)o(tra-comm)o(unicator)-2120 b Fm(CHANGE)75 887 y FD(ob)s(jects,)17 b(that)g(is,)i(b)o(y)e(calling)j Fs(MPI)p 706 887 V 16 w(COMM)p 864 887 V 16 w(FREE)p FD(.)e(F)l(or)f(suc)o(h)h(uses,)g Fs(MPI)p 1403 887 V 16 w(COMM)p 1561 887 V 17 w(FREE)g FD(is)g(only)g(a)75 944 y(sync)o(hronization)e(o)o(v)o(er)e(the)i(original)g(lo)q(cal)g(in) o(tra-comm)o(unicator.)166 1120 y Fl(Implemen)o(tati)o(on)g(note:)23 b Ft(Go)q(o)q(d)16 b(implemen)o(tations)e(of)i Fv(MPI)p 1203 1120 13 2 v 15 w(COMM)p 1348 1120 V 14 w(FREE)g Ft(will)g(w)o(ork)g(without)h(ex-)75 1177 y(plicit)c(comm)o(unication,) d(ev)o(en)15 b(for)e(in)o(ter-comm)o(unicators.)166 1436 y Fl(Discussion:)55 b Ft(This)19 b(solution)f(is)h(adequate)h(with)f (option)f(\(1\);)k(with)c(option)h(\(2\),)h(w)o(e)f(ma)o(y)e(w)o(an)o (t)i(a)-2120 b Fm(CHANGE)75 1492 y Ft(t)o(w)o(o-phase)13 b(proto)q(col,)f(with)g(an)h(async)o(hronous)g(call)f(to)g Fv(MPI)p 1047 1492 V 15 w(INTERCOMM)p 1313 1492 V 14 w(FREE)p Ft(.)f(Otherwise,)j(w)o(e)f(lo)q(c)o(k)f(our-)75 1549 y(selv)o(es)k(out)f(of)f(an)h(implemen)o(tatio)o(n)d(of)j Fv(MPI)p 797 1549 V 14 w(COMM)p 941 1549 V 15 w(FREE)f Ft(that)h(in)o(v)o(olv)o(es)f(a)g(group)h(sync)o(hronization.)22 b(Suc)o(h)75 1605 y(sync)o(hronization)12 b(pro)o(vides)g(additional)e (error)j(c)o(hec)o(king)f(in)g(that)g(it)f(allo)o(ws)g(to)h(test)h (that)f(there)h(are)f(no)g(p)q(ending)75 1662 y(comm)o(unications)f (with)i(a)h(comm)o(unicator,)d(b)q(efore)j(the)h(comm)o(unicator)c(is)j (freed.)166 1838 y FD(The)i(functions)h Fs(MPI)p 542 1838 14 2 v 16 w(INTERCOMM)p 832 1838 V 17 w(ST)l(ART)g FD(and)f Fs(MPI)p 1179 1838 V 16 w(INTERCOMM)p 1469 1838 V 16 w(FINISH)g FD(can)h(b)q(e)f(used)75 1895 y(to)i(create)h(an)f(in)o (ter-comm)o(unicator)h(from)f(t)o(w)o(o)f(existing)j(in)o(tra-comm)o (unicators,)f(in)g(the)g(follo)o(wing)75 1951 y(situation:)33 b(A)o(t)21 b(least)h(one)f(selected)i(mem)o(b)q(er)f(from)f(eac)o(h)g (group)g(\(the)h(\\group)f(leader"\))g(has)h(the)75 2008 y(abilit)o(y)f(to)d(comm)o(unicate)i(with)g(the)f(selected)i(mem)o(b)q (er)f(from)e(the)i(other)f(group;)i(i.e.)33 b(a)19 b(\\paren)o(t")75 2064 y(comm)o(unicator)h(exists)h(where)g(b)q(oth)f(leaders)h(b)q (elong,)i(and)d(eac)o(h)h(leader)g(kno)o(ws)f(the)h(rank)f(of)g(the)75 2121 y(other)h(leader)i(in)f(this)g(paren)o(t)f(comm)o(unicator)h (\(the)f(t)o(w)o(o)f(leaders)i(could)h(b)q(e)f(the)g(same)f(pro)q (cess\).)75 2177 y(F)l(urthermore,)14 b(mem)o(b)q(ers)i(of)e(eac)o(h)i (group)f(kno)o(w)f(the)h(rank)g(of)g(their)h(leader.)166 2271 y(In)24 b(static)f(MPI)g(implemen)o(tation)i(the)e Fs(MPI)p 979 2271 V 16 w(COMM)p 1137 2271 V 16 w(W)o(ORLD)g FD(comm)o(unicator)g(can)h(b)q(e)g(this)75 2327 y(paren)o(t)18 b(comm)o(unicator.)27 b(Since)20 b(this)e(paren)o(t)g(comm)o(unicator)f (ma)o(y)h(b)q(e)g(shared)g(for)g(m)o(ultiple)i(in)o(ter-)75 2384 y(comm)o(unicator)g(creating)g(functions,)i(and)f(the)f(same)g (pro)q(cess)h(ma)o(y)e(b)q(e)i(leaders)g(for)f(sev)o(eral)h(suc)o(h)75 2440 y(calls,)16 b(an)f(additional)i(user-pro)o(vided)f(tag)e(is)i (used)g(to)e(di\013eren)o(tiate)i(calls.)166 2534 y(In)23 b(dynamic)g(MPI)g(implemen)o(tations,)i(where,)f(e.g.,)f(a)g(pro)q (cess)f(ma)o(y)g(spa)o(wn)g(new)h(c)o(hildren)75 2591 y(pro)q(cesses)18 b(during)h(an)f(MPI)f(execution,)j(the)e(paren)o(t)f (pro)q(cess)h(ma)o(y)f(b)q(e)i(the)f(\\bridge")g(b)q(et)o(w)o(een)g (the)75 2647 y(old)i(comm)o(unication)h(univ)o(erse)f(and)g(the)g(new)g (comm)o(unication)g(w)o(orld)g(that)f(includes)j(the)d(paren)o(t)75 2704 y(and)c(its)h(c)o(hildren.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 139 146 139 145 bop 75 -100 a Fp(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FD(139)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(INTERCOMM)p 450 45 V 16 w(ST)l(ART\(lo)q(cal)p 714 45 V 17 w(comm)m(,)52 b(lo)q(cal)p 1002 45 V 16 w(leader,)j(p)q(eer)p 1278 45 V 17 w(comm)m(,)c(remote)p 1611 45 V 14 w(leader,)j(tag,)75 102 y(inter)p 164 102 V 17 w(request\))117 179 y Ft(IN)155 b Fs(lo)q(cal)p 409 179 V 17 w(comm)361 b Ft(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e(\(handle\))117 254 y(IN)155 b Fs(lo)q(cal)p 409 254 V 17 w(leader)370 b Ft(rank)14 b(of)f(lo)q(cal)g(group)h(leader)g(in)g Fv(lo)q(cal)p 1517 254 13 2 v 15 w(comm)8 b Ft(\(in)o(teger\))117 329 y(IN)155 b Fs(p)q(eer)p 401 329 14 2 v 18 w(comm)368 b Ft(\\paren)o(t")17 b(in)o(tra-comm)o(unicator;)e(signi\014can)o(t)i (only)f(at)h(the)905 385 y Fv(lo)q(cal)p 987 385 13 2 v 15 w(leader)f Ft(\(handle\))117 461 y(IN)155 b Fs(remote)p 455 461 14 2 v 14 w(leader)327 b Ft(rank)12 b(of)f(remote)g(group)h (leader)g(in)23 b Fv(p)q(eer)p 1548 461 13 2 v 16 w(comm)n Ft(;)8 b(signi\014can)o(t)905 517 y(only)13 b(at)h(the)h Fv(lo)q(cal)p 1201 517 V 15 w(leader)g Ft(\(in)o(teger\))117 592 y(IN)155 b Fs(tag)526 b Ft(\\safe")14 b(tag)f(\(in)o(teger\))117 667 y(OUT)108 b Fs(inter)p 407 667 14 2 v 17 w(request)349 b Ft(handle)14 b(for)27 b Fv(MPI)p 1193 667 13 2 v 15 w(INTERCOMM)p 1459 667 V 14 w(FINISH\(\))75 792 y FB(int)c(MPI)p 245 792 15 2 v 17 w(Intercomm)p 478 792 V 16 w(start\(MPI)p 710 792 V 16 w(Comm)g(local)p 965 792 V 17 w(comm,)g(int)h(local)p 1341 792 V 16 w(leader,)393 848 y(MPI)p 468 848 V 17 w(Comm)f(peer)p 700 848 V 17 w(comm,)g(int)h(remote)p 1100 848 V 16 w(leader,)f(int)g(tag,)393 905 y(MPI)p 468 905 V 17 w(Intercomm)p 701 905 V 16 w(request)g(*inter)p 1052 905 V 16 w(request\))75 991 y(MPI)p 150 991 V 17 w(INTERCOMM)p 383 991 V 16 w(START\(LOCAL)p 663 991 V 15 w(COMM,)h(LOCAL)p 942 991 V 16 w(LEADER,)f(PEER)p 1245 991 V 17 w(COMM,)g(REMOTE)p 1549 991 V 16 w(LEADER,)g(TAG,)393 1048 y(INTER)p 516 1048 V 17 w(REQUEST,)f(IERROR\))170 1104 y(INTEGER)h(LOCAL)p 484 1104 V 17 w(COMM,)g(LOCAL)p 764 1104 V 17 w(LEADER,)f(PEER)p 1067 1104 V 17 w(COMM,)h(REMOTE)p 1371 1104 V 17 w(LEADER,)g(TAG,)170 1160 y(INTER)p 293 1160 V 17 w(REQUEST,)g(IERROR)75 1247 y FD(This)d(call)g(starts)e (o\013)g(an)h(in)o(ter-comm)o(unicator)g(creation)g(op)q(eration,)h (returning)f(a)g(handle)h(for)f(the)75 1303 y(completion)h(of)e(the)h (op)q(eration)g(in)h Fs(inter)p 801 1303 14 2 v 17 w(request)p FD(;)i(it)d(is)g(collectiv)o(e)i(in)f Fs(lo)q(cal)p 1432 1303 V 16 w(comm)m FD(.)28 b(The)19 b(call)h(do)q(es)75 1360 y(not)i(w)o(ait)g(for)g(the)h(remote)f(group)g(to)g(execute)h Fs(MPI)p 1046 1360 V 16 w(INTERCOMM)p 1336 1360 V 16 w(ST)l(ART\(\))p FD(.)42 b Fs(inter)p 1671 1360 V 17 w(request)24 b FD(is)75 1416 y(conceptually)19 b(similar)g(to)d(the)i (comm)o(unication)g(request)f(handle)i(used)e(b)o(y)h(non-blo)q(c)o (king)h(p)q(oin)o(t-to-)75 1473 y(p)q(oin)o(t)13 b(routines.)20 b(The)12 b(handle)i(is)g(constructed)e(b)o(y)h(a)p 1002 1473 V 29 w Fs(sta)o(rt)g FD(routine)g(and)g(destro)o(y)o(ed)f(b)o(y)g (the)h(matc)o(hing)p 75 1529 V 89 1529 a Fs(\014nish)20 b FD(routine;)h(these)f(handles)g(are)f(not)g(v)m(alid)h(for)f(an)o(y)g (other)f(use.)32 b(It)19 b(is)h(erroneous)f(to)f(call)j(this)75 1586 y(routine)12 b(a)f(second)h(time)g(with)g(the)g(same)f Fs(b)o(ridge)p 908 1586 V 16 w(comm)n FD(,)d Fs(remote)p 1198 1586 V 14 w(leader)k FD(and)f Fs(tag)h FD(argumen)o(ts,)f(without) 75 1642 y(an)k(in)o(terv)o(ening)h(call)h(to)d Fs(MPI)p 598 1642 V 16 w(INTERCOMM)p 888 1642 V 17 w(FINISH\(\))g FD(to)h(complete)g(the)h(p)q(ending)h(call.)166 1699 y(Pro)q(cesses)h(should)h(pro)o(vide)g(iden)o(tical)h Fs(comm)m FD(,)15 b Fs(lo)q(cal)p 1106 1699 V 16 w(leader)j FD(and)g Fs(tag)g FD(argumen)o(ts)g(within)h(eac)o(h)75 1755 y(group.)75 1859 y Fs(MPI)p 160 1859 V 16 w(INTERCOMM)p 450 1859 V 16 w(FINISH\(inter)p 711 1859 V 17 w(request,)d(new)o(comm)n (\))117 1936 y Ft(INOUT)62 b Fs(inter)p 407 1936 V 17 w(request)349 b Ft(handle)14 b(returned)i(b)o(y)d Fv(MPI)p 1342 1936 13 2 v 15 w(INTERCOMM)p 1608 1936 V 14 w(ST)m(ART\(\))117 2011 y Ft(OUT)108 b Fs(new)o(comm)393 b Ft(new)15 b(in)o(ter-comm)o (unicator)c(\(handle\))75 2135 y FB(int)23 b(MPI)p 245 2135 15 2 v 17 w(Intercomm)p 478 2135 V 16 w(finish\(MPI)p 734 2135 V 16 w(Intercomm)p 966 2135 V 16 w(request)g(*inter)p 1317 2135 V 16 w(request,)393 2192 y(MPI)p 468 2192 V 17 w(Comm)g(*newcomm\))75 2278 y(MPI)p 150 2278 V 17 w(INTERCOMM)p 383 2278 V 16 w(FINISH\(INTER)p 687 2278 V 15 w(REQUEST,)g(IERROR\))170 2335 y(INTEGER)g(INTER)p 484 2335 V 17 w(REQUEST,)g(NEWCOMM,)f(IERROR)75 2421 y Fs(MPI)p 160 2421 14 2 v 16 w(INTERCOMM)p 450 2421 V 16 w(FINISH)d FD(completes)g(an)g(async)o(hronous)f(in)o(ter-comm)o (unicator)h(creation)g(op)q(er-)75 2478 y(ation.)27 b(This)18 b(routine)g(is)g(collectiv)o(e)h(in)g(the)e(goup)h(asso)q(ciated)g (with)f(comm)o(unicator)h Fs(lo)q(cal)p 1693 2478 V 16 w(comm)11 b FD(of)75 2534 y(the)16 b(corresp)q(onding)i(call)f(to)e Fs(MPI)p 673 2534 V 16 w(INTERCOMM)p 963 2534 V 17 w(ST)l(ART)p FD(.)i(It)f(w)o(aits)g(for)f(the)i(remote)e(group)h(to)g(call)75 2591 y Fs(MPI)p 160 2591 V 16 w(INTERCOMM)p 450 2591 V 16 w(ST)l(ART)h FD(but)e(do)q(es)g(not)g(w)o(ait)g(for)g(the)g (remote)g(group)f(to)h(call)75 2647 y Fs(MPI)p 160 2647 V 16 w(INTERCOMM)p 450 2647 V 16 w(FINISH)p FD(.)166 2704 y(If)f(there)g(are)g(common)g(mem)o(b)q(ers)g(of)g(b)q(oth)g(in)o (tra-comm)o(unicators,)f(then)i(these)f(common)g(mem-)95 b Fm(CHANGE)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 140 147 140 146 bop 75 -100 a FD(140)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FD(b)q(ers)22 b(m)o(ust)f(all)h(execute)h(the)e(t)o(w)o(o)g Fs(MPI)p 799 45 14 2 v 15 w(INTERCOMM)p 1088 45 V 17 w(ST)l(ART)i FD(op)q(erations)e(in)i(the)e(same)h(order,)75 102 y(follo)o(w)o(ed)16 b(b)o(y)f(the)g Fs(MPI)p 477 102 V 16 w(INTERCOMM)p 767 102 V 16 w(FINISH)g FD(calls,)h(also)f(in)i (the)e(same)g(order)g(of)f(this)i(in)o(tersection)75 158 y(of)f(pro)q(cesses.)166 314 y Fl(Implemen)o(tati)o(on)8 b(note:)33 b Fv(MPI)p 723 314 13 2 v 14 w(INTERCOMM)p 988 314 V 15 w(ST)m(ART)10 b Ft(initiates)g(async)o(hronous)h(sends)h (and)e(receiv)o(es)75 370 y(for)i(an)h(exc)o(hange)g(of)g(information)d (b)q(et)o(w)o(een)k(leaders.)k Fv(MPI)p 1031 370 V 15 w(INTERCOMM)p 1297 370 V 14 w(FINISH)12 b Ft(completes)h(the)g(exc)o (hange)75 427 y(of)g(information,)e(next)j(execute)i(a)e(broadcast)g (in)f(eac)o(h)i(lo)q(cal)e(group,)g(with)h(the)g(lo)q(cal)f(leader)h (at)g(the)h(ro)q(ot.)75 630 y Fs(MPI)p 160 630 14 2 v 16 w(INTERCOMM)p 450 630 V 16 w(MERGE\(intercomm)o(,)c(high,)16 b(inter)p 1062 630 V 16 w(request\))117 724 y Ft(IN)155 b Fs(intercomm)380 b Ft(In)o(ter-Comm)o(unicator)11 b(\(handle\))117 833 y(IN)155 b Fs(high)507 b Ft(\(logical\))117 942 y(OUT)108 b Fs(inter)p 407 942 V 17 w(request)349 b Ft(in)o(ter-comm)o(unicator) 11 b(request)16 b(\(handle\))75 1083 y FB(int)23 b(MPI)p 245 1083 15 2 v 17 w(Intercomm)p 478 1083 V 16 w(merge\(MPI)p 710 1083 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 1139 y(MPI)p 468 1139 V 17 w(Intercomm)p 701 1139 V 16 w(request)g(*inter)p 1052 1139 V 16 w(request\))75 1243 y(MPI)p 150 1243 V 17 w(INTERCOMM)p 383 1243 V 16 w(MERGE\(INTERCOMM,)e(HIGH,)j(INTER)p 1068 1243 V 16 w(REQUEST,)f(IERROR\))170 1299 y(INTEGER)g(INTERCOMM,)g (INTER)p 747 1299 V 16 w(REQUEST,)g(IERROR)170 1355 y(LOGICAL)g(HIGH)75 1459 y FD(This)15 b(function)h(starts)e(the)h(creation)g(of)f(an)h(in)o (tra-comm)o(unicator)f(for)h(the)g(union)g(of)g(the)g(t)o(w)o(o)e (groups)-2120 b Fm(CHANGE)75 1515 y FD(that)22 b(are)g(asso)q(ciated)h (with)g Fs(intercomm)n FD(.)39 b(All)24 b(pro)q(cesses)f(should)g(pro)o (vide)h(the)e(same)h Fs(high)g FD(v)m(alue)75 1572 y(within)c(eac)o(h)g (of)f(the)g(t)o(w)o(o)f(groups.)29 b(If)18 b(pro)q(cesses)h(in)g(one)f (group)g(pro)o(vided)i(the)e(v)m(alue)h Fs(high)g(=)g(false)75 1628 y FD(and)d(pro)q(cesses)g(in)h(the)e(other)h(group)f(pro)o(vided)i (the)e(v)m(alue)i Fs(high)g(=)f(true)g FD(then)g(the)g(union)g(orders)g (the)75 1685 y(\\lo)o(w")j(group)g(b)q(efore)h(the)g(\\high")g(group.) 33 b(If)20 b(all)h(pro)q(cesses)f(pro)o(vided)g(the)g(same)g Fs(high)g FD(argumen)o(t)75 1741 y(then)h(the)f(order)g(of)g(the)g (union)i(is)e(arbitrary)l(.)35 b(This)21 b(call)g(is)g(blo)q(c)o(king)h (and)e(collectiv)o(e)i(within)g(the)75 1797 y(union)g(of)e(the)h (groups.)35 b(The)21 b(creation)g(of)f(the)h(in)o(tra-comm)o(unicator)f (is)h(completed)h(b)o(y)f(a)f(call)i(to)75 1854 y Fs(MPI)p 160 1854 14 2 v 16 w(INTERCOMM)p 450 1854 V 16 w(FINISH)p FD(.)166 1927 y(If)14 b(there)g(are)g(common)g(mem)o(b)q(ers)g(of)g(b)q (oth)g(in)o(tra-comm)o(unicators,)f(then)i(these)f(common)g(mem-)-2120 b Fm(CHANGE)75 1984 y FD(b)q(ers)21 b(m)o(ust)f(all)i(execute)f(the)f (t)o(w)o(o)g Fs(MPI)p 793 1984 V 16 w(INTERCOMM)p 1083 1984 V 16 w(MERGE)h FD(op)q(erations)g(in)h(the)e(same)h(order,)75 2040 y(follo)o(w)o(ed)16 b(b)o(y)f(the)g Fs(MPI)p 477 2040 V 16 w(INTERCOMM)p 767 2040 V 16 w(FINISH)g FD(calls,)h(also)f(in) i(the)e(same)g(order)g(of)f(this)i(in)o(tersection)75 2096 y(of)f(pro)q(cesses.)75 2313 y Fs(Alternatives)75 2431 y FD(The)g(c)o(hanges)h(prop)q(osed)f(in)h(this)g(subsection)g (are)f(indep)q(enden)o(t)j(of)c(eac)o(h)i(other.)-1742 b Fm(CHANGE)75 2647 y Fs(Use)16 b(regula)o(r)e(request)j(objects)75 2704 y(MPI)p 160 2704 V 16 w(INTERCOMM)p 450 2704 V 16 w(MAKE\(lo)q(cal)p 701 2704 V 16 w(comm)m(,)55 b(lo)q(cal)p 991 2704 V 16 w(leader,)j(p)q(eer)p 1270 2704 V 18 w(comm)m(,)d(remo)o (te)p 1607 2704 V 14 w(leader,)j(tag,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 141 148 141 147 bop 75 -100 a Fp(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FD(141)75 45 y Fs(request\))117 122 y Ft(IN)155 b Fs(lo)q(cal)p 409 122 14 2 v 17 w(comm)361 b Ft(lo)q(cal)13 b(in)o(tra-comm)o (unicator)e(\(handle\))117 195 y(IN)155 b Fs(lo)q(cal)p 409 195 V 17 w(leader)370 b Ft(rank)14 b(of)f(lo)q(cal)g(group)h (leader)g(in)g Fv(lo)q(cal)p 1517 195 13 2 v 15 w(comm)8 b Ft(\(in)o(teger\))117 267 y(IN)155 b Fs(p)q(eer)p 401 267 14 2 v 18 w(comm)368 b Ft(\\paren)o(t")17 b(in)o(tra-comm)o (unicator;)e(signi\014can)o(t)i(only)f(at)h(the)905 324 y Fv(lo)q(cal)p 987 324 13 2 v 15 w(leader)f Ft(\(handle\))117 396 y(IN)155 b Fs(remote)p 455 396 14 2 v 14 w(leader)327 b Ft(rank)12 b(of)f(remote)g(group)h(leader)g(in)23 b Fv(p)q(eer)p 1548 396 13 2 v 16 w(comm)n Ft(;)8 b(signi\014can)o(t)905 453 y(only)13 b(at)h(the)h Fv(lo)q(cal)p 1201 453 V 15 w(leader)g Ft(\(in)o(teger\))117 525 y(IN)155 b Fs(tag)526 b Ft(\\safe")14 b(tag)f(\(in)o(teger\))117 597 y(OUT)108 b Fs(request)452 b Ft(request)16 b(ob)r(ject)e(for)g(op)q(eration)g (completion)e(\(handle\))75 722 y FD(This)20 b(call)g(starts)e(o\013)g (an)h(in)o(ter-comm)o(unicator)g(creation)g(op)q(eration,)h(returning)f (a)g(handle)h(for)f(the)75 778 y(completion)c(of)e(the)h(op)q(eration)g (in)g Fs(request)p FD(;)i(it)e(is)g(collectiv)o(e)h(in)g Fs(lo)q(cal)p 1276 778 14 2 v 16 w(comm)m FD(.)h(The)e(call)h(do)q(es)f (not)f(w)o(ait)75 835 y(for)e(the)g(remote)g(group)g(to)g(execute)h Fs(MPI)p 784 835 V 15 w(INTERCOMM)p 1073 835 V 17 w(MAKE\(\))p FD(.)18 b(The)11 b(op)q(eration)h(is)g(completed)g(b)o(y)75 891 y(a)h(call)i(to)e Fs(MPI)p 332 891 V 16 w(W)l(AIT)p FD(,)g Fs(MPI)p 569 891 V 16 w(TEST)g FD(or)h(an)o(y)f(other)g(of)g (the)h(functions)g(that)f(complete)h(comm)o(unication)75 948 y(op)q(erations.)40 b(A)21 b(handle)i(to)e(the)h(in)o(ter-comm)o (unicator)g(created)g(will)h(b)q(e)g(returned)f(in)g(the)g Fs(status)75 1004 y FD(argumen)o(t.)d(This)d(handle)g(can)g(b)q(e)g (retriev)o(ed)f(b)o(y)g(a)g(call)i(to)d Fs(MPI)p 1206 1004 V 16 w(GET)p 1310 1004 V 17 w(INTERCOMM)p FD(.)166 1137 y Fl(Implemen)o(tati)o(on)j(note:)27 b Fv(MPI)p 726 1137 13 2 v 14 w(W)m(AIT)m(ANY)p Ft(,)16 b Fv(MPI)p 1029 1137 V 14 w(W)m(AIT)m(ALL)p Ft(,)i(and)g Fv(MPI)p 1406 1137 V 14 w(TEST)m(ANY)g Ft(ha)o(v)o(e)g(only)f(ap-)75 1187 y(plied)e(to)g(p)q(oin)o(t-to-p)q(oin)o(t)g(op)q(erations.)23 b(Greater)16 b(implemen)o(tatio)o(n)d(e\013ort)j(ma)o(y)e(b)q(e)i (required)g(to)g(supp)q(ort)g(this)75 1236 y(functionalit)o(y)m(.)166 1293 y(If)c(there)i(is)e(in)o(tersection)i(b)q(et)o(w)o(een)g(the)f (groups)g(of)e(the)j(in)o(tra-comm)n(unicators,)c(then)j(the)g Fv(MPI)p 1689 1293 V 15 w(*ANY)f Ft(are)75 1349 y(going)k(to)h(ha)o(v)o (e)h(to)f(guaran)o(tee)h(the)g(completion)d(of)i(these)i(requests)g(in) e(the)h(same)f(order)h(in)f(all)f(in)o(tersecting)75 1406 y(pro)q(cesses.)75 1592 y Fs(MPI)p 160 1592 14 2 v 16 w(GET)p 264 1592 V 17 w(INTERCOMM\(status,comm)n(\))117 1669 y Ft(IN)155 b Fs(status)476 b Ft(status)21 b(ob)r(ject)h(returned) g(b)o(y)e(a)g(call)g(that)g(completed)g(an)905 1726 y(in)o(ter-comm)o (unicator)11 b(creation)k(op)q(eration)e(\(Status\))117 1798 y(OUT)108 b Fs(comm)466 b Ft(in)o(ter-comm)o(unicator)11 b(\(handle\))189 1954 y Fo(R)n(ationale.)60 b FD(The)19 b(main)g(reason)f(for)g(the)h(c)o(hange)f(\(aside)h(from)f(decreasing)i (the)e(n)o(um)o(b)q(er)h(of)189 2010 y(functions\))11 b(is)h(that)e(it)i(allo)o(ws)f(to)g(use)g Fs(MPI)p 918 2010 V 16 w(W)l(AIT)p 1047 2010 V 16 w(ANY)h FD(or)f Fs(MPI)p 1301 2010 V 16 w(W)l(AIT)p 1430 2010 V 16 w(SOME)p FD(,)g(for)g(completing)189 2067 y(the)k(creation)h(of)f(an)g(in)o (ter-comm)o(unicator.)21 b(The)16 b(use)g(of)f(these)g(functions)i(is)f (the)f(only)h(w)o(a)o(y)f(to)189 2123 y(get)i(a)g(b)q(eha)o(vior)h (equiv)m(alen)o(t)h(to)e(the)h(b)q(eha)o(vior)g(of)f(a)g(guarded)h (command)g(or)f(of)g(an)g(OCCAM)189 2179 y Fs(AL)l(T)p FD(.)189 2252 y(Consider)22 b(the)g(case)g(of)g(parallel)h(serv)o(er)f (that)f(needs)i(to)e(listen)i(b)q(oth)f(for)f(requests)h(of)g(new)189 2308 y(clien)o(ts)g(to)f(bind)i(to)e(it,)i(and)e(for)g(requests)h(of)f (existing)h(clien)o(ts)h(for)d(services.)40 b(The)22 b(former)189 2365 y(are)h(more)g(naturally)h(handled)g(as)f(non)o(blo)q (c)o(king)i(calls)g(to)d Fs(MPI)p 1361 2365 V 16 w(INTERCOMM)p 1651 2365 V 17 w(MAKE)h FD(\(or)189 2421 y Fs(MPI)p 274 2421 V 15 w(INTERCOMM)p 563 2421 V 17 w(NAME)p 709 2421 V 17 w(ST)l(ART)p FD(\).)d(The)g(later)f(are)g(more)g(lik)o(ely)j (handled)f(as)e(non)o(blo)q(c)o(k-)189 2478 y(ing)j(receiv)o(es)g(to)f (p)q(oin)o(t)h(to)f(p)q(oin)o(t)h(messages.)38 b(The)22 b(serv)o(er)f(has)g(to)g(w)o(ait)g(for)g(b)q(oth)h(t)o(yp)q(e)f(of)189 2534 y(requests.)e(Ha)o(ving)d(b)q(oth)f(requests)g(in)h(an)f(arra)o(y) f(that)h(is)g(passed)h(to)e Fs(MPI)p 1485 2534 V 16 w(W)l(AIT)h FD(is)h(the)f(con)o(v)o(e-)189 2591 y(nien)o(t)g(w)o(a)o(y)e(to)h(do)h (this.)20 b(\(The)14 b(only)i(alternativ)o(e)e(is)h(that)f(requests)h (of)f(new)h(clien)o(ts)g(to)f(bind)i(are)189 2647 y(passed)h(as)g(p)q (oin)o(t)g(to)g(p)q(oin)o(t)g(messages)g(to)f(the)i(serv)o(er)e (leader.)27 b(But,)17 b(then,)h(the)f(name)g(service)189 2704 y(b)q(ecomes)e(somewhat)g(useless.\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 142 149 142 148 bop 75 -100 a FD(142)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)189 45 y FD(The)e(do)o(wn)g(side)h(of)e(suc)o(h)i(c)o(hange)f(is)g(an)g (additional)i(test)d(on)h(the)g(path)g(for)f(receiving)j(messages.)189 102 y(Ho)o(w)o(ev)o(er,)g(an)h(in)o(telligen)o(t)i(implemen)o(tation)g (will)f(ha)o(v)o(e)f(only)h(one)f(test)g(on)g(the)h(most)e(frequen)o(t) 189 158 y(path)g(for)g(the)h(execution)h(of)f Fs(MPI)p 786 158 14 2 v 15 w(W)l(AIT)p FD(.)g(Adding)h(one)f(more)f(use)h(of)f (request)h(ob)s(jects)f(adds)h(a)189 214 y(test)e(on)h(the)h(least)f (frequen)o(t)g(execution)h(path.)k(\()p Fo(End)c(of)g(r)n(ationale.)p FD(\))75 341 y Fs(Allo)o(w)f(anonymous)e(leader)75 397 y FD(The)j(lo)q(cal)h(leader)f(of)f(exactly)h(one)g(of)f(the)h(t)o(w)o (o)e(groups)h(ma)o(y)g(pro)o(vide)h(the)g(v)m(alue)h Fv(MPI)p 1603 397 13 2 v 14 w(ANY)p 1702 397 V 15 w(SOURCE)75 453 y FD(to)e(argumen)o(t)f Fs(remote)p 471 453 14 2 v 14 w(leader)p FD(.)189 565 y Fo(R)n(ationale.)39 b FD(This)14 b(allo)o(ws)h(to)e(supp)q(ort)h(an)g(asymmetric)g(proto)q (col,)g(where)g(a)g(clien)o(t)h(that)f(binds)189 621 y(to)j(a)g(serv)o(er)h(kno)o(ws)f(the)h(address)g(of)g(the)f(serv)o(er) h(leader,)h(but)f(the)g(serv)o(er)f(that)h(binds)h(to)e(the)189 678 y(clien)o(t)e(need)g(ha)o(v)o(e)f(no)g(kno)o(wledge)h(of)f(the)g (clien)o(t.)21 b(The)15 b(extension)g(en)o(tails)g(little)g(c)o(hange)g (in)g(the)189 734 y(implemen)o(tation.)21 b(\()p Fo(End)15 b(of)i(r)n(ationale.)p FD(\))75 862 y Fn(5.5.3)49 b(Name)14 b(Service)75 950 y FD(MPI)g(pro)o(vides)h(a)g(name)f(service)h(to)f (simplify)j(construction)d(of)g(in)o(ter-comm)o(unicators.)20 b(This)15 b(service)75 1006 y(allo)o(ws)f(a)g(lo)q(cal)h(pro)q(cess)f (group)g(to)f(create)h(an)g(in)o(ter-comm)o(unicator)g(when)g(the)g (only)h(a)o(v)m(ailable)h(infor-)75 1063 y(mation)10 b(ab)q(out)h(the)f(remote)g(group)h(is)g(a)f(user-de\014ned)i(c)o (haracter)e(string.)18 b(A)11 b(lo)q(osely)h(sync)o(hronous)e(v)o(er-) 75 1119 y(sion)h(is)f(pro)o(vided)h(b)o(y)f(routines)h Fs(MPI)p 699 1119 V 16 w(INTERCOMM)p 989 1119 V 16 w(NAME)p 1134 1119 V 17 w(ST)l(ART)g FD(and)g Fs(MPI)p 1470 1119 V 15 w(INTERCOMM)p 1759 1119 V 17 w(FINISH)p FD(.)75 1176 y(Note)22 b(that)g(for)g(rendezv)o(ous)h(to)f(o)q(ccur,)j(all)e (in)o(v)o(o)q(cations)h(of)e Fs(MPI)p 1283 1176 V 16 w(INTERCOMM)p 1573 1176 V 16 w(NAME)p 1718 1176 V 17 w(ST)l(ART)75 1232 y FD(clearly)16 b(m)o(ust)f(sp)q(ecify)h(the)g(same) f(v)m(alue)h(for)f(the)g Fs(name)e FD(argumen)o(t.)75 1337 y Fs(MPI)p 160 1337 V 16 w(INTERCOMM)p 450 1337 V 16 w(NAME)p 595 1337 V 17 w(ST)l(ART\(comm)n(,)f(name,)g(inter)p 1131 1337 V 17 w(request\))117 1415 y Ft(IN)155 b Fs(comm)466 b Ft(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e(\(handle\))117 1492 y(IN)155 b Fs(name)483 b Ft(c)o(haracter)15 b(string)117 1570 y(OUT)108 b Fs(inter)p 407 1570 V 17 w(request)349 b Ft(handle)14 b(for)g(MPI)p 1186 1570 13 2 v 15 w(INTER)o(COMM)p 1473 1570 V 15 w(FINISH\(\))75 1695 y FB(int)23 b(MPI)p 245 1695 15 2 v 17 w(Intercomm)p 478 1695 V 16 w(name)p 590 1695 V 17 w(start\(MPI)p 823 1695 V 16 w(Comm)g(comm,)g(char)h (*name,)393 1752 y(MPI)p 468 1752 V 17 w(Intercomm)p 701 1752 V 16 w(request)f(*inter)p 1052 1752 V 16 w(request\))75 1839 y(MPI)p 150 1839 V 17 w(INTERCOMM)p 383 1839 V 16 w(NAME)p 495 1839 V 16 w(START\(COMM,)g(NAME,)g(NEW)p 1013 1839 V 17 w(COMM,)g(INTER)p 1293 1839 V 16 w(REQUEST,)g(IERROR\)) 170 1896 y(CHARACTER)g(*\(*\))g(NAME)170 1952 y(INTEGER)g(COMM,)h(NEW)p 580 1952 V 16 w(COMM,)f(INTER)p 859 1952 V 17 w(REQUEST,)g(IERROR)75 2039 y FD(This)d(is)h(the)f(name-serv)o(ed)g(equiv)m(alen)o(t)h(of)f Fs(MPI)p 944 2039 14 2 v 16 w(INTERCOMM)p 1234 2039 V 16 w(ST)l(ART)p FD(.)h(The)f(op)q(eration)g(is)g(com-)75 2096 y(pleted)c(b)o(y)f(a)g(call)i(to)d Fs(MPI)p 536 2096 V 16 w(INTERCOMM)p 826 2096 V 17 w(FINISH)p FD(.)75 2222 y Fs(Alternatives)75 2310 y FD(A)g(comm)o(unicator)g(ma)o(y)f(b)q (e)i(asso)q(ciated)f(with)g(an)g Fx(in)o(ter-comm)o(unicator)j(binding) g(service)p FD(.)i(The)-2120 b Fm(CHANGE)75 2366 y FD(enquiry)15 b(function)g Fs(MPI)p 500 2366 V 15 w(TEST)p 629 2366 V 17 w(SERVER)g FD(can)f(b)q(e)h(used)f(to)g(\014nd)g(whether)g(a)g (serv)o(er)f(is)i(a)o(v)m(ailable)g(for)e(a)75 2423 y(particular)f (comm)o(unicator.)18 b(The)12 b(service)h(can)e(b)q(e)i(used)f(to)f (bind)h(together)f(t)o(w)o(o)f(groups)i(in)o(to)f(an)h(in)o(ter-)75 2479 y(comm)o(unicator,)h(in)h(the)f(situation)h(where)f(the)h(group)f (leaders)h(b)q(elong)g(to)e(the)i(group)f(comm)o(unicator,)75 2536 y(ev)o(en)j(when)f(neither)h(leader)g(kno)o(ws)f(the)g(rank)g(of)g (the)g(other)g(leader.)189 2647 y Fo(R)n(ationale.)51 b FD(There)17 b(are)g(t)o(w)o(o)f(adv)m(an)o(tages)g(for)g(this)i (approac)o(h,)f(as)f(compared)h(to)g(the)g(one)g(in)189 2704 y(the)e(draft:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 143 150 143 149 bop 75 -100 a Fp(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FD(143)231 45 y(1.)22 b(One)15 b(ma)o(y)g(not)f(w)o(an)o(t)g(to)h(ha) o(v)o(e)f(a)h(name)g(serv)o(er)f(in)i(some)f(execution)h(\(a)e(naiv)o (e)i(implemen-)289 102 y(tation)g(for)g(a)g(name)h(serv)o(er)f(w)o (ould)h(dev)o(ote)g(a)f(dedicated)i(pro)q(cess)f(for)f(it)h({)f(whic)o (h)h(is)h(to)q(o)289 158 y(w)o(asteful)d(for)f(a)h(small)h (computation\).)231 224 y(2.)22 b(One)16 b(ma)o(y)g(w)o(an)o(t)e(serv)o (ers)i(to)f(ha)o(v)o(e)h(a)g(con)o(text,)f(so)g(that)h(they)g(need)g (not)g(b)q(e)h(global.)22 b(This)289 280 y(is)c(imp)q(ortan)o(t)f(once) h(w)o(e)f(mo)o(v)o(e)g(to)g(dynamic)h(pro)q(cess)g(managemen)o(t,)f (and)h(also)f(for)g(sup-)289 337 y(p)q(orting)e(a)g(mo)q(dular)h(en)o (vironmen)o(t)f(for)g(names)g(and)g(for)g(service)h(binding.)189 428 y(\()p Fo(End)f(of)i(r)n(ationale.)p FD(\))75 556 y Fs(MPI)p 160 556 14 2 v 16 w(TEST)p 290 556 V 16 w(SERVER\(comm)o(,) 12 b(\015ag\))117 633 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 704 y(OUT)108 b Fs(\015ag)518 b Fv(true)16 b Ft(if)e(an)h(in)o(ter-comm)o(unicator)d(binding)i(service)i(is)f (asso-)905 761 y(ciated)f(with)g Fv(comm)m Ft(,)d Fv(false)p Ft(,)i(otherwise)166 961 y Fl(Discussion:)29 b Ft(The)10 b(initialization)c(pro)q(cess)12 b(could)d(sp)q(ecify)g(whether)i(suc)o (h)f(a)f(service)i(exists)f(for)f Fv(MPI)p 1777 961 13 2 v 14 w(COMM)p 1921 961 V 15 w(W)o(ORLD)p Ft(.)166 1018 y(This)14 b(function)f(can)h(mo)o(v)o(e)f(to)g(the)i(enquiry)f(c)o (hapter.)75 1204 y Fs(MPI)p 160 1204 14 2 v 16 w(INTERCOMM)p 450 1204 V 16 w(NAME)p 595 1204 V 17 w(ST)l(ART\(comm)n(,)e(name,)g (inter)p 1131 1204 V 17 w(request\))117 1281 y Ft(IN)155 b Fs(comm)466 b Ft(in)o(tra-comm)o(unicator)11 b(\(handle\))117 1353 y(IN)155 b Fs(name)483 b Ft(c)o(haracter)15 b(string)117 1424 y(OUT)108 b Fs(inter)p 407 1424 V 17 w(request)349 b Ft(handle)14 b(for)g(MPI)p 1186 1424 13 2 v 15 w(INTER)o(COMM)p 1473 1424 V 15 w(FINISH\(\))166 1549 y FD(The)h(call)i(is)e(erroneous)g (if)h(no)f(in)o(tercomm)g(binding)j(service)e(is)f(pro)o(vided)h(with)g Fs(comm)m FD(.)75 1665 y Fs(Delete)g(MPI)p 296 1665 14 2 v 16 w(INTERCOMM)p 586 1665 V 16 w(NAME)p 731 1665 V 17 w(ST)l(ART)g(I.)75 1722 y FD(In)g(a)e(call)i(to)f Fs(MPI)p 393 1722 V 15 w(INTERCOMM)p 682 1722 V 17 w(MAKE)p FD(,)f(b)q(oth)i(group)e(leaders)i(can)f(pro)o(vide)g(in)h(the)f Fs(remote)p 1752 1722 V 14 w(leader)75 1778 y FD(argumen)o(t)f(the)g(v) m(alue)i Fv(MPI)p 549 1778 13 2 v 14 w(BINDER)p FD(.)d(This)i(will)i (ha)o(v)o(e)d(the)g(same)g(e\013ect)g(as)h(if)f(they)h(had)g(pro)o (vided)g(the)75 1835 y(rank)j(of)f(the)h(remote)g(leader,)h(and)f(will) i(result)e(in)h(the)f(creation)g(of)g(an)f(in)o(ter-comm)o(unicator.)29 b(The)75 1891 y(p)q(eer)16 b(comm)o(unicator)f(is)g Fs(MPI)p 602 1891 14 2 v 16 w(COMM)p 760 1891 V 17 w(W)o(ORLD)p FD(.)1021 b Fm(CHANGE)166 2030 y Fl(Implemen)o(tati)o(on)13 b(note:)39 b Ft(If)15 b(the)g(binding)f(services)i(is)f(pro)o(vided)f (b)o(y)h(a)f(pro)q(cess)j(within)d(the)h(comm)o(u-)75 2087 y(nicator)e(group,)f(then)h(the)g(co)q(de)h(executed)h(within)c (eac)o(h)j(group)e(sta)o(ys)h(unc)o(hanged:)18 b(eac)o(h)13 b(group)g(selects)h(a)f(new)75 2143 y(con)o(text;)h(the)g(lo)q(cal)e (leader)i(sends)h(a)e(description)h(of)f(the)h(group)f(and)g(the)h(con) o(text)g(to)f(the)h(serv)o(er,)h(and)e(p)q(osts)h(a)75 2200 y(receiv)o(e)g(with)e Fv(source)i(=)f(remote)p 594 2200 13 2 v 12 w(leader)p Ft(.)19 b(In)13 b(order)g(to)g(terminate)f (the)h(op)q(eration,)f(it)g(w)o(aits)g(for)g(the)i(completion)75 2256 y(of)k(the)h(p)q(osted)g(receiv)o(e,)i(next)e(broadcasts)g(the)g (information)c(within)j(the)h(group.)32 b(The)19 b(co)q(de)g(executed)h (b)o(y)75 2312 y(the)e(serv)o(er)h(is)f(simple:)24 b(It)18 b(w)o(aits)f(for)g(arriving)g(messages)h(from)d Fv(source)k Ft(=)f Fv(MPI)p 1395 2312 V 14 w(ANY)p 1494 2312 V 15 w(SOURCE)e Ft(and)h Fv(tag)h Ft(=)75 2369 y Fv(MPI)p 152 2369 V 14 w(ANY)p 251 2369 V 15 w(T)m(A)o(G)p Ft(.)11 b(When)i(t)o(w)o(o)f(messages)h(that)g(carry)g(the)h(same)d(tag)i(ha)o (v)o(e)f(arriv)o(ed,)h(then)g(the)g(meesage)g(arriv)o(ed)75 2425 y(from)f(the)j(\014rst)f(source)h(is)f(sen)o(t)h(to)f(the)g (second)h(source,)g(and)f(vice-v)o(ersa.)166 2647 y Fl(Discussion:)32 b Ft(With)12 b(this)g(de\014nition)g(w)o(e)g(sa)o(v)o(e)h(one)f (function,)g(at)g(the)h(exp)q(ense)h(of)d(restricting)i(the)g(name)75 2704 y(space)i(to)f(an)f(in)o(teger)i(tag,)e(rather)h(than)g(a)g (string.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 144 151 144 150 bop 75 -100 a FD(144)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fs(Delete)h(MPI)p 296 45 14 2 v 16 w(INTERCOMM)p 586 45 V 16 w(NAME)p 731 45 V 17 w(ST)l(ART)g(I)q(I.)75 102 y FD(In)f(a)e(call)i(to)f Fs(MPI)p 389 102 V 15 w(INTERCOMM)p 678 102 V 17 w(ST)l(ART)p FD(,)h(b)q(oth)f(group)f(leaders)i(can)f(pro) o(vide)h(an)f(implemen)o(tation-)75 158 y(de\014ned)j(p)q(eer)e(comm)o (unicator)g(and)h(p)q(eer)g(leader.)166 297 y Fl(Implemen)o(tati)o(on)f (note:)47 b Ft(The)16 b(implemen)o(tation-de\014ned)e(comm)o(unicator)g (could)i(b)q(e)h(a)g(pure)g(in)o(ter-)75 354 y(comm)o(unicator)h(whose) j(lo)q(cal)e(group)h(is)h(a)f(single)g(pro)q(cess)i(\(itself)s(\),)g (and)e(whose)h(remote)f(group)h(v)n(aries)f(as)75 410 y(pro)q(cesses)f(are)e(added)f(or)g(deleted)i(dynamically)l(,)13 b(outside)k(of)f(the)g(capabilities)g(sp)q(eci\014ed)i(within)d(MPI.)h (This)75 467 y(in)o(tercomm)o(unicator)11 b(serv)o(er)16 b(pro)q(cess)f(w)o(aits)f(on)g Fv(tag)f(=)h(MPI)p 1053 467 13 2 v 15 w(ANY)p 1153 467 V 14 w(T)m(A)o(G)f Ft(and)h Fv(src)g(=)28 b(MPI)p 1538 467 V 14 w(ANY)p 1637 467 V 15 w(SRC)p Ft(.)166 689 y Fl(Discussion:)k Ft(With)12 b(this)g(de\014nition)g(w)o(e)g(sa)o(v)o(e)h(one)f(function,)g(at)g (the)h(exp)q(ense)h(of)d(restricting)i(the)g(name)75 745 y(space)i(to)f(an)f(in)o(teger)i(tag,)e(rather)h(than)g(a)g (string.)75 973 y Fq(5.6)59 b(Caching)75 1075 y FD(MPI)22 b(pro)o(vides)h(a)e(\\cac)o(hing")h(facilit)o(y)i(that)d(allo)o(ws)h (an)g(application)i(to)d(attac)o(h)g(arbitrary)h(pieces)75 1132 y(of)17 b(information,)h(called)h Fx(attributes)p FD(,)g(to)e(comm)o(unicators.)27 b(More)17 b(precisely)l(,)j(the)e(cac) o(hing)g(facilit)o(y)75 1188 y(allo)o(ws)d(a)g(p)q(ortable)h(library)g (to)e(do)i(the)f(follo)o(wing:)143 1283 y FA(\017)23 b FD(pass)15 b(information)g(b)q(et)o(w)o(een)g(calls)i(b)o(y)e(asso)q (ciating)g(it)h(with)f(an)h(MPI)f(comm)o(unicator,)143 1379 y FA(\017)23 b FD(quic)o(kly)16 b(retriev)o(e)g(that)e (information,)h(and)143 1474 y FA(\017)23 b FD(b)q(e)18 b(guaran)o(teed)e(that)h(out-of-date)g(information)g(is)h(nev)o(er)g (retriev)o(ed,)f(ev)o(en)h(if)g(the)f(comm)o(uni-)189 1530 y(cator)d(is)i(freed)f(and)h(its)f(handle)h(subsequen)o(tly)h (reused)f(b)o(y)f(MPI.)75 1625 y(These)j(capabilities,)i(in)f(some)e (form,)g(are)h(required)g(b)o(y)g(built-in)i(MPI)e(routines)g(suc)o(h)g (as)f(collectiv)o(e)75 1682 y(comm)o(unication)g(and)g(virtual)g(top)q (ology)l(.)23 b(De\014ning)17 b(an)g(in)o(terface)f(to)g(these)h (capabilities)i(as)d(part)f(of)75 1738 y(the)f(MPI)f(standard)g(is)h(v) m(aluable)h(b)q(ecause)g(it)f(p)q(ermits)g(routines)g(lik)o(e)g (collectiv)o(e)i(comm)o(unication)e(and)75 1795 y(virtual)h(top)q (ology)g(to)f(b)q(e)h(implemen)o(ted)h(as)f(p)q(ortable)g(co)q(de,)g (and)g(also)f(b)q(ecause)i(it)f(mak)o(es)f(MPI)g(more)75 1851 y(extensible)j(b)o(y)e(allo)o(wing)h(user-written)g(routines)f(to) g(use)h(standard)e(MPI)h(calling)i(sequences.)75 1975 y Fn(5.6.1)49 b(F)o(unctionalit)o(y)75 2061 y FD(A)o(ttributes)21 b(are)f(lo)q(cal)i(to)e(the)h(pro)q(cess)g(and)g(sp)q(eci\014c)i(to)d (the)h(comm)o(unicator)g(to)f(whic)o(h)h(they)g(are)75 2118 y(attac)o(hed.)d(They)13 b(are)e(not)h(propagated)f(b)o(y)h(MPI)g (from)g(one)g(comm)o(unicator)g(to)f(another)h(except)g(when)75 2174 y(the)f(comm)o(unicator)f(is)h(duplicated)i(using)e Fs(MPI)p 899 2174 14 2 v 16 w(COMM)p 1057 2174 V 16 w(DUP)g FD(\(and)g(ev)o(en)g(then)g(the)f(application)j(m)o(ust)75 2231 y(giv)o(e)j(sp)q(eci\014c)i(p)q(ermission)f(for)e(the)h(attribute) g(to)f(b)q(e)i(copied\).)22 b(A)o(ttributes)16 b(are)g(scalar)f(v)m (alues,)i(equal)75 2287 y(in)f(size)g(to,)e(or)h(larger)g(than)g(a)g (C-language)g(p)q(oin)o(ter.)166 2344 y(The)10 b(cac)o(hing)h(in)o (terface)g(de\014ned)g(here)g(represen)o(ts)f(that)f(attributes)h(are)g (stored)g(b)o(y)g(MPI)g(opaquely)75 2400 y(within)16 b(a)f(comm)o(unicator.)20 b(Accessor)15 b(functions)h(include)i(the)d (follo)o(wing:)143 2496 y FA(\017)23 b FD(obtain)16 b(a)f(k)o(ey)h(v)m (alue)h(\(used)f(to)g(iden)o(tify)h(an)f(attribute\);)f(the)h(user)g (sp)q(eci\014es)i(\\callbac)o(k")e(func-)189 2552 y(tions)g(b)o(y)h (whic)o(h)h(MPI)e(informs)h(the)g(application)h(when)f(the)g(comm)o (unicator)f(is)h(destro)o(y)o(ed)f(or)189 2608 y(copied.)143 2704 y FA(\017)23 b FD(store)14 b(and)i(retriev)o(e)f(the)g(v)m(alue)i (of)d(an)i(attribute;)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 145 152 145 151 bop 75 -100 a Fp(5.6.)34 b(CA)o(CHING)1407 b FD(145)166 45 y Fl(Discussion:)49 b Ft(Cac)o(hing)17 b(and)g(callbac)o(k)g(functions)h(are)g(only)f(called)g(sync)o (hronously)m(,)h(in)g(resp)q(onse)h(to)75 95 y(explicit)f(application)e (request.)33 b(This)18 b(a)o(v)o(oid)e(problems)h(due)i(to)f(rep)q (eated)i(crossings)f(form)d(user)j(to)f(system)75 145 y(space.)166 197 y(The)10 b(c)o(hoice)g(of)e(k)o(ey)i(v)n(alues)f(is)g (under)i(con)o(trol)e(of)g(MPI.)g(This)g(allo)o(ws)f(MPI)i(to)f (optimize)f(its)h(implemen)o(tation)75 246 y(of)15 b(attribute)i(sets.) 26 b(It)16 b(also)g(a)o(v)o(oids)f(con\015ict)h(b)q(et)o(w)o(een)i (indep)q(enden)o(t)f(mo)q(dules)e(cac)o(hing)h(information)d(on)j(the) 75 296 y(same)d(comm)o(unicators.)166 355 y(A)i(m)o(uc)o(h)f(smaller)g (in)o(terface,)h(consisting)g(of)g(just)g(a)g(callbac)o(k)g(facilit)o (y)m(,)e(w)o(ould)h(allo)o(w)f(the)j(en)o(tire)g(cac)o(hing)75 411 y(facilit)o(y)f(to)h(b)q(e)h(implemen)o(ted)d(b)o(y)j(p)q(ortable)f (co)q(de.)27 b(Ho)o(w)o(ev)o(er,)17 b(with)f(the)h(minim)o(al)c (callbac)o(k)j(in)o(terface,)h(some)75 468 y(form)d(of)g(table)h(searc) o(hing)h(is)f(implied)e(b)o(y)i(the)h(need)g(to)f(handle)h(arbitrary)f (comm)o(unicator.)k(In)c(con)o(trast,)h(the)75 524 y(more)d(complete)g (in)o(terface)i(de\014ned)g(here)g(p)q(ermits)f(rapid)f(access)j(to)e (attributes)h(through)f(the)g(use)h(of)e(p)q(oin)o(ters)75 581 y(in)f(comm)o(unicators)f(\(to)i(\014nd)g(the)g(attribute)h (table\))f(and)f(clev)o(erly)i(c)o(hosen)f(k)o(ey)g(v)n(alues)g(\(to)g (retriev)o(e)h(individual)75 637 y(attributes\).)39 b(In)20 b(ligh)o(t)f(of)h(the)h(e\016ciency)g(hit)f(inheren)o(t)h(in)f(the)h (minim)o(al)c(in)o(terface,)22 b(the)f(more)f(complete)75 694 y(in)o(terface)14 b(prop)q(osed)h(here)g(is)f(seen)h(to)f(b)q(e)g (sup)q(erior.)75 835 y FD(MPI)h(pro)o(vides)h(the)f(follo)o(wing)h (services)g(related)g(to)e(cac)o(hing.)21 b(They)15 b(are)g(all)h(lo)q (cal.)75 941 y Fs(MPI)p 160 941 14 2 v 16 w(A)l(TTR)p 293 941 V 17 w(CREA)l(TE)p 479 941 V 17 w(KEY\(cop)o(y)p 689 941 V 17 w(fn,)f(delete)p 881 941 V 17 w(fn,)h(k)o(eyval,)e(extra)p 1202 941 V 16 w(state\))117 1020 y Ft(IN)155 b Fs(cop)o(y)p 407 1020 V 17 w(fn)447 b Ft(Cop)o(y)14 b(callbac)o(k)f(function)g(for)h Fv(k)o(eyval)117 1099 y Ft(IN)155 b Fs(delete)p 431 1099 V 18 w(fn)422 b Ft(Delete)15 b(callbac)o(k)e(function)g(for)h Fv(k)o(eyval)117 1179 y Ft(OUT)108 b Fs(k)o(eyval)471 b Ft(k)o(ey)14 b(v)n(alue)f(for)h(future)g(access)i(\(in)o(teger\))117 1258 y(IN)155 b Fs(extra)p 416 1258 V 17 w(state)384 b Ft(Extra)14 b(state)h(for)f(callbac)o(k)f(functions)75 1385 y FB(int)23 b(MPI)p 245 1385 15 2 v 17 w(Attr)p 358 1385 V 17 w(create)p 519 1385 V 16 w(key\(int)g(copy)p 822 1385 V 17 w(fn\(\),)g(int)g(delete)p 1221 1385 V 17 w(fn\(\),)g(int)g(keyval,)393 1441 y(void*)g(extra)p 659 1441 V 17 w(state\))75 1530 y(MPI)p 150 1530 V 17 w(ATTR)p 263 1530 V 16 w(CREATE)p 423 1530 V 17 w(KEY\(COPY)p 632 1530 V 16 w(FN,)g(DELETE)p 887 1530 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1310 1530 V 17 w(STATE,)g(IERROR\))170 1586 y(EXTERNAL)g(COPY)p 484 1586 V 17 w(FN,)g(DELETE)p 740 1586 V 17 w(FN)170 1643 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 1643 V 17 w(STATE,)g(IERROR)166 1731 y FD(Generates)13 b(a)g(new)h(attribute)g(k)o(ey)l(.)19 b(Keys)14 b(are)g(lo)q(cally)h(unique)g(in)g(pro)q(cess,)f(and)f (opaque)h(to)f(user,)75 1788 y(though)i(they)h(are)f(explicitly)k (stored)c(in)h(in)o(tegers.)21 b(Once)c(allo)q(cated,)f(the)f(k)o(ey)h (v)m(alue)h(can)e(b)q(e)h(used)h(to)75 1844 y(asso)q(ciate)e (attributes)g(and)h(access)f(them)g(on)g(an)o(y)g(lo)q(cally)i (de\014ned)g(comm)o(unicator.)166 1903 y(The)h Fs(cop)o(y)p 351 1903 14 2 v 17 w(fn)g FD(function)h(is)g(in)o(v)o(ok)o(ed)f(when)h (a)f(comm)o(unicator)g(is)h(duplicated.)30 b(A)19 b(C)f(protot)o(yp)q (e)75 1959 y(for)d(suc)o(h)g(a)g(function)h(is)166 2018 y Fs(int)g(cop)o(y)p 321 2018 V 16 w(fn\(oldcomm)n(,)11 b(new)o(comm)n(,)h(k)o(eyval,)i(extra)p 1047 2018 V 17 w(state\))166 2076 y FD(This)22 b(function)h(do)q(es)g(whatev)o(er)e (it)h(needs)h(to)e(do)h(with)g(the)h(new)f(comm)o(unicator)f Fs(new)o(comm)75 2133 y FD(based)16 b(on)f(what)f(attribute)h (information)h(w)o(as)e(in)i(the)f(old)h(comm)o(unicator)f Fs(oldcomm)m FD(;)d(it)k(returns)f(1)f(if)75 2189 y(successful,)h(0)e (if)h(not.)19 b(A)14 b(v)m(alid)h(cop)o(y)f(function)g(is)g(one)g(that) f(completely)i(duplicates)g(the)f(information)75 2246 y(b)o(y)20 b(making)g(a)f(full)i(duplicate)g(cop)o(y)f(of)f(the)h(data) f(structures)h(implied)i(b)o(y)d(an)h(attribute;)h(another)75 2302 y(migh)o(t)f(just)g(mak)o(e)g(another)g(reference)h(to)e(that)h (data)g(structure,)g(while)i(using)f(a)f(reference-coun)o(t)75 2359 y(mec)o(hanism.)37 b(Other)21 b(t)o(yp)q(es)g(of)f(attributes)h (migh)o(t)g(not)f(cop)o(y)h(at)f(all)h(\(they)g(migh)o(t)g(b)q(e)g(sp)q (eci\014c)i(to)75 2415 y Fs(oldcomm)11 b FD(only\).)24 b Fs(MPI)p 488 2415 V 16 w(COMM)p 646 2415 V 16 w(DUP)17 b FD(is)g(the)g(only)g(function)g(that)f(calls)i(the)e Fs(cop)o(y)p 1547 2415 V 17 w(fn)p FD(.)24 b Fs(cop)o(y)p 1724 2415 V 17 w(fn)17 b FD(ma)o(y)75 2471 y(b)q(e)f(sp)q(eci\014ed)h (as)e(n)o(ull,)h(in)g(whic)o(h)g(case)g(no)f(cop)o(y)g(callbac)o(k)h(o) q(ccurs)g(for)e Fs(k)o(eyval)p FD(.)166 2530 y(Analogous)h(to)g Fs(cop)o(y)p 532 2530 V 16 w(fn)h FD(is)f(a)g(callbac)o(k)i(deletion)f (function,)g(de\014ned)g(as)f(follo)o(ws:)166 2589 y Fs(delete)p 279 2589 V 18 w(fn\(comm)m(,)d(k)o(eyval,)i(extra)p 733 2589 V 16 w(state\))166 2647 y FD(This)e(function)g(is)g(called)i (b)o(y)d Fs(MPI)p 752 2647 V 16 w(COMM)p 910 2647 V 16 w(FREE)h FD(and)g Fs(MPI)p 1214 2647 V 16 w(A)l(TTR)p 1347 2647 V 17 w(DELETE)f FD(to)g(do)h(whatev)o(er)e(is)75 2704 y(needed)15 b(to)f(remo)o(v)o(e)f(an)h(attribute.)19 b(It)14 b(ma)o(y)f(b)q(e)i(sp)q(eci\014ed)h(as)d(n)o(ull,)j(in)e(whic)o (h)h(case)f(no)g(delete)h(callbac)o(k)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 146 153 146 152 bop 75 -100 a FD(146)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FD(o)q(ccurs)h(for)e Fs(k)o(eyval)p FD(.)75 150 y Fs(MPI)p 160 150 14 2 v 16 w(A)l(TTR)p 293 150 V 17 w(FREE)p 419 150 V 17 w(KEY\(k)o(eyval\))117 229 y Ft(IN)155 b Fs(k)o(eyval)471 b Ft(F)m(rees)15 b(the)g(in)o(teger)f(k)o(ey)g(v)n (alue)f(\(in)o(teger\))75 354 y FB(int)23 b(MPI)p 245 354 15 2 v 17 w(Attr)p 358 354 V 17 w(free)p 471 354 V 16 w(key\(int)g(keyval\))75 442 y(MPI)p 150 442 V 17 w(ATTR)p 263 442 V 16 w(FREE)p 375 442 V 17 w(KEY\(KEYVAL,)f(IERROR\)) 170 499 y(INTEGER)h(KEYVAL,)g(IERROR)166 586 y FD(F)l(rees)13 b(an)g(extan)o(t)f(attribute)h(k)o(ey)l(.)19 b(It)14 b(is)f(not)g(erroneous)g(to)f(free)h(an)g(attribute)g(k)o(ey)g(that)g (is)g(in)h(use,)75 643 y(b)q(ecause)h(the)e(actual)h(free)f(do)q(es)h (not)f(transpire)h(un)o(til)h(after)e(all)h(references)g(\(in)g(other)f (comm)o(unicators)75 699 y(on)20 b(the)g(pro)q(cess\))f(to)h(the)g(k)o (ey)f(ha)o(v)o(e)h(b)q(een)h(freed.)34 b(These)20 b(references)g(need)h (to)e(b)q(e)i(explictly)h(freed)75 756 y(b)o(y)d(the)h(program,)f (either)h(via)f(calls)i(to)e Fs(MPI)p 887 756 14 2 v 15 w(A)l(TTR)p 1019 756 V 18 w(DELETE)g FD(that)f(free)i(one)f (attribute)h(instance,)75 812 y(or)c(b)o(y)g(calls)h(to)e Fs(MPI)p 439 812 V 16 w(COMM)p 597 812 V 17 w(FREE)h FD(that)g(free)g(all)h(attribute)f(instances)h(asso)q(ciated)f(with)g (the)h(freed)75 869 y(comm)o(unicator.)75 974 y Fs(MPI)p 160 974 V 16 w(PUT)p 267 974 V 16 w(A)l(TTR\(comm)n(,)12 b(k)o(eyval,)i(attribute)p 868 974 V 18 w(val\))117 1052 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)7 b(to)i(whic)o(h)g (attribute)h(will)e(b)q(e)h(attac)o(hed)h(\(han-)905 1109 y(dle\))117 1186 y(IN)155 b Fs(k)o(eyval)471 b Ft(k)o(ey)14 b(v)n(alue,)f(as)h(returned)h(b)o(y)905 1243 y Fv(MPI)p 982 1243 13 2 v 15 w(A)m(TTR)p 1105 1243 V 14 w(CREA)m(TE)p 1276 1243 V 13 w(KEY)e Ft(\(in)o(teger\))117 1320 y(IN)155 b Fs(attribute)p 484 1320 14 2 v 18 w(val)352 b Ft(attribute)15 b(v)n(alue)75 1446 y FB(int)23 b(MPI)p 245 1446 15 2 v 17 w(Put)p 334 1446 V 17 w(attr\(MPI)p 543 1446 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 1446 V 16 w(val\))75 1534 y(MPI)p 150 1534 V 17 w(PUT)p 239 1534 V 17 w(ATTR\(COMM,)f(KEYVAL,)h(ATTRIBUTE)p 925 1534 V 16 w(VAL,)g(IERROR\))170 1590 y(INTEGER)g(COMM,)h(KEYVAL,)e (ATTRIBUTE)p 914 1590 V 16 w(VAL,)i(IERROR)166 1678 y FD(This)14 b(function)g(stores)f(the)g(stipulated)i(attribute)e(v)m (alue)i Fs(attribute)p 1331 1678 14 2 v 18 w(val)e FD(for)g(subsequen)o (t)h(retriev)m(al)75 1735 y(b)o(y)c Fs(MPI)p 218 1735 V 16 w(GET)p 322 1735 V 17 w(A)l(TTR)p FD(.)g(If)h(the)f(v)m(alue)h(is) g(already)f(presen)o(t,)h(then)f(the)h(outcome)e(is)i(as)f(if)h Fs(MPI)p 1627 1735 V 15 w(DELETE)p 1812 1735 V 16 w(A)l(TTR)75 1791 y FD(w)o(as)17 b(\014rst)h(called)i(to)e(delete)h(the)f(previous)h (v)m(alue)h(\(and)e(the)g(callbac)o(k)i(function)f Fs(delete)p 1635 1791 V 17 w(fn)g FD(w)o(as)e(exe-)75 1848 y(cuted\),)d(and)h(a)e (new)i(v)m(alue)g(w)o(as)f(next)g(stored.)19 b(The)14 b(call)i(is)e(erroneous)g(if)h(there)f(is)h(no)f(k)o(ey)g(with)h(v)m (alue)75 1904 y Fs(k)o(eyval)p FD(.)75 2009 y Fs(MPI)p 160 2009 V 16 w(GET)p 264 2009 V 17 w(A)l(TTR\(comm)m(,)d(k)o(eyval,)i (attribute)p 865 2009 V 18 w(val,)h(found\))117 2088 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)10 b(to)i(whic)o(h)g (attribute)h(is)f(attac)o(hed)h(\(handle\))117 2165 y(IN)155 b Fs(k)o(eyval)471 b Ft(k)o(ey)14 b(v)n(alue)f(\(in)o(teger\))117 2243 y(OUT)108 b Fs(attribute)p 484 2243 V 18 w(val)352 b Ft(attribute)15 b(v)n(alue,)d(unless)j Fv(found)g Ft(=)f(false)117 2321 y(OUT)108 b Fs(found)481 b Fb(true)14 b Ft(if)h(an)f(attribute)i (v)n(alue)e(w)o(as)h(extracted;)i Fb(false)d Ft(if)g(no)905 2377 y(attribute)h(is)e(asso)q(ciated)i(with)f(the)g(k)o(ey)75 2503 y FB(int)23 b(MPI)p 245 2503 15 2 v 17 w(Get)p 334 2503 V 17 w(attr\(MPI)p 543 2503 V 16 w(Comm)g(comm,)h(int)f(keyval,)g (void*)g(attribute)p 1467 2503 V 16 w(val,)g(int)h(*found\))75 2591 y(MPI)p 150 2591 V 17 w(GET)p 239 2591 V 17 w(ATTR\(COMM,)e (KEYVAL,)h(ATTRIBUTE)p 925 2591 V 16 w(VAL,)g(FOUND,)g(IERROR\))170 2647 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 2647 V 16 w(VAL,)i(IERROR)170 2704 y(LOGICAL)f(FOUND)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 147 154 147 153 bop 75 -100 a Fp(5.6.)29 b(CA)o(CHING)1412 b FD(147)166 45 y(Retriev)o(es)19 b(attribute)g(v)m(alue)h(b)o(y)f(k)o (ey)l(.)30 b(The)19 b(call)h(is)f(erroneous)g(if)g(there)g(is)g(no)g(k) o(ey)g(with)g(v)m(alue)75 102 y Fs(k)o(eyval)p FD(.)30 b(On)19 b(the)g(other)f(hand,)h(the)g(call)h(is)f(correct)f(if)h(the)g (k)o(ey)f(v)m(alue)i(exists,)g(but)e(no)h(attribute)f(is)75 158 y(attac)o(hed)d(on)g FB(comm)f FD(for)h(that)f(k)o(ey;)h(in)h(suc)o (h)g(case,)e(the)i(call)g(returns)f FB(found)23 b(=)h(false)p FD(.)75 262 y Fs(MPI)p 160 262 14 2 v 16 w(DELETE)p 346 262 V 16 w(A)l(TTR\(comm)n(,)11 b(k)o(eyval\))117 339 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)10 b(to)i(whic)o(h)g (attribute)h(is)f(attac)o(hed)h(\(handle\))117 413 y(IN)155 b Fs(k)o(eyval)471 b Ft(The)15 b(k)o(ey)e(v)n(alue)h(of)f(the)i (deleted)f(attribute)h(\(in)o(teger\))75 537 y FB(int)23 b(MPI)p 245 537 15 2 v 17 w(Delete)p 406 537 V 17 w(Attr\(MPI)p 615 537 V 16 w(Comm)g(comm,)g(int)h(keyval\))75 623 y(MPI)p 150 623 V 17 w(DELETE)p 311 623 V 16 w(ATTR\(COMM,)f(KEYVAL,)g (IERROR\))170 680 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)166 766 y FD(Delete)14 b(attribute)g(from)f(cac)o(he)h(b)o(y)f(k)o(ey)l(.) 20 b(This)14 b(function)g(in)o(v)o(ok)o(es)g(the)g(attribute)f(delete)i (function)75 823 y Fs(delete)p 188 823 14 2 v 18 w(fn)g FD(sp)q(eci\014ed)j(when)d(the)h Fs(k)o(eyval)e FD(w)o(as)h(created.) 166 879 y(Whenev)o(er)h(a)g(comm)o(unicator)f(is)h(replicated)i(using)e (the)g(function)h Fs(MPI)p 1435 879 V 16 w(COMM)p 1593 879 V 16 w(DUP)p FD(,)f(all)h(call-)75 936 y(bac)o(k)h(cop)o(y)f (functions)i(for)e(attributes)g(that)g(are)h(curren)o(tly)g(set)f(are)h (in)o(v)o(ok)o(ed)g(\(in)g(arbitrary)f(order\).)75 992 y(Whenev)o(er)23 b(a)f(comm)o(unicator)g(is)g(deleted)i(using)f(the)g (function)g Fs(MPI)p 1343 992 V 15 w(COMM)p 1500 992 V 17 w(FREE)g FD(all)g(callbac)o(k)75 1049 y(delete)16 b(functions)g(for)f(attributes)g(that)f(are)h(curren)o(tly)h(set)f(are) g(in)o(v)o(ok)o(ed.)75 1167 y Fs(Example)75 1253 y FD(This)j(example)g (sho)o(ws)e(ho)o(w)h(to)f(write)i(a)e(collectiv)o(e)j(comm)o(unication) f(op)q(eration)f(that)g(uses)g(cac)o(hing)75 1310 y(to)d(b)q(e)h(more)f (e\016cien)o(t)h(after)f(the)h(\014rst)f(call.)21 b(The)14 b(co)q(ding)i(st)o(yle)f(assumes)f(that)g(MPI)g(function)i(results)75 1366 y(return)f(only)h(error)e(statuses.)147 1462 y FB(static)23 b(int)g(gop_key_assigned)f(=)h(0;)48 b(/*)24 b(0)f(only)h(on)f(first)g (entry)h(*/)147 1518 y(static)f(int)g(gop_key;)357 b(/*)24 b(key)f(for)g(this)h(module's)f(stuff)g(*/)147 1631 y(typedef)f(struct) 147 1687 y({)218 1744 y(int)i(ref_count;)237 b(/*)23 b(reference)g(count)g(*/)218 1800 y(/*)h(other)f(stuff,)g(whatever)g (else)g(we)h(want)f(*/)147 1857 y(})g(gop_stuff_type;)147 1970 y(Efficient_Collective_Op)d(\(comm,)j(...\))147 2026 y(MPI_Comm)f(comm;)147 2083 y({)194 2139 y(gop_stuff_type)g (*gop_stuff;)194 2195 y(MPI_Group)166 b(group;)194 2252 y(int)310 b(foundflag;)194 2365 y(MPI_Comm_group\(comm,)21 b(&group\);)194 2478 y(if)j(\(!gop_key_assigned\))117 b(/*)23 b(get)h(a)f(key)h(on)f(first)h(call)f(ever)g(*/)194 2534 y({)242 2591 y(gop_key_assigned)f(=)h(1;)242 2647 y(if)h(\()f(!)h(MPI_Attr_get_key\()e(gop_stuff_copier,)839 2704 y(gop_stuff_destructor,)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 148 155 148 154 bop 75 -100 a FD(148)123 b Fp(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)839 45 y FB(&gop_key,)22 b(\(void)h(*\)0\)\);)242 102 y(/*)h(get)f(the)h (key)f(while)g(assigning)g(its)g(copy)h(and)f(delete)g(callback)314 158 y(behavior.)f(*/)242 271 y(MPI_Abort)h(\("Insufficient)f(keys)h (available"\);)194 327 y(})194 440 y(MPI_Attr_get_value)f(\(comm,)h (gop_key,)g(&gop_stuff,)f(&foundflag\);)194 497 y(if)i(\(foundflag\)) 194 553 y({)g(/*)g(This)f(module)g(has)g(executed)g(in)h(this)f(group)g (before.)314 610 y(We)g(will)h(use)f(the)g(cached)g(information)g(*/) 194 666 y(})194 723 y(else)194 779 y({)h(/*)g(This)f(is)h(a)f(group)g (that)h(we)f(have)h(not)f(yet)h(cached)f(anything)f(in.)314 835 y(We)h(will)h(now)f(do)h(so.)242 892 y(*/)242 1005 y(/*)g(First,)f(allocate)f(storage)h(for)h(the)f(stuff)g(we)h(want,)314 1061 y(and)f(initialize)g(the)g(reference)g(count)g(*/)242 1174 y(gop_stuff)g(=)g(\(gop_stuff_type)f(*\))i(malloc)f (\(sizeof\(gop_stuff_type\)\);)242 1231 y(if)h(\(gop_stuff)e(==)i (NULL\))f({)h(/*)f(abort)g(on)h(out-of-memory)e(error)h(*/)h(})242 1344 y(gop_stuff)f(->)g(ref_count)g(=)h(1;)242 1456 y(/*)g(Second,)f (fill)g(in)g(*gop_stuff)g(with)g(whatever)g(we)h(want.)314 1513 y(This)f(part)g(isn't)g(shown)h(here)f(*/)242 1626 y(/*)h(Third,)f(store)g(gop_stuff)f(as)i(the)g(attribute)e(value)h(*/) 242 1682 y(MPI_Attr_put_value)e(\()j(comm,)f(gop_key,)g(gop_stuff\);) 194 1739 y(})194 1795 y(/*)h(Then,)f(in)h(any)f(case,)g(use)h(contents) f(of)g(*gop_stuff)266 1852 y(to)g(do)h(the)g(global)f(op)g(...)h(*/)147 1908 y(})147 2021 y(/*)f(The)h(following)e(routine)h(is)h(called)f(by)g (MPI)h(when)f(a)h(group)f(is)h(freed)f(*/)147 2134 y (gop_stuff_destructor)e(\(comm,)i(keyval,)g(gop_stuff,)f(extra\))147 2190 y(MPI_Comm)g(comm;)147 2247 y(int)h(keyval;)147 2303 y(gop_stuff_type)f(*gop_stuff;)147 2360 y(void)h(*extra;)147 2416 y({)194 2473 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h (--)f(programming)g(error)g(*/)g(})194 2586 y(/*)h(The)f(group's)g (being)g(freed)h(removes)f(one)g(reference)g(to)g(gop_stuff)g(*/)194 2642 y(gop_stuff)g(->)h(ref_count)e(-=)i(1;)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 149 156 149 155 bop 75 -100 a Fp(5.7.)29 b(F)o(ORMALIZING)11 b(THE)g(LOOSEL)l(Y)i(SYNCHR)o(ONOUS)g(MODEL)e(\(USA)o(GE,)f(SAFETY\))p FD(149)194 45 y FB(/*)24 b(If)f(no)h(references)f(remain,)f(then)i (free)f(the)h(storage)e(*/)194 102 y(if)i(\(gop_stuff)e(->)i(ref_count) f(==)g(0\))h({)242 158 y(free\(\(void)f(*\)gop_stuff\);)194 214 y(})147 271 y(})147 384 y(/*)g(The)h(following)e(routine)h(is)h (called)f(by)g(MPI)h(when)f(a)h(group)f(is)h(copied)f(*/)147 440 y(gop_stuff_copier)e(\(comm,)i(keyval,)g(gop_stuff,)g(extra\))147 497 y(MPI_Comm)f(comm;)147 553 y(int)h(keyval;)147 610 y(gop_stuff_type)f(*gop_stuff;)147 666 y(void)h(*extra;)147 723 y({)194 779 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h(--) f(programming)g(error)g(*/)g(})194 892 y(/*)h(The)f(new)h(group)f(adds) g(one)h(reference)e(to)i(this)f(gop_stuff)g(*/)194 948 y(gop_stuff)g(->)h(ref_count)e(+=)i(1;)147 1005 y(})75 1174 y Fq(5.7)59 b(F)n(o)n(rmalizing)21 b(the)e(Lo)r(osely)g (Synchronous)g(Mo)r(del)g(\(Usage,)g(Safet)n(y\))75 1286 y Fn(5.7.1)49 b(Basic)17 b(Stateme)o(nts)75 1380 y FD(When)e(a)g (caller)h(passes)e(a)h(comm)o(unicator)f(\(that)g(con)o(tains)h(a)g (con)o(text)f(and)h(group\))f(to)g(a)h(callee,)h(that)75 1437 y(comm)o(unicator)g(m)o(ust)h(b)q(e)g(free)g(of)f(side)i (e\013ects)e(throughout)g(execution)i(of)f(the)g(subprogram:)22 b(there)75 1493 y(should)16 b(b)q(e)f(no)g(activ)o(e)g(op)q(erations)g (on)g(that)f(comm)o(unicator)g(that)g(migh)o(t)h(in)o(v)o(olv)o(e)g (the)g(pro)q(cess.)20 b(This)75 1549 y(pro)o(vides)i(one)g(mo)q(del)h (in)f(whic)o(h)h(libraries)g(can)f(b)q(e)h(written,)g(and)f(w)o(ork)f (\\safely)l(.")39 b(F)l(or)21 b(libraries)75 1606 y(so)e(designated,)h (the)g(callee)h(has)e(p)q(ermission)h(to)f(do)g(whatev)o(er)g(comm)o (unication)h(it)f(lik)o(es)h(with)g(the)75 1662 y(comm)o(unicator,)e (and)h(under)g(the)f(ab)q(o)o(v)o(e)g(guaran)o(tee)g(kno)o(ws)g(that)f (no)h(other)g(comm)o(unications)h(will)75 1719 y(in)o(terfere.)36 b(Since)22 b(w)o(e)f(p)q(ermit)g(go)q(o)q(d)f(implemen)o(tations)i(to)e (create)g(new)h(comm)o(unicators)f(without)75 1775 y(sync)o (hronization)14 b(\(suc)o(h)g(as)f(b)o(y)g(preallo)q(cated)i(con)o (texts)e(on)g(comm)o(unicators\),)g(this)g(do)q(es)h(not)f(imp)q(ose)75 1832 y(a)i(signi\014can)o(t)h(o)o(v)o(erhead.)166 1893 y(This)k(form)f(of)g(safet)o(y)f(is)i(analogous)f(to)g(other)g(common)g (computer-science)j(usages,)e(suc)o(h)f(as)75 1949 y(passing)c(a)f (descriptor)h(of)f(an)h(arra)o(y)e(to)h(a)g(library)i(routine.)k(The)15 b(library)g(routine)g(has)g(ev)o(ery)f(righ)o(t)h(to)75 2006 y(exp)q(ect)h(suc)o(h)f(a)g(descriptor)h(to)f(b)q(e)g(v)m(alid)i (and)f(mo)q(di\014able.)75 2153 y Fn(5.7.2)49 b(Mo)q(dels)17 b(of)g(Execution)75 2247 y FD(In)12 b(the)g(lo)q(osely)h(sync)o (hronous)f(mo)q(del,)h(transfer)e(of)h(con)o(trol)f(to)h(a)f Fx(parallel)16 b(pro)q(cedure)c FD(is)g(e\013ected)g(b)o(y)75 2304 y(ha)o(ving)f(eac)o(h)g(executing)g(pro)q(cess)g(in)o(v)o(ok)o(e)g (the)f(pro)q(cedure.)19 b(The)11 b(in)o(v)o(o)q(cation)g(is)g(a)g (collectiv)o(e)h(op)q(eration:)75 2360 y(it)i(is)h(executed)g(b)o(y)f (all)h(pro)q(cesses)f(in)h(the)g(execution)g(group,)e(and)i(in)o(v)o(o) q(cations)f(are)g(similarly)i(ordered)75 2417 y(at)f(all)h(pro)q (cesses.)k(Ho)o(w)o(ev)o(er,)14 b(the)h(in)o(v)o(o)q(cation)h(need)g (not)f(b)q(e)h(sync)o(hronized.)166 2478 y(W)l(e)d(sa)o(y)g(that)g(a)g (parallel)i(pro)q(cedure)f(is)g Fo(active)f FD(in)h(a)g(pro)q(cess)f (if)h(the)f(pro)q(cess)h(b)q(elongs)g(to)f(a)g(group)75 2534 y(that)j(ma)o(y)h(collectiv)o(ely)i(execute)f(the)f(pro)q(cedure,) h(and)f(some)f(mem)o(b)q(er)h(of)g(that)f(group)h(is)g(curren)o(tly)75 2591 y(executing)c(the)f(pro)q(cedure)h(co)q(de.)19 b(If)13 b(a)e(parallel)j(pro)q(cedure)f(is)f(activ)o(e)g(in)h(a)f(pro)q(cess,)g (then)g(this)h(pro)q(cess)75 2647 y(ma)o(y)e(b)q(e)i(receiving)g (messages)e(p)q(ertaining)i(to)f(this)g(pro)q(cedure,)h(ev)o(en)f(if)g (it)g(do)q(es)h(not)e(curren)o(tly)h(execute)75 2704 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 150 157 150 156 bop 75 -100 a FD(150)123 b Fp(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fs(Static)h(comm)m(unicato)o(r)d(allo)q(cation)75 134 y FD(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o(y)g (p)q(oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(o)q(cation)g(of) f(a)g(parallel)75 190 y(pro)q(cedure)c(can)f(b)q(e)h(activ)o(e)f(at)f (an)o(y)h(pro)q(cess,)g(and)g(the)g(group)f(of)h(executing)h(pro)q (cesses)f(is)h(\014xed.)23 b(F)l(or)75 247 y(example,)d(all)g(in)o(v)o (o)q(cations)f(of)g(parallel)h(pro)q(cedures)g(in)o(v)o(olv)o(e)f(all)h (pro)q(cesses,)g(pro)q(cesses)f(are)f(single-)75 303 y(threaded,)d(and)h(there)f(are)g(no)g(recursiv)o(e)h(in)o(v)o(o)q (cations.)166 361 y(In)k(suc)o(h)g(a)g(case,)g(a)g(comm)o(unicator)f (can)h(b)q(e)g(statically)h(allo)q(cated)f(to)g(eac)o(h)f(pro)q (cedure.)35 b(The)75 418 y(static)18 b(allo)q(cation)h(can)f(b)q(e)h (done)g(in)g(a)e(pream)o(ble,)j(as)d(part)h(of)g(initialization)j(co)q (de.)29 b(If)18 b(the)g(parallel)75 474 y(pro)q(cedures)g(can)f(b)q(e)g (organized)g(in)o(to)g(libraries,)h(so)f(that)f(only)h(one)g(pro)q (cedure)h(of)e(eac)o(h)h(library)h(can)75 531 y(b)q(e)e(concurren)o (tly)g(activ)o(e)g(in)g(eac)o(h)f(pro)q(cessor,)g(then)h(it)g(is)g (su\016cien)o(t)g(to)f(allo)q(cate)h(one)f(comm)o(unicator)75 587 y(p)q(er)h(library)l(.)75 716 y Fs(Dynamic)c(comm)n(unicato)o(r)g (allo)q(cation)75 805 y FD(Calls)j(of)g(parallel)h(pro)q(cedures)g(are) e(w)o(ell-nested)i(if)g(a)e(new)h(parallel)h(pro)q(cedure)g(is)f(alw)o (a)o(ys)f(in)o(v)o(ok)o(ed)h(in)75 861 y(a)j(subset)h(of)e(a)h(group)g (executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 918 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 976 y(In)i(suc)o(h)g(a)g(case,)f(a)h(new)g (comm)o(unicator)f(needs)h(to)f(b)q(e)i(dynamically)g(allo)q(cated)g (for)e(eac)o(h)h(new)75 1032 y(in)o(v)o(o)q(cation)11 b(of)f(a)g(parallel)i(pro)q(cedure.)19 b(The)11 b(allo)q(cation)g(is)g (done)g(b)o(y)f(the)h(caller.)19 b(A)11 b(new)f(comm)o(unicator)75 1089 y(can)k(b)q(e)h(generated)g(b)o(y)f(a)g(call)h(to)f Fs(MPI)p 746 1089 14 2 v 15 w(COMM)p 903 1089 V 17 w(DUP)p FD(,)g(if)g(the)h(callee)g(execution)h(group)e(is)g(iden)o(tical)j(to) 75 1145 y(the)f(caller)h(execution)g(group,)e(or)g(b)o(y)h(a)f(call)i (to)e Fs(MPI)p 1005 1145 V 16 w(COMM)p 1163 1145 V 16 w(SPLIT)g FD(if)i(the)f(caller)g(execution)h(group)75 1202 y(is)f(split)h(in)o(to)f(sev)o(eral)f(subgroups)h(executing)h (distinct)g(parallel)g(routines.)22 b(The)15 b(new)h(comm)o(unicator)75 1258 y(is)g(passed)f(as)g(an)g(argumen)o(t)g(to)f(the)h(in)o(v)o(ok)o (ed)h(routine.)166 1316 y(The)h(need)h(for)e(generating)h(a)f(new)h (comm)o(unicator)f(at)h(eac)o(h)f(in)o(v)o(o)q(cation)i(can)f(b)q(e)g (alleviated)h(or)75 1373 y(a)o(v)o(oided)e(altogether)e(in)j(some)e (cases:)20 b(If)15 b(the)h(execution)g(group)f(is)h(not)f(split,)h (then)f(one)h(can)f(allo)q(cate)75 1429 y(a)g(stac)o(k)g(of)g(comm)o (unicators)h(in)g(a)f(pream)o(ble,)h(and)g(next)g(manage)f(the)h(stac)o (k)f(in)h(a)f(w)o(a)o(y)g(that)g(mimics)75 1485 y(the)g(stac)o(k)g(of)f (recursiv)o(e)i(calls.)166 1543 y(One)h(can)f(also)g(tak)o(e)f(adv)m (an)o(tage)g(of)h(the)g(w)o(ell-ordering)h(prop)q(ert)o(y)f(of)f(comm)o (unication)i(to)e(a)o(v)o(oid)75 1600 y(confusing)f(caller)h(and)e (callee)i(comm)o(unication,)f(ev)o(en)g(if)g(b)q(oth)f(use)h(the)f (same)g(comm)o(unicator.)19 b(Do)13 b(do)75 1656 y(so,)h(one)i(needs)g (to)e(abide)j(b)o(y)e(the)g(follo)o(wing)h(t)o(w)o(o)e(rules:)143 1756 y FA(\017)23 b FD(messages)13 b(sen)o(t)g(b)q(efore)h(a)f(pro)q (cedure)h(call)h(\(or)e(b)q(efore)h(a)f(return)g(from)g(the)h(pro)q (cedure\))g(are)f(also)189 1813 y(receiv)o(ed)j(b)q(efore)f(the)h(matc) o(hing)f(call)h(\(or)f(return\))g(at)f(the)h(receiving)i(end;)143 1913 y FA(\017)23 b FD(messages)14 b(are)h(alw)o(a)o(ys)g(selected)h(b) o(y)f(source)g(\(no)g(use)h(is)g(made)f(of)g Fv(MPI)p 1430 1913 13 2 v 14 w(ANY)p 1529 1913 V 15 w(SOURCE)p FD(\).)75 2042 y Fs(The)h(General)f(case)75 2131 y FD(In)22 b(the)g(general)g(case,)h(there)e(ma)o(y)g(b)q(e)h(m)o(ultiple)i (concurren)o(tly)e(activ)o(e)g(in)o(v)o(o)q(cations)g(of)f(the)g(same) 75 2187 y(parallel)i(pro)q(cedure)g(within)g(the)f(same)f(group;)j(in)o (v)o(o)q(cations)e(ma)o(y)f(not)h(b)q(e)g(w)o(ell-nested.)41 b(A)22 b(new)75 2244 y(comm)o(unicator)15 b(need)h(to)f(b)q(e)h (created)f(for)g(eac)o(h)h(in)o(v)o(o)q(cation.)21 b(It)15 b(is)h(the)f(user)h(resp)q(onsibilit)o(y)i(to)c(mak)o(e)75 2300 y(sure)j(that,)g(if)h(t)o(w)o(o)e(distinct)i(parallel)h(pro)q (cedures)f(are)f(in)o(v)o(ok)o(ed)g(concurren)o(tly)h(on)f(o)o(v)o (erlapping)h(sets)75 2357 y(of)d(pro)q(cesses,)g(then)h(comm)o (unicator)e(creation)i(is)f(prop)q(erly)i(co)q(ordinated.)75 2509 y Fq(5.8)59 b(Motivating)19 b(Examples)75 2615 y Fn(5.8.1)49 b(Current)15 b(Practice)g(#1)75 2704 y FD(Example)h(#1a:) 1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 151 158 151 157 bop 75 -100 a Fp(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FD(151)147 45 y FB(main\(int)22 b(argc,)i(char)f (**argv\))147 102 y({)194 158 y(int)h(me,)f(size;)194 214 y(...)194 271 y(MPI_Init)g(\()h(&argc,)f(argv)g(\);)194 327 y(MPI_Comm_rank)f(\(MPI_COMM_WORLD,)g(&me\);)194 384 y(MPI_Comm_size)g(\(MPI_COMM_WORLD,)g(&size\);)194 497 y(\(void\)printf)h(\("Process)f(\045d)i(size)f(\045d\\n",)g(me,)h (size\);)194 553 y(...)194 610 y(MPI_Finalize\(\);)147 666 y(})75 764 y FD(Example)19 b(#1a)f(is)h(a)f(do-nothing)h(program)e (that)h(initializes)k(itself)d(legally)l(,)i(and)d(refers)g(to)g(the)h (the)75 821 y(\\all")f(comm)o(unicator,)g(and)g(prin)o(ts)g(a)f (message.)27 b(It)18 b(terminates)g(itself)h(legally)g(to)q(o.)27 b(This)18 b(example)75 877 y(do)q(es)e(not)e(imply)j(that)d(MPI)i(supp) q(orts)f FB(printf)p FD(-lik)o(e)g(comm)o(unication)h(itself.)75 935 y(Example)g(#1b)f(\(supp)q(osing)h(that)e FB(size)h FD(is)h(ev)o(en\):)170 1034 y FB(main\(int)23 b(argc,)g(char)h (**argv\))170 1091 y({)242 1147 y(int)f(me,)h(size;)242 1204 y(int)f(SOME_TAG)g(=)h(0;)242 1260 y(...)242 1317 y(MPI_Init\(&argc,)e(argv\);)242 1429 y(MPI_Comm_rank\(MPI_COMM_WOR)o (LD,)f(&me\);)71 b(/*)23 b(local)h(*/)242 1486 y (MPI_Comm_size\(MPI_COMM_WOR)o(LD,)d(&size\);)i(/*)g(local)h(*/)242 1599 y(if\(\(me)f(\045)h(2\))f(==)h(0\))242 1655 y({)314 1712 y(/*)f(send)h(unless)f(highest-numbered)e(process)i(*/)314 1768 y(if\(\(me)g(+)g(1\))h(<)g(size\))385 1825 y(MPI_Send\(...,)e(me)i (+)g(1,)f(SOME_TAG,)g(MPI_COMM_WORLD\);)242 1881 y(})242 1938 y(else)314 1994 y(MPI_Recv\(...,)f(me)h(-)h(1,)g(SOME_TAG,)e (MPI_COMM_WORLD\);)242 2107 y(...)242 2163 y(MPI_Finalize\(\);)170 2220 y(})75 2318 y FD(Example)16 b(#1b)f(sc)o(hematically)i (illustrates)g(message)e(exc)o(hanges)h(b)q(et)o(w)o(een)g(\\ev)o(en")f (and)h(\\o)q(dd")f(pro-)75 2375 y(cesses)h(in)g(the)f(\\all")g(comm)o (unicator.)75 2503 y Fn(5.8.2)49 b(Current)15 b(Practice)g(#2)147 2591 y FB(main\(int)22 b(argc,)i(char)f(**argv\))147 2647 y({)194 2704 y(int)h(me,)f(count;)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 152 159 152 158 bop 75 -100 a FD(152)123 b Fp(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)194 45 y FB(void)24 b(*data;)194 102 y(...)194 214 y(MPI_Init\(&argc,)e (argv\);)194 271 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&me\);)194 384 y(if\(me)i(==)h(0\))194 440 y({)290 497 y(/*)f(get)h(input,)f (create)g(buffer)g(``data'')g(*/)290 553 y(...)194 610 y(})194 723 y(MPI_Bcast\(data,)f(count,)h(MPI_BYTE,)g(0,)g (MPI_COMM_WORLD\);)194 835 y(...)194 948 y(MPI_Finalize\(\);)147 1005 y(})75 1123 y FD(This)16 b(example)g(illustrates)g(the)f(use)h(of) f(a)g(collectiv)o(e)h(comm)o(unication.)75 1258 y Fn(5.8.3)49 b(\(App)o(ro)o(ximate\))14 b(Current)h(Practice)g(#3)123 1349 y FB(main\(int)22 b(argc,)i(char)f(**argv\))123 1405 y({)170 1462 y(int)h(me,)f(count,)g(count2;)170 1518 y(void)h(*send_buf,)e(*recv_buf,)h(*send_buf2,)f(*recv_buf2;)170 1574 y(MPI_Group)h(MPI_GROUP_WORLD,)f(grp0,)h(grprem;)170 1631 y(MPI_Comm)g(commslave;)170 1687 y(static)g(int)h(ranks[])f(=)h ({0};)170 1744 y(...)170 1800 y(MPI_Init\(&argc,)e(argv\);)170 1857 y(MPI_Comm_group\(MPI_COMM_WORLD)o(,)f(&MPI_GROUP_WORLD\);)170 1913 y(MPI_Comm_rank\(MPI_COMM_WORLD,)f(&me\);)47 b(/*)24 b(local)f(*/)170 2026 y(MPI_Group_excl\(MPI_GROUP_WORL)o(D,)e(1,)j (ranks,)f(&grprem\);)46 b(/*)24 b(local)f(*/)170 2083 y(MPI_Comm_make\(MPI_COMM_WORLD,)d(grprem,)j(&commslave\);)170 2195 y(if\(me)h(!=)f(0\))170 2252 y({)218 2308 y(/*)h(compute)f(on)g (slave)g(*/)218 2365 y(...)218 2421 y(MPI_Reduce\(send_buf,recv_bu)o (ff,count)o(,)e(MPI_INT,)i(MPI_SUM,)g(1,)g(commslave\);)218 2478 y(...)170 2534 y(})170 2591 y(/*)h(zero)f(falls)h(through)e (immediately)h(to)g(this)h(reduce,)f(others)g(do)g(later...)g(*/)170 2647 y(MPI_Reduce\(send_buf2,)e(recv_buff2,)i(count2,)433 2704 y(MPI_INT,)g(MPI_SUM,)f(0,)i(MPI_COMM_WORLD\);)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 153 160 153 159 bop 75 -100 a Fp(5.8.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b FD(153)170 45 y FB(MPI_Finalize\(\);)123 102 y(})75 220 y FD(This)18 b(example)h(illustrates)g(ho)o(w)e(a)h (group)f(consisting)i(of)e(all)i(but)f(the)g(zeroth)f(pro)q(cess)h(of)g (the)g(\\all")75 276 y(group)13 b(is)h(created,)f(and)h(then)g(ho)o(w)f (a)g(comm)o(unicator)g(is)h(formed)f(\()g Fs(comm)m(slave)p FD(\))d(for)j(that)g(new)h(group.)75 333 y(The)f(new)h(comm)o(unicator) e(is)i(used)g(in)g(a)e(collectiv)o(e)j(call,)g(and)e(all)h(pro)q (cesses)f(execute)h(a)f(collectiv)o(e)i(call)75 389 y(in)j(the)33 b Fv(MPI)p 303 389 13 2 v 15 w(COMM)p 448 389 V 14 w(W)o(ORLD)18 b FD(con)o(text.)25 b(This)18 b(example)h(illustrates)f(ho)o(w)f(the)g (t)o(w)o(o)f(comm)o(unicators)75 446 y(\(that)c(inheren)o(tly)k(p)q (ossess)d(distinct)i(con)o(texts\))d(protect)h(comm)o(unication.)20 b(That)13 b(is,)h(comm)o(unication)75 502 y(in)30 b Fv(MPI)p 219 502 V 14 w(COMM)p 363 502 V 15 w(W)o(ORLD)15 b FD(is)h(insulated)g (from)f(comm)o(unication)h(in)31 b Fs(comm)m(slave)p FD(,)12 b(and)j(vice)i(v)o(ersa.)166 561 y(In)h(summary)l(,)g(\\group)f (safet)o(y")f(is)i(ac)o(hiev)o(ed)h(via)f(comm)o(unicators)f(b)q (ecause)i(distinct)g(con)o(texts)75 618 y(within)d(comm)o(unicators)f (are)g(enforced)h(to)e(b)q(e)i(unique)h(on)e(an)o(y)g(pro)q(cess.)75 753 y Fn(5.8.4)49 b(Example)15 b(#4)75 844 y FD(The)g(follo)o(wing)g (example)g(is)g(mean)o(t)f(to)f(illustrate)j(\\safet)o(y")d(b)q(et)o(w) o(een)h(p)q(oin)o(t-to-p)q(oin)o(t)i(and)e(collectiv)o(e)75 900 y(comm)o(unication.)20 b(MPI)12 b(guaran)o(tees)g(that)g(a)g (single)i(comm)o(unicator)e(can)h(do)f(safe)g(p)q(oin)o(t-to-p)q(oin)o (t)i(and)75 957 y(collectiv)o(e)j(comm)o(unication.)147 1063 y FB(#define)22 b(TAG_ARBITRARY)h(12345)147 1119 y(#define)f(SOME_COUNT)166 b(50)147 1232 y(main\(int)22 b(argc,)i(char)f(**argv\))147 1288 y({)194 1345 y(int)h(me;)194 1401 y(MPI_Group)f(MPI_GROUP_WORLD,)f(subgroup;)194 1458 y(int)i(ranks[])f(=)g({2,)h(4,)f(6,)h(8};)194 1514 y(...)194 1571 y(MPI_Init\(&argc,)e(argv\);)194 1627 y (MPI_Comm_group\(MPI_COMM_WORL)o(D,)f(&MPI_GROUP_WORLD\);)194 1740 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g(4,)j(ranks,)f (&subgroup\);)f(/*)i(local)f(*/)194 1797 y(MPI_Group_rank\(subgroup,)e (&me\);)119 b(/*)23 b(local)g(*/)194 1909 y (MPI_Comm_make\(MPI_COMM_WORLD)o(,)e(subgroup,)i(&the_comm\);)194 2022 y(if\(me)g(!=)h(MPI_UNDEFINED\))194 2079 y({)290 2135 y(MPI_Irecv\(buff1,)d(count,)i(MPI_DOUBLE,)g(MPI_ANY_SOURCE,)f (TAG_ARBITRARY,)719 2192 y(the_comm,)h(request\);)290 2248 y(MPI_Isend\(buff2,)e(count,)i(MPI_DOUBLE,)g(\(me+1\)\0454,)f (TAG_ARBITRARY,)719 2305 y(the_comm,)h(request+1\);)194 2361 y(})194 2474 y(for\(i)g(=)h(0;)g(i)g(<)f(SOME_COUNT,)g(i++\))242 2530 y(MPI_Reduce\(...,)f(the_comm\);)194 2587 y(MPI_Waitall\(2,)g (request,)h(status\);)194 2643 y(MPI_Finalize\(\);)147 2700 y(})-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 154 161 154 160 bop 75 -100 a FD(154)123 b Fp(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fn(5.8.5)49 b(Lib)o(ra)o(ry)16 b(Example)f(#1)75 132 y FD(The)g(main)h(program:)147 229 y FB(main\(int)22 b(argc,)i(char)f(**argv\))147 285 y({)194 342 y(int)h(done)f(=)h(0;)194 398 y(user_lib_t)f(*libh_a,)g(*libh_b;)194 455 y(void)h(*dataset1,)e (*dataset2;)194 511 y(...)194 568 y(MPI_Init\(&argc,)g(argv\);)194 624 y(...)194 680 y(init_user_lib\(MPI_COMM_WORLD)o(,)f(&libh_a\);)194 737 y(init_user_lib\(MPI_COMM_WORLD)o(,)g(&libh_b\);)194 793 y(...)194 850 y(user_start_op\(libh_a,)g(dataset1\);)194 906 y(user_start_op\(libh_b,)g(dataset2\);)194 963 y(...)194 1019 y(while\(!done\))194 1076 y({)266 1132 y(/*)i(work)h(*/)266 1189 y(...)266 1245 y(MPI_Reduce\(...,)e(MPI_COMM_WORLD\);)266 1301 y(...)266 1358 y(/*)h(see)h(if)g(done)f(*/)266 1414 y(...)194 1471 y(})194 1527 y(user_end_op\(libh_a\);)194 1584 y(user_end_op\(libh_b\);)194 1640 y(MPI_Finalize\(\);)147 1697 y(})75 1793 y FD(The)15 b(user)h(library)g(initialization)i(co)q (de:)147 1890 y FB(void)23 b(init_user_lib\(MPI_Comm)e(*comm,)i (user_lib_t)f(**handle\))147 1946 y({)194 2003 y(user_lib_t)h(*save;) 194 2115 y(user_lib_initsave\(&save\);)e(/*)i(local)h(*/)194 2172 y(MPI_Comm_dup\(comm,)e(&\(save)h(->)g(comm\)\);)194 2285 y(/*)h(other)f(inits)g(*/)194 2341 y(...)194 2454 y(*handle)g(=)h(save;)147 2511 y(})75 2607 y FD(User)15 b(start-up)g(co)q(de:)147 2704 y FB(void)23 b (user_start_op\(user_lib_t)d(*handle,)j(void)h(*data\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 155 162 155 161 bop 75 -100 a Fp(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FD(155)147 45 y FB({)194 102 y(MPI_Irecv\()23 b(...,)g(handle->comm,)f(&\(handle)h(->)h(irecv_handle\))e(\);)194 158 y(MPI_Isend\()h(...,)g(handle->comm,)f(&\(handle)h(->)h (isend_handle\))e(\);)147 214 y(})75 313 y FD(User)15 b(clean-up)i(co)q(de:)147 412 y FB(void)23 b(user_end_op\(user_lib_t)e (*handle\))147 469 y({)194 525 y(MPI_Status)i(*status;)194 582 y(MPI_Wait\(handle)f(->)i(isend_handle,)e(status\);)194 638 y(MPI_Wait\(handle)g(->)i(irecv_handle,)e(status\);)147 694 y(})75 823 y Fn(5.8.6)49 b(Lib)o(ra)o(ry)16 b(Example)f(#2)75 911 y FD(The)g(main)h(program:)147 1010 y FB(main\(int)22 b(argc,)i(char)f(**argv\))147 1066 y({)194 1123 y(int)h(ma,)f(mb;)194 1179 y(...)194 1292 y(static)g(int)h(list_a[])f(=)g({0,)h(1};)75 1349 y(#if)47 b(defined\(EXAMPLE_2B\))22 b(|)h(defined\(EXAMPLE_2C\)) 194 1405 y(static)g(int)h(list_b[])f(=)g({0,)h(2)f(,3};)75 1462 y(#else/*)g(EXAMPLE_2A)f(*/)194 1518 y(static)h(int)h(list_b[])f (=)g({0,)h(2};)75 1574 y(#endif)194 1631 y(int)g(size_list_a)e(=)i (sizeof\(list_a\)/sizeof\(int)o(\);)194 1687 y(int)g(size_list_b)e(=)i (sizeof\(list_b\)/sizeof\(int)o(\);)194 1800 y(...)194 1857 y(MPI_Init\(&argc,)e(argv\);)194 1913 y(...)194 2026 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)f(size_list_a,)h(list_a,)h (&group_a\);)194 2083 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)e (size_list_b,)h(list_b,)h(&group_b\);)194 2195 y (MPI_Comm_make\(MPI_COMM_WORLD)o(,)e(group_a,)i(&comm_a\);)194 2252 y(MPI_Comm_make\(MPI_COMM_WORLD)o(,)e(group_b,)i(&comm_b\);)194 2365 y(MPI_Comm_rank\(comm_a,)e(&ma\);)194 2421 y (MPI_Comm_rank\(comm_b,)g(&mb\);)194 2534 y(if\(ma)i(!=)h (MPI_UNDEFINED\))266 2591 y(lib_call\(comm_a\);)194 2704 y(if\(mb)f(!=)h(MPI_UNDEFINED\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 156 163 156 162 bop 75 -100 a FD(156)123 b Fp(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)194 45 y FB({)242 102 y(lib_call\(comm_b\);)242 158 y(lib_call\(comm_b\);) 194 214 y(})194 327 y(MPI_Finalize\(\);)147 384 y(})75 495 y FD(The)g(library:)147 612 y FB(void)23 b(lib_call\(MPI_Comm)e (comm\))147 668 y({)194 724 y(int)j(me,)f(done)h(=)f(0;)194 781 y(MPI_Comm_rank\(comm,)f(&me\);)194 837 y(if\(me)h(==)h(0\))266 894 y(while\(!done\))266 950 y({)337 1007 y(MPI_Recv\(...,)f (MPI_ANY_SOURCE,)e(MPI_ANY_TAG,)i(comm\);)337 1063 y(...)266 1120 y(})194 1176 y(else)194 1233 y({)242 1289 y(/*)h(work)f(*/)242 1345 y(MPI_Send\(...,)f(0,)i(ARBITRARY_TAG,)e(comm\);)242 1402 y(....)194 1458 y(})75 1515 y(#ifdef)h(EXAMPLE_2C)194 1571 y(/*)h(include)f(\(resp,)g(exclude\))g(for)g(safety)g(\(resp,)g (no)h(safety\):)f(*/)194 1628 y(MPI_Barrier\(comm\);)75 1684 y(#endif)147 1741 y(})75 1852 y FD(The)e(ab)q(o)o(v)o(e)f(example) h(is)g(really)g(three)g(examples,)h(dep)q(ending)g(on)f(whether)f(or)g (not)g(one)h(includes)75 1909 y(rank)f(3)h(in)g Fs(list)p 345 1909 14 2 v 17 w(b)p FD(,)h(and)f(whether)g(or)f(not)g(a)g(sync)o (hronize)i(is)f(included)i(in)f Fs(lib)p 1477 1909 V 16 w(call)p FD(.)36 b(This)21 b(example)75 1965 y(illustrates)14 b(that,)e(despite)j(con)o(texts,)d(subsequen)o(t)i(calls)g(to)e Fs(lib)p 1166 1965 V 17 w(call)h FD(with)g(the)g(same)g(con)o(text)f (need)i(not)75 2022 y(b)q(e)h(safe)e(from)g(one)i(another)e(\(collo)q (quially)l(,)k(\\bac)o(k-masking"\).)i(Safet)o(y)13 b(is)h(realized)i (if)e(the)28 b Fs(MPI)p 1772 2022 V 16 w(Sync)75 2078 y FD(is)17 b(added.)25 b(What)16 b(this)h(demonstrates)f(is)h(that)f (libraries)i(ha)o(v)o(e)e(to)g(b)q(e)h(written)g(carefully)l(,)h(ev)o (en)f(with)75 2135 y(con)o(texts.)23 b(When)17 b(rank)f(3)g(is)h (excluded,)i(then)d(the)h(sync)o(hronize)g(is)g(not)g(needed)g(to)f (get)g(safet)o(y)g(from)75 2191 y(bac)o(k)f(masking.)166 2252 y(Algorithms)d(lik)o(e)i(\\com)o(bine")e(ha)o(v)o(e)g(strong)f (enough)i(source)f(selectivit)o(y)h(so)f(that)g(they)g(are)g(inher-)75 2308 y(en)o(tly)k(ok)m(a)o(y)l(.)22 b(So)16 b(are)f(m)o(ultiple)j (calls)f(to)e(a)h(t)o(ypical)g(tree-broadcast)g(algorithm)f(with)i(the) f(same)f(ro)q(ot.)75 2365 y(Ho)o(w)o(ev)o(er,)21 b(m)o(ultiple)i(calls) f(to)f(a)g(t)o(ypical)h(tree-broadcast)e(algorithm)h(|)h(with)f (di\013eren)o(t)g(ro)q(ots)g(|)75 2421 y(could)16 b(bac)o(k-mask,)f (pro)q(ducing)i(undesirable)g(results.)k(Therefore,)15 b(suc)o(h)h(algorithms)f(w)o(ould)h(ha)o(v)o(e)f(to)75 2478 y(utilize)f(the)e(monotonically)h(increasing)g(tags)e(\(with)g (comm)o(unicator)h(scop)q(e\))g(to)f(k)o(eep)h(things)g(straigh)o(t.)75 2534 y(All)19 b(of)e(the)h(foregoing)f(is)h(a)f(supp)q(osition)i(of)f (\\collectiv)o(e)h(calls")f(implemen)o(ted)h(with)f(p)q(oin)o(t-to-p)q (oin)o(t)75 2591 y(op)q(erations.)25 b(MPI)17 b(implemen)o(tations)h (ma)o(y)e(or)g(ma)o(y)h(not)f(implemen)o(t)i(collectiv)o(e)h(calls)f (using)f(p)q(oin)o(t-)75 2647 y(to-p)q(oin)o(t)h(op)q(erations.)28 b(These)19 b(algorithms)f(are)f(used)i(to)e(illustrate)i(the)g(issues)f (of)g(correctness)g(and)75 2704 y(safet)o(y)l(,)c(indep)q(enden)o(t)k (of)c(ho)o(w)h(MPI)g(implemen)o(ts)i(its)e(collectiv)o(e)i(calls.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 157 164 157 163 bop 75 -100 a Fp(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FD(157)75 45 y Fn(5.8.7)49 b(Inter-Comm)n(unication) 14 b(Examples)75 133 y Fs(Example)f(1:)19 b(Three-Group)e(\\Pip)q (eline")409 277 y FB(+---------+)213 b(+---------+)h(+---------+)409 334 y(|)h(|)f(|)h(|)g(|)f(|)409 390 y(|)24 b(Group)f(0)h(|)f(<----->)g (|)h(Group)f(1)h(|)g(<----->)f(|)g(Group)h(2)f(|)409 447 y(|)215 b(|)f(|)h(|)g(|)f(|)409 503 y(+---------+)f(+---------+)h (+---------+)75 671 y FD(Groups)16 b(0)g(and)h(1)f(comm)o(unicate.)24 b(Groups)16 b(1)h(and)f(2)g(comm)o(unicate.)24 b(Therefore,)17 b(group)f(0)g(requires)75 728 y(one)f(in)o(ter-comm)o(unicator,)f (group)g(1)g(requires)i(t)o(w)o(o)d(in)o(ter-comm)o(unicators,)h(and)h (group)f(2)g(requires)h(1)75 784 y(in)o(ter-comm)o(unicator.)147 897 y FB(main\(int)22 b(argc,)i(char)f(**argv\))147 953 y({)194 1010 y(MPI_Comm)g(myComm;)166 b(/*)24 b(intra-communicator)d (of)j(local)f(sub-group)g(*/)194 1066 y(MPI_Comm)g(myFirstComm;)46 b(/*)24 b(inter-communicator)d(*/)194 1123 y(MPI_Comm)i(mySecondComm;)f (/*)i(second)f(inter-communicator)e(\(group)i(1)h(only\))f(*/)194 1179 y(MPI_Intercomm_request)e(inter_request1,)h(inter_request2;)194 1236 y(int)i(membershipKey;)194 1292 y(int)g(rank;)194 1405 y(MPI_Init\(&argc,)e(argv\);)194 1462 y (MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&rank\);)194 1574 y(/*)j(User)f(code)h(must)f(generate)g(membershipKey)f(in)h(the)h (range)f([0,)g(1,)h(2])g(*/)194 1631 y(membershipKey)e(=)i(rank)f(\045) h(3;)194 1744 y(/*)g(Build)f(intra-communicator)e(for)j(local)f (sub-group)g(*/)194 1800 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e (membershipKey,)h(rank,)h(&myComm\);)194 1913 y(/*)h(Build)f (inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 1970 y(if)i(\(membershipKey)e(==)h(0\))194 2026 y({)501 b(/*)24 b(Group)f(0)h(communicates)e(with)h(group)h(1.)f(*/)242 2083 y(MPI_Intercomm_start\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 2139 y(1,)j(&inter_request1)e(\);)242 2195 y(MPI_Intercomm_finish\()f (inter_request1,)h(myFirstComm)g(\);)194 2252 y(})194 2308 y(else)i(if)f(\(membershipKey)f(==)i(1\))194 2365 y({)334 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)g (2.)f(*/)242 2421 y(MPI_Intercomm_start\()e(myComm,)i(0,)h (MPI_COMM_WORLD,)d(0,)743 2478 y(1,)j(&inter_request1)e(\);)242 2534 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 2591 y(12,)j(&inter_request2)d(\);)242 2647 y(MPI_Intercomm_finish\()g (inter_request1,)h(myFirstComm)g(\);)242 2704 y(MPI_Intercomm_finish\() f(inter_request2,)h(mySecondComm)g(\);)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 158 165 158 164 bop 75 -100 a FD(158)123 b Fp(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)194 45 y FB(})194 102 y(else)24 b(if)f(\(membershipKey)f(==)i(2\))194 158 y({)501 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)h(1.)f (*/)242 214 y(MPI_Intercomm_start\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d (1,)743 271 y(12,)j(&inter_request1)d(\);)242 327 y (MPI_Intercomm_finish\()g(inter_request1,)h(myFirstComm)g(\);)194 384 y(})194 497 y(/*)i(Do)f(work)h(...)f(*/)194 610 y (MPI_Finalize\(\);)147 666 y(})75 785 y Fs(Example)13 b(2:)19 b(Three-Group)e(\\Ring")290 871 y FB(+-----------------------)o (--------)o(-------)o(-------)o(--------)o(------+)290 927 y(|)1408 b(|)290 984 y(|)95 b(+---------+)213 b(+---------+)h (+---------+)94 b(|)290 1040 y(|)h(|)215 b(|)f(|)h(|)g(|)f(|)96 b(|)290 1097 y(+-->)23 b(|)h(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h (|)g(<----->)f(|)g(Group)h(2)f(|)h(<--+)409 1153 y(|)215 b(|)f(|)h(|)g(|)f(|)409 1210 y(+---------+)f(+---------+)h(+---------+) 166 1307 y FD(Groups)14 b(0)g(and)h(1)f(comm)o(unicate.)20 b(Groups)14 b(1)g(and)h(2)f(comm)o(unicate.)20 b(Groups)14 b(0)g(and)h(2)f(comm)o(u-)75 1364 y(nicate.)21 b(Therefore,)14 b(eac)o(h)h(requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators.)147 1462 y FB(main\(int)22 b(argc,)i(char)f(**argv\))147 1518 y({)194 1574 y(MPI_Comm)47 b(myComm;)118 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)g(*/)194 1631 y(MPI_Comm)g(myFirstComm;)f(/*)i(inter-communicators)d(*/)194 1687 y(MPI_Comm)i(mySecondComm;)194 1744 y(MPI_Intercomm_request)e (inter_request1,)h(inter_request2;)194 1800 y(int)i(membershipKey;)194 1857 y(int)g(rank;)194 1970 y(MPI_Init\(&argc,)e(argv\);)194 2026 y(MPI_Comm_rank\(MPI_COMMM_WORL)o(D,)f(&rank\);)194 2083 y(...)194 2195 y(/*)j(User)f(code)h(must)f(generate)g (membershipKey)f(in)h(the)h(range)f([0,)g(1,)h(2])g(*/)194 2252 y(membershipKey)e(=)i(rank)f(\045)h(3;)194 2365 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/)194 2421 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e(membershipKey,)h(rank,)h (&myComm\);)194 2534 y(/*)h(Build)f(inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 2591 y(if)i(\(membershipKey)e (==)h(0\))194 2647 y({)310 b(/*)24 b(Group)f(0)h(communicates)e(with)h (groups)g(1)h(and)g(2.)f(*/)242 2704 y(MPI_Intercomm_start\()e(myComm,) i(0,)h(MPI_COMM_WORLD,)d(1,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 159 166 159 165 bop 75 -100 a Fp(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FD(159)743 45 y FB(1,)24 b(&inter_request1)e(\);)242 102 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 158 y(2,)j(&inter_request2)e(\);)194 214 y(})194 271 y(else)i(if)f(\(membershipKey)f(==)i(1\))194 327 y({)215 b(/*)24 b(Group)f(1)g(communicates)g(with)g(groups)g(0)h(and)f(2.)h(*/) 242 384 y(MPI_Intercomm_start\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,) 743 440 y(1,)j(&inter_request1)e(\);)242 497 y(MPI_Intercomm_start\()f (myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 553 y(12,)j(&inter_request2)d (\);)194 610 y(})194 666 y(else)j(if)f(\(membershipKey)f(==)i(2\))194 723 y({)191 b(/*)24 b(Group)f(2)h(communicates)e(with)h(groups)g(0)h (and)f(1.)h(*/)242 779 y(MPI_Intercomm_start\()d(myComm,)i(0,)h (MPI_COMM_WORLD,)d(0,)743 835 y(2,)j(&inter_request1)e(\);)242 892 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 948 y(12,)j(&inter_request2)d(\);)194 1005 y(})194 1118 y(/*)j(Everyone)f(has)g(the)h(same)f("FINISH")g(code...)g(*/)194 1174 y(MPI_Intercomm_finish\(inter_r)o(equest1,)d(&myFirstComm\);)194 1231 y(MPI_Intercomm_finish\(inter_r)o(equest2,)g(&mySecondComm\);)194 1344 y(/*)k(Do)f(some)h(work)f(...)h(*/)194 1456 y(MPI_Finalize\(\);) 147 1513 y(})75 1632 y Fs(Example)13 b(3:)19 b(Three-Group)e(\\Pip)q (eline")e(Using)h(Name)d(Service)409 1718 y FB(+---------+)213 b(+---------+)h(+---------+)409 1774 y(|)h(|)f(|)h(|)g(|)f(|)409 1831 y(|)24 b(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f (|)g(Group)h(2)f(|)409 1887 y(|)215 b(|)f(|)h(|)g(|)f(|)409 1943 y(+---------+)f(+---------+)h(+---------+)166 2041 y FD(Groups)21 b(0)g(and)h(1)f(comm)o(unicate.)39 b(Groups)21 b(1)g(and)h(2)f(comm)o(unicate.)39 b(Therefore,)23 b(group)e(0)75 2098 y(requires)e(one)f(in)o(ter-comm)o(unicator,)h(group)e(1)h (requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators,)h(and)g(group)g(2)75 2154 y(requires)e(1)f(in)o(ter-comm)o(unicator.)147 2252 y FB(main\(int)22 b(argc,)i(char)f(**argv\))147 2308 y({)194 2365 y(MPI_Comm)g(myComm;)142 b(/*)24 b(intra-communicator)d (of)j(local)f(sub-group)g(*/)194 2421 y(MPI_Comm)g(myFirstComm;)f(/*)i (inter-communicator)d(*/)194 2478 y(MPI_Comm)i(mySecondComm;/*)f (second)h(inter-communicator)e(\(group)i(1)h(only\))f(*/)194 2534 y(MPI_Intercomm_request)e(inter_request1,)h(inter_request2;)194 2591 y(int)i(color,)f(rank,)g(size;)194 2704 y(MPI_Init\(&argc,)f (argv\);)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 160 167 160 166 bop 75 -100 a FD(160)123 b Fp(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)194 45 y FB(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)21 b(&rank\);)194 102 y(MPI_Comm_size\(MPI_COMM_WORLD)o(,)g(&size\);)194 158 y(...)194 271 y(/*)j(User)f(code)h(must)f(generate)g(color)g(in)g (the)h(range)f([0,)g(1,)h(2])g(*/)194 327 y(if)g(\()g(rank)f(<)h (\(\(size+1\)/3\))e(\))242 384 y(color)h(=)h(0;)194 440 y(else)g(if)f(\()h(rank)f(<)h(\(2*\(size+1\)/3\))e(\))242 497 y(color)h(=)h(1;)194 553 y(else)242 610 y(color)f(=)h(2;)194 723 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/) 194 779 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e(color,)i(rank,)g (&myComm\);)194 892 y(/*)h(Build)f(inter-communicators.)45 b(Group)23 b(membership)g(conditions)f(must)i(be)f(*/)194 948 y(/*)h(provided)f(by)g(the)h(user.)f(*/)194 1005 y(if)h(\(color)f(==)h(0\))194 1061 y({)167 b(/*)24 b(Group)f(0)h (communicates)e(with)h(group)g(1.)h(*/)242 1118 y (MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(01",)i (&inter_request1\);)242 1174 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)194 1231 y(})194 1287 y(else)i(if)f(\(color)g(==)h(1\))194 1344 y({)167 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)f(2.)h(*/) 242 1400 y(MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(01",)i (&inter_request1\);)242 1456 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(12",)i(&inter_request2\);)242 1513 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)242 1569 y(MPI_Intercomm_finish)f (\(inter_request2,)h(&mySecondComm\);)194 1626 y(})194 1682 y(else)i(if)f(\(color)g(==)h(2\))194 1739 y({)167 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)g(1.)h(*/)242 1795 y(MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(12",)i (&inter_request1\);)242 1852 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)194 1908 y(})194 2021 y(/*)i(Do)f(some)h(work)f(...)h(*/)194 2134 y(MPI_Finalize\(\);) 147 2190 y(})75 2329 y Fs(Example)13 b(4:)19 b(Three-Group)e(\\Ring")e (Using)g(Name)e(Service)290 2421 y FB(+-----------------------)o (--------)o(-------)o(-------)o(--------)o(------+)290 2478 y(|)1408 b(|)290 2534 y(|)95 b(+---------+)213 b(+---------+)h (+---------+)94 b(|)290 2591 y(|)h(|)215 b(|)f(|)h(|)g(|)f(|)96 b(|)290 2647 y(+-->)23 b(|)h(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h (|)g(<----->)f(|)g(Group)h(2)f(|)h(<--+)409 2704 y(|)215 b(|)f(|)h(|)g(|)f(|)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 161 168 161 167 bop 75 -100 a Fp(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FD(161)409 45 y FB(+---------+)213 b(+---------+)h(+---------+)75 160 y FD(Groups)12 b(0)g(and)h(1)f(comm) o(unicate.)19 b(Groups)13 b(1)f(and)g(2)h(comm)o(unicate.)19 b(Groups)12 b(0)g(and)h(2)f(comm)o(unicate.)75 216 y(Therefore,)j(eac)o (h)g(requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators.)147 332 y FB(main\(int)22 b(argc,)i(char)f(**argv\))147 389 y({)194 445 y(MPI_Comm)47 b(myComm;)71 b(/*)23 b(intra-communicator)f (of)h(local)g(sub-group)g(*/)194 502 y(MPI_Comm)47 b(myFirstComm;)380 b(/*)24 b(inter-communicators)d(*/)194 558 y(MPI_Comm)47 b(mySecondComm;)194 615 y(MPI_Intercomm_request)21 b(inter_request1,)h (inter_request2;)194 671 y(int)i(color,)f(rank,)g(size;)194 784 y(MPI_Init\(&argc,)f(argv\);)194 841 y (MPI_Comm_size\(MPI_COMM_WORLD)o(,)f(&size\);)194 897 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)g(&rank\);)194 953 y(...)194 1066 y(/*)j(User)f(code)h(must)f(generate)g(color)g(in)g(the) h(range)f([0,)g(1,)h(2])g(*/)194 1123 y(if)g(\()g(rank)f(==)h(0)f(\)) 242 1179 y(color)g(=)h(0;)194 1236 y(else)g(if)f(\()h(rank)f(==)h(1)g (\))242 1292 y(color)f(=)h(1;)194 1349 y(else)242 1405 y(color)f(=)h(2;)194 1518 y(/*)g(Build)f(intra-communicator)e(for)j (local)f(sub-group)g(*/)194 1574 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e (color,)i(rank,)g(&myComm\);)194 1687 y(/*)h(Build)f (inter-communicators.)45 b(Group)23 b(membership)g(conditions)f(must)i (be)f(*/)194 1744 y(/*)h(provided)f(by)g(the)h(user.)f(*/)194 1800 y(if)h(\(color)f(==)h(0\))194 1857 y({)119 b(/*)24 b(Group)f(0)h(communicates)e(with)i(groups)f(1)g(and)h(2.)f(*/)242 1913 y(MPI_Intercomm_name_start\(m)o(yComm,)e("Connect)h(01",)i (&inter_request1\);)242 1970 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(02",)i(&inter_request2\);)194 2026 y(})194 2083 y(else)g(if)f(\(color)g(==)h(1\))194 2139 y({)119 b(/*)24 b(Group)f(1)h(communicates)e(with)i(groups)f(0)g(and)h(2.)f(*/) 242 2195 y(MPI_Intercomm_name_start\(m)o(yComm,)e("Connect)h(01",)i (&inter_request1\);)242 2252 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(12",)i(&inter_request2\);)194 2308 y(})194 2365 y(else)g(if)f(\(\))194 2421 y({)96 b(/*)23 b(Group)g(2)h(communicates)e(with)i(groups)f(0)g (and)h(1.)f(*/)242 2478 y(MPI_Intercomm_name_start\(m)o(yComm,)e ("Connect)h(02",)i(&inter_request1\);)242 2534 y (MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(12",)i (&inter_request2\);)194 2591 y(})194 2704 y(/*)g(Everyone)f(has)g(the)h (same)f("FINISH")g(code...)g(*/)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 162 169 162 168 bop 75 -100 a FD(162)118 b Fp(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)194 45 y FB(MPI_Intercomm_finish)21 b(\(inter_request1,)h(&myFirstComm\);) 194 102 y(MPI_Intercomm_finish)f(\(inter_request2,)h(&mySecondComm\);) 194 214 y(/*)i(Do)f(some)h(work)f(...)h(*/)194 327 y(MPI_Finalize\(\);) 147 384 y(})1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 163 170 163 169 bop 75 358 a Fr(Chapter)34 b(6)75 568 y Fy(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)75 811 y Fq(6.1)59 b(Intro)r(duction)75 916 y FD(A)13 b(pro)q(cess)h(group)f(in)h(MPI)g(is)g(a)f(collection)i (of)e Fw(n)g FD(pro)q(cesses.)20 b(Eac)o(h)13 b(pro)q(cess)h(in)g(the)f (group)g(is)h(assigned)75 972 y(a)j(rank)g(b)q(et)o(w)o(een)g(0)g(and)g Fw(n)12 b FA(\000)f FD(1.)25 b(In)18 b(man)o(y)f(parallel)h (applications)h(a)e(linear)h(pro)q(cess)f(ordering)h(do)q(es)75 1029 y(not)c(re\015ect)h(the)g(comm)o(unication)g(pattern)f(of)g(the)h (pro)q(cesses,)f(whic)o(h)i(is)f(usually)h(determined)f(b)o(y)g(the)75 1085 y(underlying)20 b(problem)e(geometry)f(and)h(the)f(n)o(umerical)i (algorithm)f(used.)28 b(Often)18 b(the)f(pro)q(cesses)h(are)75 1142 y(arranged)13 b(in)g(top)q(ological)h(patterns)f(as)g(simple)h(as) f(a)g(t)o(w)o(o{)e(or)i(three{dimensional)i(grid.)k(In)14 b(the)f(more)75 1198 y(general)19 b(case)f(the)h(logical)g(pro)q(cess)g (arrangemen)o(t)e(is)i(describ)q(ed)h(b)o(y)e(a)g(graph.)29 b(In)19 b(this)g(c)o(hapter)f(w)o(e)75 1255 y(will)f(refer)e(to)f(this) i(logical)h(pro)q(cess)e(arrangemen)o(t)f(as)h(the)g(\\virtual)h(top)q (ology)l(.")166 1313 y(A)f(clear)h(distinction)h(m)o(ust)d(b)q(e)i (made)f(b)q(et)o(w)o(een)h(the)f(virtual)h(pro)q(cess)f(top)q(ology)g (and)g(the)h(top)q(ol-)75 1369 y(ogy)21 b(of)g(the)g(ph)o(ysical)i (hardw)o(are)e(pro)q(cessors.)38 b(The)21 b(application-sp)q(eci)q (\014c,)26 b(virtual)c(top)q(ology)f(can)75 1426 y(b)q(e)f(exploited)h (b)o(y)f(the)f(system)g(in)i(the)e(assignmen)o(t)h(of)f(pro)q(cesses)h (to)f(ph)o(ysical)i(pro)q(cessors,)f(if)g(this)75 1482 y(helps)14 b(to)f(impro)o(v)o(e)g(the)g(comm)o(unication)h(p)q (erformance)f(on)g(a)g(giv)o(en)h(mac)o(hine.)20 b(Ho)o(w)12 b(this)i(mapping)f(is)75 1539 y(done)i(in)h(detail,)f(ho)o(w)o(ev)o (er,)f(is)h(outside)g(the)g(scop)q(e)g(of)g(MPI.)f(The)h(description)h (of)f(the)f(virtual)i(pro)q(cess)75 1595 y(top)q(ology)l(,)g(on)h(the)f (other)g(hand,)h(only)g(dep)q(ends)h(on)e(the)h(application,)h(and)e (is,)h(therefore,)f(mac)o(hine{)75 1652 y(indep)q(enden)o(t.)22 b(The)13 b(functions)h(whic)o(h)g(are)f(prop)q(osed)h(in)g(this)g(pap)q (er)g(only)f(deal)h(with)g(this)g(side)g(of)f(the)75 1708 y(mapping)j(problem.)166 1767 y(There)e(are)g(w)o(ell{kno)o(wn)h (tec)o(hniques)g(for)f(mapping)h(grid/torus)e(structures)h(to)g(hardw)o (are)f(top)q(o-)75 1823 y(logies)j(suc)o(h)g(as)g(h)o(yp)q(ercub)q(es)h (or)e(grids.)22 b(F)l(or)15 b(more)g(complicated)i(graph)f(structures)f (go)q(o)q(d)g(heuristics)75 1879 y(often)j(yield)j(near{optimal)e (results)g([17)o(].)30 b(On)19 b(the)g(other)g(hand,)g(if)g(there)g(is) g(no)g(w)o(a)o(y)f(for)g(the)h(user)75 1936 y(to)f(sp)q(ecify)i(the)f (logical)h(pro)q(cess)f(arrangemen)o(t)f(as)g(a)g(\\virtual)h(top)q (ology)l(,")g(a)g(random)f(mapping)h(is)75 1992 y(most)f(lik)o(ely)i (the)e(consequence.)31 b(On)19 b(some)g(mac)o(hines)g(this)g(will)h (lead)f(to)f(unnecessary)h(con)o(ten)o(tion)75 2049 y(in)d(the)g(in)o (terconnect)h(net)o(w)o(ork.)j(Therefore,)15 b(though)h(the)f(ph)o (ysical)i(mapping)g(is)f(not)f(discussed,)i(the)75 2105 y(mere)g(existence)i(of)d(the)i(virtual)f(top)q(ology)g(information)g (ma)o(y)g(b)q(e)h(used)f(as)g(advice)h(b)o(y)f(the)h(run)o(time)75 2162 y(system.)41 b(Some)23 b(details)g(ab)q(out)f(predicted)i(and)f (measured)f(p)q(erformance)h(impro)o(v)o(emen)o(ts)f(whic)o(h)75 2218 y(result)14 b(from)e(go)q(o)q(d)h(pro)q(cess{to{pro)q(cessor)f (mapping)i(on)f(mo)q(dern)h(w)o(ormhole{routing)f(arc)o(hitectures)75 2275 y(can)i(b)q(e)h(found)g(in)g([9)o(,)f(8].)75 2429 y Fq(6.2)59 b(Virtual)20 b(T)-5 b(op)r(ologies)75 2534 y FD(In)13 b(the)g(most)f(general)h(case)g(the)g(comm)o(unication)g (pattern)g(of)f(a)g(set)h(of)f(pro)q(cesses)h(can)g(b)q(e)g(represen)o (ted)75 2591 y(b)o(y)g(a)g(graph.)19 b(The)14 b(no)q(des)f(stand)h(for) e(the)i(pro)q(cesses,)f(and)h(the)f(edges)h(connect)f(pro)q(cesses)h (whic)o(h)g(com-)75 2647 y(m)o(unicate)19 b(with)h(eac)o(h)f(other.)30 b(MPI)19 b(pro)o(vides)g(message{passing)g(b)q(et)o(w)o(een)g(an)o(y)g (pair)g(of)g(pro)q(cesses)75 2704 y(in)f(a)f(group.)25 b(There)17 b(is)h(no)f(requiremen)o(t)h(for)e(op)q(ening)i(a)f(c)o (hannel)h(explicitly)l(.)29 b(Therefore,)17 b(a)g(\\miss-)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 164 171 164 170 bop 75 -100 a FD(164)882 b Fp(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y FD(ing)k(link")g(in)g(the)f (user{de\014ned)h(pro)q(cess)g(graph)e(do)q(es)i(not)e(prev)o(en)o(t)h (the)h(corresp)q(onding)g(pro)q(cesses)75 102 y(from)e(exc)o(hanging)h (messages.)27 b(It)17 b(only)h(means)g(that)f(this)h(connection)g(is)g (neglected)h(in)g(the)e(virtual)75 158 y(top)q(ology)l(.)23 b(This)17 b(implies)i(that)c(the)i(top)q(ology)f(giv)o(es)g(no)h(con)o (v)o(enien)o(t)f(w)o(a)o(y)g(of)g(naming)h(this)f(path)o(w)o(a)o(y)75 214 y(of)d(comm)o(unication.)20 b(Another)13 b(p)q(ossible)i (consequence)g(is)f(that)f(an)g(automatic)g(mapping)h(to)q(ol)f(\(if)h (one)75 271 y(exists)h(for)e(the)i(run)o(time)f(en)o(vironmen)o(t\))g (will)i(not)e(try)g(to)f(assign)i(the)f(pro)q(cesses)h(to)e(neigh)o(b)q (oring)j(pro-)75 327 y(cessors.)i(Edges)12 b(in)h(the)e(comm)o (unication)i(graph)e(are)h(not)f(w)o(eigh)o(ted.)19 b(Pro)q(cesses)11 b(are)h(either)g(neigh)o(b)q(ors)75 384 y(or)i(not.)20 b(Exp)q(erience)d(with)e(similar)h(tec)o(hniques)g(in)g(P)l(ARMA)o(CS)g ([5)o(,)e(7])h(sho)o(w)f(that)g(this)i(information)75 440 y(is)i(usually)h(su\016cien)o(t)g(for)e(a)g(go)q(o)q(d)h(mapping.) 28 b(Additionally)l(,)20 b(a)e(more)f(precise)i(sp)q(eci\014cation)h(w) o(ould)75 497 y(b)q(e)g(more)f(di\016cult)i(for)d(the)i(user)f(to)g (set)g(up,)h(and)f(it)h(w)o(ould)g(mak)o(e)e(the)i(in)o(terface)f (functions)h(more)75 553 y(complicated.)166 610 y(In)c(principle,)h(a)e (function)h(for)f(the)g(de\014nition)i(of)e(suc)o(h)h(a)f(graph)g (structure)g(w)o(ould)g(b)q(e)h(su\016cien)o(t)75 667 y(for)d(all)i(applications.)21 b(Ho)o(w)o(ev)o(er,)13 b(in)h(man)o(y)g(applications)h(the)f(graph)g(structure)f(is)h(v)o(ery) g(regular,)g(and)75 723 y(the)e(detailed)i(set{up)f(of)e(the)i(graph)f (w)o(ould)h(b)q(e)g(incon)o(v)o(enien)o(t)g(for)f(the)g(user)h(and)f (resource{consuming)75 780 y(during)g(run)g(time.)19 b(A)12 b(large)f(fraction)g(of)g(all)i(parallel)g(applications)g(use)f (pro)q(cess)g(top)q(ologies)f(lik)o(e)i(rings,)75 836 y(t)o(w)o(o{)i(or)i(higher{dimensional)i(grids,)e(or)g(tori.)24 b(These)18 b(structures)e(are)h(completely)h(de\014ned)g(b)o(y)f(the)75 892 y(n)o(um)o(b)q(er)d(of)g(dimensions)i(and)e(the)g(n)o(um)o(b)q(ers) h(of)f(pro)q(cesses)g(in)h(eac)o(h)f(co)q(ordinate)h(direction.)21 b(Also,)14 b(the)75 949 y(mapping)h(of)e(grids)h(and)h(tori)e(is)i (generally)g(an)f(easier)g(problem)h(then)f(that)f(of)h(general)g (graphs.)19 b(Th)o(us,)75 1005 y(it)c(is)h(desirable)h(to)d(separate)h (out)g(this)h(case)f(explicitly)l(.)166 1062 y(Pro)q(cess)i(co)q (ordinates)g(in)h(a)e(Cartesian)h(structure)g(b)q(egin)h(their)f(n)o (um)o(b)q(ering)h(at)e(0.)25 b(Ro)o(w)16 b(ma)s(jor)75 1119 y(n)o(um)o(b)q(ering)e(is)g(used)f(for)g(the)g(pro)q(cesses)g(in)h (a)f(Cartesian)g(structure.)19 b(This)14 b(means)f(that,)f(for)g (example,)75 1175 y(the)17 b(relation)h(b)q(et)o(w)o(een)g(group)e (rank)h(and)h(co)q(ordinates)f(for)g(four)g(pro)q(cesses)g(in)h(a)f (\(2)11 b FA(\002)h FD(2\))k(grid)i(is)f(as)75 1232 y(follo)o(ws.)206 1314 y(co)q(ord)e(\(0,0\):)48 b(rank)15 b(0)206 1371 y(co)q(ord)g(\(0,1\):)48 b(rank)15 b(1)206 1427 y(co)q(ord)g(\(1,0\):) 48 b(rank)15 b(2)206 1484 y(co)q(ord)g(\(1,1\):)48 b(rank)15 b(3)75 1618 y Fq(6.3)59 b(Emb)r(edding)18 b(in)i(MPI)75 1720 y FD(The)13 b(supp)q(ort)f(for)g(virtual)h(top)q(ologies)g(as)f (de\014ned)h(in)h(this)e(c)o(hapter)h(is)f(consisten)o(t)h(with)g (other)f(parts)f(of)75 1777 y(MPI,)h(and,)h(whenev)o(er)g(p)q(ossible,) h(mak)o(es)e(use)h(of)f(functions)h(whic)o(h)h(are)e(de\014ned)i (elsewhere.)20 b(T)l(op)q(ology)75 1833 y(information)h(is)h(asso)q (ciated)g(with)g(comm)o(unicators.)38 b(It)21 b(can)g(b)q(e)h(main)o (tained)h(using)f(the)f(cac)o(hing)75 1890 y(mec)o(hanism)d(describ)q (ed)g(in)g(Chapter)f(5.)25 b(The)17 b(top)q(ology)f(functions)i (complemen)o(t)g(and)f(in)o(terop)q(erate)75 1946 y(with)f(the)f (collectiv)o(e)i(comm)o(unication)f(functions)g(describ)q(ed)h(in)f (Chapter)f(4.)166 2003 y(The)d(curren)o(t)g(design)g(of)g(the)f(pro)q (cess)h(top)q(ology)g(functions)g(assumes)g(that)f(MPI)h(is)g(free)g (to)f(c)o(ho)q(ose)75 2059 y(an)o(y)g(pro)q(cess)h(in)h(a)e(group)g (for)g(an)o(y)h(lo)q(cation)g(in)h(the)e(top)q(ology)l(.)19 b(There)12 b(are)f(no)h(sp)q(ecial)h(pro)q(cesses)f(whic)o(h)75 2116 y(ha)o(v)o(e)g(to)f(b)q(e)i(aligned)g(with)g(top)q(ology)e (substructures.)19 b(This)13 b(assumption)f(\014ts)g(b)q(est)g(to)g(a)g (\\lo)q(cal)g(SPMD)75 2172 y(mo)q(del")k(in)g(whic)o(h)g(all)g(pro)q (cesses)g(in)g(the)f(same)g(group)g(execute)h(the)f(same)g(program.)75 2318 y Fq(6.4)59 b(Overview)19 b(of)h(the)f(p)n(rop)r(osed)g(MPI)h (functions)75 2421 y FD(The)13 b(functions)h Fs(MPI)p 445 2421 14 2 v 16 w(MAKE)p 590 2421 V 16 w(GRAPH)g FD(and)f Fs(MPI)p 938 2421 V 16 w(MAKE)p 1083 2421 V 16 w(CART)g FD(are)g(used)g(to)f(create)h(general)g(virtual)75 2477 y(top)q(ologies)21 b(and)f(Cartesian)g(top)q(ologies,)h(resp)q(ectiv)o (ely)l(.)37 b(These)20 b(top)q(ology)g(creation)g(functions)h(are)75 2534 y(collectiv)o(e.)f(As)11 b(for)g(other)g(collectiv)o(e)i(calls,)f (the)f(co)q(de)h(should)g(b)q(e)g(written)f(to)g(w)o(ork)f(correctly)l (,)i(whether)75 2590 y(the)j(call)i(sync)o(hronizes)f(or)f(not.)166 2647 y(The)g(top)q(ology)g(de\014nition)i(function)g Fs(MPI)p 912 2647 V 15 w(MAKE)p 1056 2647 V 17 w(CART)e FD(can)h(b)q(e)g(used)g(to)e(describ)q(e)j(Cartesian)75 2704 y(structures)k(of)g(arbitrary)f(dimension.)40 b(F)l(or)21 b(eac)o(h)g(co)q(ordinate)h(direction)g(it)g(is)f(sp)q(eci\014ed)j (whether)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 165 172 165 171 bop 75 -100 a Fp(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FD(165)75 45 y(the)16 b(pro)q(cess)g(structure)g(is)g(p)q(erio)q(dic) i(or)d(not.)22 b(Similar)17 b(functions)g(are)e(con)o(tained)i(in)g (EXPRESS)f([19)o(])75 102 y(and)g(P)l(ARMA)o(CS.)h(Note)f(that)f(an)h Fw(n)p FD({dimensional)j(h)o(yp)q(ercub)q(e)f(is)e(an)h Fw(n)p FD({dimensional)h(torus)e(with)g(2)75 158 y(pro)q(cesses)i(p)q (er)h(co)q(ordinate)f(direction.)29 b(Th)o(us,)18 b(sp)q(ecial)i(supp)q (ort)e(for)f(h)o(yp)q(ercub)q(e)i(structures)f(is)g(not)75 214 y(necessary)l(.)h(The)12 b(lo)q(cal)h(auxiliary)g(function)g Fs(MPI)p 929 214 14 2 v 16 w(MAKE)p 1074 214 V 16 w(DIMS)e FD(can)h(b)q(e)h(used)f(to)f(compute)h(an)g(\\ev)o(en")75 271 y(distribution)17 b(of)e(pro)q(cesses)g(among)g(a)g(giv)o(en)g(n)o (um)o(b)q(er)h(of)f(dimensions.)166 329 y(The)j(function)g Fs(MPI)p 527 329 V 16 w(TOPO)p 669 329 V 18 w(ST)l(A)l(TUS)h FD(can)e(b)q(e)i(used)f(to)f(inquire)j(on)d(the)h(top)q(ology)g(asso)q (ciated)75 385 y(with)e(a)f(comm)o(unicator.)20 b(The)c(top)q(ological) g(information)f(can)h(b)q(e)g(extracted)f(from)g(the)g(comm)o(unica-)75 442 y(tor)h(using)h(the)g(functions)h Fs(MPI)p 633 442 V 15 w(GET)p 736 442 V 17 w(GRAPHDIMS)f FD(and)g Fs(MPI)p 1203 442 V 16 w(GET)p 1307 442 V 17 w(GRAPH)p FD(,)g(for)f(general)i (graphs,)75 498 y(and)c Fs(MPI)p 247 498 V 16 w(GET)p 351 498 V 16 w(CARTDIM)g FD(and)g Fs(MPI)p 755 498 V 16 w(GET)p 859 498 V 16 w(CART)p FD(,)g(for)f(Cartesian)h(top)q (ologies.)19 b(Sev)o(eral)c(additional)75 555 y(functions)f(are)e(pro)o (vided)i(to)e(manipulate)i(Cartesian)f(top)q(ologies:)19 b(the)13 b(functions)h Fs(MPI)p 1603 555 V 16 w(CART)p 1738 555 V 17 w(RANK)75 611 y FD(and)f Fs(MPI)p 246 611 V 15 w(CART)p 380 611 V 17 w(COORDS)h FD(translate)e(Cartesian)g(co)q (ordinates)h(in)o(to)f(a)g(group)g(rank,)g(and)h(vice-v)o(ersa;)75 668 y(the)g(function)h Fs(MPI)p 412 668 V 16 w(CART)p 547 668 V 17 w(SUB)g FD(can)f(b)q(e)h(used)f(to)g(extract)f(a)h (Cartesian)g(subspace;)h(and)f(the)g(function)75 724 y Fs(MPI)p 160 724 V 16 w(CART)p 295 724 V 17 w(SHIFT)g FD(pro)o(vides)h(the)f(information)h(needed)g(to)f(execute)h(a)f(shift) g(in)i(a)e(Cartesian)g(dimen-)75 781 y(sion.)20 b(The)c(t)o(w)o(o)e (functions)i Fs(MPI)p 647 781 V 16 w(GRAPH)p 813 781 V 17 w(NEIGHBORS)p 1081 781 V 18 w(COUNT)g FD(and)g Fs(MPI)p 1442 781 V 16 w(GRAPH)p 1608 781 V 17 w(NEIGHBORS)75 837 y FD(can)e(b)q(e)h(used)f(to)g(extract)f(the)h(neigh)o(b)q(ors)h (of)f(a)f(no)q(de)i(in)g(a)f(graph.)19 b(The)14 b(function)h Fs(MPI)p 1595 837 V 16 w(CART)p 1730 837 V 17 w(SUB)f FD(is)75 893 y(collectiv)o(e;)j(all)f(other)f(functions)h(are)e(lo)q (cal.)166 951 y(Tw)o(o)d(additional)i(functions,)g Fs(MPI)p 767 951 V 16 w(MAP)p 882 951 V 17 w(GRAPH)g FD(and)f Fs(MPI)p 1229 951 V 16 w(MAP)p 1344 951 V 16 w(CART)h FD(are)f(presen)o(ted)g(in)h(the)75 1008 y(last)19 b(section.)31 b(In)19 b(general)g(these)g(functions)h(will)g(not)e(b)q(e)i(called)g (b)o(y)f(the)f(user)h(directly)l(.)32 b(Ho)o(w)o(ev)o(er,)75 1064 y(together)16 b(with)i(the)f(comm)o(unicator)g(manipulation)h (functions)g(presen)o(ted)g(in)g(Chapter)e(5,)h(they)g(are)75 1121 y(su\016cien)o(t)23 b(to)e(implemen)o(t)j(all)f(other)e(top)q (ology)h(functions.)41 b(The)23 b(last)f(section)g(outlines)i(suc)o(h)e (an)75 1177 y(implemen)o(tation.)75 1329 y Fq(6.5)59 b(T)-5 b(op)r(ology)19 b(Constructo)n(rs)75 1481 y Fs(MPI)p 160 1481 V 16 w(MAKE)p 305 1481 V 16 w(CART\(comm)p 576 1481 V 11 w(old,)c(ndims,)e(dims,)f(p)q(erio)q(ds,)k(reo)o(rder,)e (comm)p 1371 1481 V 11 w(ca)o(rt\))117 1559 y Ft(IN)171 b Fs(comm)p 454 1559 V 10 w(old)383 b Ft(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g(\(handle\))117 1638 y(IN)171 b Fs(ndims)458 b Ft(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i (Cartesian)g(grid)f(\(in)o(teger\))117 1716 y(IN)171 b Fs(dims)481 b Ft(in)o(teger)19 b(arra)o(y)f(of)f(size)i Fv(ndims)d Ft(sp)q(ecifying)i(the)g(n)o(um)o(b)q(er)g(of)905 1772 y(pro)q(cesses)f(in)c(eac)o(h)i(dimension)117 1850 y(IN)171 b Fs(p)q(erio)q(ds)436 b Ft(logical)12 b(arra)o(y)h(of)g(size) h Fv(ndims)d Ft(sp)q(ecifying)i(whether)h(the)g(grid)905 1907 y(is)d(p)q(erio)q(dic)g(\()p Fv(true)p Ft(\))h(or)f(ap)q(erio)q (dic)g(\()p Fv(false)p Ft(\))g(in)f(eac)o(h)i(dimension)117 1985 y(IN)171 b Fs(reo)o(rder)438 b Ft(ranking)10 b(ma)o(y)g(b)q(e)h (reordered)i(\()p Fv(true)p Ft(\))f(or)f(not)g(\()p Fv(false)p Ft(\))g(\(logical\))117 2063 y(OUT)124 b Fs(comm)p 454 2063 V 10 w(ca)o(rt)367 b Ft(comm)o(unicator)11 b(for)j(new)g (Cartesian)g(group)g(\(handle\))75 2189 y FB(int)23 b(MPI)p 245 2189 15 2 v 17 w(Make)p 358 2189 V 17 w(cart\(MPI)p 567 2189 V 16 w(comm)g(comm)p 798 2189 V 17 w(old,)g(int)h(ndims,)f (int)g(*dims,)g(int)h(*periods,)393 2245 y(int)g(reorder,)e(MPI)p 778 2245 V 17 w(comm)h(*comm)p 1034 2245 V 17 w(cart\))75 2333 y(MPI)p 150 2333 V 17 w(MAKE)p 263 2333 V 16 w(CART\(COMM)p 495 2333 V 16 w(OLD,)h(NDIMS,)f(DIMS,)g(PERIODS,)g(REORDER,)f(COMM)p 1466 2333 V 17 w(CART,)h(IERROR\))170 2390 y(INTEGER)g(COMM)p 460 2390 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(PERIODS\(*\),)f(REORDER,)h (COMM)p 1575 2390 V 17 w(CART,)170 2446 y(IERROR)166 2534 y FD(The)13 b(function)h(returns)f(a)g(handle)h(to)e(a)h(new)g (comm)o(unicator)g(to)f(whic)o(h)i(the)g(Cartesian)e(top)q(ology)75 2591 y(information)i(is)g(attac)o(hed.)19 b(If)14 b Fs(reo)o(rder)f(=)h (false)h FD(then)f(the)g(rank)f(of)h(eac)o(h)g(pro)q(cess)g(in)h(the)f (new)g(group)f(is)75 2647 y(iden)o(tical)19 b(to)d(its)h(rank)g(in)g (the)g(old)h(group.)25 b(Otherwise,)17 b(the)g(function)h(ma)o(y)e (reorder)h(the)g(pro)q(cesses,)75 2704 y(so)i(as)f(to)g(c)o(ho)q(ose)h (a)g(go)q(o)q(d)g(em)o(b)q(edding)i(of)d(the)h(virtual)h(top)q(ology)e (on)o(to)g(the)h(ph)o(ysical)i(mac)o(hine.)32 b(If)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 166 173 166 172 bop 75 -100 a FD(166)882 b Fp(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y FD(the)j(total)f(size)h(of)g(the)f (Cartesian)h(grid)g(is)g(smaller)h(than)e(the)h(size)h(of)e(the)h (group)f(of)g Fs(comm)m FD(,)e(then)j(a)75 102 y(pro)q(cess)e(whic)o(h) h(is)g(not)e(a)h(mem)o(b)q(er)g(of)f(the)i(grid)f(is)g(returned)h(a)e (n)o(ull)j(handle.)23 b(The)17 b(call)g(is)f(erroneous)75 158 y(if)g(it)f(sp)q(eci\014es)i(a)e(grid)h(that)e(is)i(larger)f(than)g (the)g(group)g(size.)166 327 y Fl(Discussion:)40 b Ft(The)16 b(option)e(of)h(not)g(reordering)h(the)g(pro)q(cess)h(ranks)f(can)g(b)q (e)g(useful)f(in)g(the)h(case)g(of)f(an)75 383 y(o)o(v)o(erla)o(y)f(of) g(di\013eren)o(t)i(top)q(ologies.)21 b(The)15 b(user)h(can)f(then)h (fully)e(con)o(trol)g(whic)o(h)h(lo)q(cation)f(in)h(the)g(\014rst)h (top)q(ology)75 440 y(coincides)e(with)f(whic)o(h)h(lo)q(cation)e(in)h (the)h(second)h(one.)j(The)c(price)g(whic)o(h)g(has)f(to)h(b)q(e)g (paid)e(in)i(this)f(case)i(is)e(that)75 496 y(the)h(pro)q(cess)i (arrangemen)o(t)d(can)i(only)e(b)q(e)h(optimized)f(for)g(one)h(top)q (ology)m(.)166 665 y FD(In)22 b(a)g(p)q(ortable)g(program)f(the)h(n)o (um)o(b)q(er)g(of)f(a)o(v)m(ailable)j(pro)q(cessors)d(is)h(usually)i (not)d(kno)o(wn)h(at)75 721 y(compile)g(time.)36 b(In)21 b(the)f(case)h(of)f(a)g(Cartesian)g(top)q(ology)g(the)h(function)g Fs(MPI)p 1485 721 14 2 v 16 w(MAKE)p 1630 721 V 16 w(DIMS)f FD(helps)75 778 y(in)e(setting)e(the)h(n)o(um)o(b)q(ers)g(of)f(pro)q (cesses)i(p)q(er)f(co)q(ordinate)g(direction,)h(dep)q(ending)h(on)e (the)f(n)o(um)o(b)q(er)h(of)75 834 y(pro)q(cesses)f(in)g(the)f(group)g (and)g(optional)h(constrain)o(ts)f(whic)o(h)h(can)f(b)q(e)h(sp)q (eci\014ed)h(b)o(y)e(the)h(user:)75 968 y Fs(MPI)p 160 968 V 16 w(MAKE)p 305 968 V 16 w(DIMS\(nno)q(des,)g(ndims,)d(dims\))117 1075 y Ft(IN)171 b Fs(nno)q(des)441 b Ft(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(a)f(grid)h(\(in)o(teger\))117 1209 y(IN)171 b Fs(ndims)458 b Ft(n)o(um)o(b)q(er)13 b(of)h(Cartesian)g(dimensions)e(\(in)o(teger\))117 1343 y(INOUT)78 b Fs(dims)481 b Ft(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fb(ndims)e Ft(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 1400 y(no)q(des)e(in)e(eac)o(h)i(dimension)75 1554 y FB(int)23 b(MPI)p 245 1554 15 2 v 17 w(Make)p 358 1554 V 17 w(dims\(int)g(nnodes,)g(int)g(ndims,)g(int)g(*dims\))75 1670 y(MPI)p 150 1670 V 17 w(MAKE)p 263 1670 V 16 w(DIMS\(NNODES,)g (NDIMS,)g(DIMS,)g(IERROR\))170 1726 y(INTEGER)g(NNODES,)g(NDIMS,)g (DIMS\(*\),)g(IERROR)166 1843 y FD(The)12 b(en)o(tries)h(in)g(the)f (arra)o(y)f Fs(dims)f FD(are)i(set)g(to)f(describ)q(e)j(a)e(Cartesian)g (grid)g(with)h Fs(ndims)d FD(dimensions)75 1899 y(and)j(a)g(total)g(of) f Fs(nno)q(des)k FD(no)q(des.)j(The)14 b(dimensions)h(are)d(set)h(to)g (b)q(e)h(as)e(close)i(to)f(eac)o(h)g(other)g(as)g(p)q(ossible.)75 1955 y(The)20 b(caller)h(ma)o(y)e(further)h(constrain)g(the)g(op)q (eration)g(of)g(this)g(routine)g(b)o(y)g(sp)q(ecifying)i(elemen)o(ts)e (of)75 2012 y(arra)o(y)13 b FB(dims)p FD(.)19 b(If)14 b FB(dims[i])g FD(is)g(set)g(to)g(a)g(p)q(ositiv)o(e)h(n)o(um)o(b)q (er,)g(the)f(routine)h(will)h(not)d(mo)q(dify)i(the)g(n)o(um)o(b)q(er) 75 2068 y(of)20 b(no)q(des)h(in)g(the)f(corresp)q(onding)h(dimension;)k (only)20 b(those)g(en)o(tries)h(where)g FB(dims[i])i(=)g(0)d FD(will)i(b)q(e)75 2125 y(mo)q(di\014ed)17 b(b)o(y)e(the)g(call.)166 2211 y(Note)g(that)f(an)h(error)g(will)i(o)q(ccur)e(if)h FB(nnodes)e FD(is)i(not)f(a)g(m)o(ultiple)i(of)1413 2171 y Fe(Y)1352 2265 y Ff(i;dims)p Fi([)p Ff(i)p Fi(])p Fg(6)p Fi(=0)1535 2211 y Fw(dims)p FD([)p Fw(i)p FD(].)166 2342 y(Negativ)o(e)f(input)i(v)m(alues)f(of)f FB(dims[i])g FD(will)i(also)e(cause)h(an)f(error.)23 b(All)18 b(elemen)o(ts)f(of)f FB(dims)g FD(m)o(ust)75 2399 y(b)q(e)g(set)f(to)f(0)h(for)g(completely) h(automatic)f(pro)q(cess)g(distribution.)166 2485 y(F)l(or)h FB(dims[i])h FD(set)f(b)o(y)h(the)h(call,)g FB(dims[i])e FD(will)j(b)q(e)f(ordered)f(in)h(non{increasing)g(order.)26 b(Arra)o(y)75 2541 y FB(dims)10 b FD(is)g(suitable)i(for)d(use)i(as)e (input)j(to)d(routine)i Fs(MPI)p 978 2541 14 2 v 15 w(MAKE)p 1122 2541 V 17 w(CART)p FD(.)f(The)g(function)h Fs(MPI)p 1624 2541 V 16 w(MAKE)p 1769 2541 V 16 w(DIMS)75 2598 y FD(is)16 b(lo)q(cal.)166 2704 y(Examples:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 167 174 167 173 bop 75 -100 a Fp(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FD(167)p 166 5 1206 2 v 165 62 2 57 v 191 45 a FB(dims)p 420 62 V 158 w FD(function)16 b(call)p 1047 62 V 383 w FB(dims)p 1370 62 V 165 118 V 191 101 a FD(b)q(efore)f(call)p 420 118 V 1047 118 V 679 w(on)g(return)p 1370 118 V 166 120 1206 2 v 165 176 2 57 v 191 159 a(\(0,0\))p 420 176 V 159 w Fs(MPI)p 530 159 14 2 v 16 w(MAKE)p 675 159 V 16 w(DIMS\(6,)f(2,)g(dims\))p 1047 176 2 57 v 48 w FD(\(3,2\))p 1370 176 V 165 233 V 191 216 a(\(0,0\))p 420 233 V 159 w Fs(MPI)p 530 216 14 2 v 16 w(MAKE)p 675 216 V 16 w(DIMS\(7,)g(2,)g (dims\))p 1047 233 2 57 v 48 w FD(\(7,1\))p 1370 233 V 165 289 V 191 272 a(\(0,3,0\))p 420 289 V 123 w Fs(MPI)p 530 272 14 2 v 16 w(MAKE)p 675 272 V 16 w(DIMS\(6,)g(3,)g(dims\))p 1047 289 2 57 v 48 w FD(\(2,3,1\))p 1370 289 V 165 346 V 191 329 a(\(0,3,0\))p 420 346 V 123 w Fs(MPI)p 530 329 14 2 v 16 w(MAKE)p 675 329 V 16 w(DIMS\(7,)g(3,)g(dims\))p 1047 346 2 57 v 48 w FD(erroneous)h(call)p 1370 346 V 166 347 1206 2 v 166 462 a(F)l(or)f(a)h(general)h(graph)f(structure,)g (the)g(constructor)f(is)75 566 y Fs(MPI)p 160 566 14 2 v 16 w(MAKE)p 305 566 V 16 w(GRAPH\(comm)p 608 566 V 11 w(old,)h(nno)q(des,)i(index,)f(edges,)g(reo)o(rder,)d(comm)p 1402 566 V 11 w(graph\))117 644 y Ft(IN)171 b Fs(comm)p 454 644 V 10 w(old)383 b Ft(comm)o(unicator)11 b(for)j(paren)o(t)g (group)g(\(handle\))117 721 y(IN)171 b Fs(nno)q(des)441 b Ft(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(graph)g(\(in)o(teger\)) 117 797 y(IN)171 b Fs(index)473 b Ft(arra)o(y)14 b(of)f(in)o(tegers)i (describing)f(no)q(de)h(degrees)g(\(see)g(b)q(elo)o(w\))117 874 y(IN)171 b Fs(edges)468 b Ft(arra)o(y)14 b(of)f(in)o(tegers)i (describing)f(graph)g(edges)h(\(see)g(b)q(elo)o(w\))117 950 y(IN)171 b Fs(reo)o(rder)438 b Ft(ranking)10 b(ma)o(y)g(b)q(e)h (reordered)i(\()p Fv(true)p Ft(\))f(or)f(not)g(\()p Fv(false)p Ft(\))g(\(logical\))117 1027 y(OUT)124 b Fs(comm)p 454 1027 V 10 w(graph)333 b Ft(comm)o(unicator)11 b(for)j(new)g(graph)g (group)g(\(handle\))75 1152 y FB(int)23 b(MPI)p 245 1152 15 2 v 17 w(Make)p 358 1152 V 17 w(graph\(MPI)p 591 1152 V 16 w(comm)g(comm)p 822 1152 V 17 w(old,)g(int)h(nnodes,)e(int)i (*index,)f(int)g(*edges,)393 1208 y(int)h(reorder,)e(MPI)p 778 1208 V 17 w(comm)h(*comm)p 1034 1208 V 17 w(graph\))75 1296 y(MPI)p 150 1296 V 17 w(MAKE)p 263 1296 V 16 w(GRAPH\(COMM)p 519 1296 V 16 w(OLD,)h(NNODES,)e(INDEX,)h(EDGES,)h(REORDER,)e(COMM)p 1490 1296 V 17 w(GRAPH,)h(IERROR\))170 1352 y(INTEGER)g(COMM)p 460 1352 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g(EDGES\(*\),)f(REORDER,)h (COMM)p 1575 1352 V 17 w(GRAPH,)170 1408 y(IERROR)166 1496 y FD(The)c(function)h(returns)f(a)f(handle)i(to)e(a)h(new)g(comm)o (unicator)g(to)f(whic)o(h)i(the)f(graph)f(top)q(ology)75 1552 y(information)c(is)g(attac)o(hed.)19 b(If)14 b Fs(reo)o(rder)f(=)h (false)h FD(then)f(the)g(rank)f(of)h(eac)o(h)g(pro)q(cess)g(in)h(the)f (new)g(group)f(is)75 1608 y(iden)o(tical)19 b(to)d(its)h(rank)g(in)g (the)g(old)h(group.)25 b(Otherwise,)17 b(the)g(function)h(ma)o(y)e (reorder)h(the)g(pro)q(cesses,)75 1665 y(so)d(as)g(to)f(c)o(ho)q(ose)i (a)f(go)q(o)q(d)g(em)o(b)q(edding)i(of)d(the)i(virtual)g(top)q(ology)f (on)o(to)f(the)h(ph)o(ysical)i(mac)o(hine.)k(If)15 b(the)75 1721 y(size)f Fs(nno)q(des)i FD(of)c(the)i(graph)f(is)h(smaller)g(than) f(the)g(size)h(of)f(the)h(group)f(of)g Fs(comm)m FD(,)d(then)k(a)f(pro) q(cess)g(whic)o(h)75 1778 y(is)j(not)f(a)g(mem)o(b)q(er)g(of)g(the)g (grid)h(is)g(returned)g(a)f(n)o(ull)h(handle.)22 b(The)15 b(call)i(is)f(erroneous)f(if)h(it)f(sp)q(eci\014es)i(a)75 1834 y(graph)e(that)f(is)i(larger)f(than)g(the)g(group)g(size.)166 1891 y(The)e(three)h(parameters)e Fs(nno)q(des,)j(index)f FD(and)g Fs(edges)g FD(de\014ne)g(the)f(graph)g(structure.)19 b Fs(nno)q(des)c FD(is)f(the)75 1948 y(n)o(um)o(b)q(er)i(of)e(no)q(des) i(of)f(the)g(graph.)20 b(The)15 b(no)q(des)h(are)f(n)o(um)o(b)q(ered)h (from)e FB(0)h FD(to)g FB(nnodes-1)p FD(.)166 2005 y(The)g FB(i)p FD(-th)g(en)o(try)g(of)g(arra)o(y)f Fs(index)i FD(stores)e(the)i(total)e(n)o(um)o(b)q(er)i(of)f(neigh)o(b)q(ors)g(of)g (the)g(\014rst)g FB(i)g FD(graph)75 2061 y(no)q(des.)40 b(The)21 b(lists)i(of)e(neigh)o(b)q(ors)h(of)f(no)q(des)h FB(0,)i(1,)g Fw(:)8 b(:)g(:)d FB(,)24 b(nnodes-1)c FD(are)i(stored)f (in)h(consecutiv)o(e)75 2118 y(lo)q(cations)16 b(in)g(arra)o(y)e Fs(edges)p FD(.)166 2175 y(The)19 b(de\014nitions)h(of)e(the)h(argumen) o(ts)f FB(nnodes)p FD(,)g FB(index)p FD(,)h(and)g FB(edges)f FD(are)g(illustrated)i(with)f(the)75 2232 y(follo)o(wing)d(simple)h (example:)166 2289 y(Assume)e(there)h(are)f(four)f(pro)q(cesses)i(0,)e (1,)h(2,)f(3)h(with)h(the)f(follo)o(wing)h(adjacency)g(matrix:)p 181 2342 433 2 v 180 2399 2 57 v 206 2382 a(pro)q(cess)p 373 2399 V 50 w(neigh)o(b)q(ors)p 613 2399 V 181 2401 433 2 v 180 2457 2 57 v 266 2440 a(0)p 373 2457 V 110 w(1,)f(3)p 613 2457 V 180 2513 V 266 2497 a(1)p 373 2513 V 110 w(0)p 613 2513 V 180 2570 V 266 2553 a(2)p 373 2570 V 110 w(3)p 613 2570 V 180 2626 V 266 2609 a(3)p 373 2626 V 110 w(0,)g(2)p 613 2626 V 181 2628 433 2 v 166 2704 a(Then,)g(the)h(input)g(argumen)o(ts)e(are:)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 168 175 168 174 bop 75 -100 a FD(168)882 b Fp(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)206 43 y FD(nno)q(des)h(=)50 b(4)206 100 y(index)16 b(=)81 b(2,)15 b(3,)f(4,)h(6)206 156 y(edges)g(=)82 b(1,)15 b(3,)f(0,)h(3,)f(0,)h(2)166 249 y(Th)o(us,)h(in)h(C,)e FB(index[0])g FD(is)i(the)f(degree)g(of)g (no)q(de)h(zero,)f(and)g FB(index[i])23 b(-)h(index[i-1])14 b FD(is)j(the)75 306 y(degree)22 b(of)g(no)q(de)g FB(i,)i(i=1,)f Fw(:)8 b(:)g(:)e FB(,)23 b(nnodes-1)p FD(;)h(the)e(list)h(of)e(neigh)o (b)q(ors)i(of)e(no)q(de)i(zero)e(is)i(stored)e(in)75 362 y FB(edges[j])p FD(,)d FB(0)g FA(\024)g FB(j)g FA(\024)g FB(index)p FD([)p FB(0)p FD(])11 b FA(\000)h FB(1)18 b FD(and)h(the)g(list)g(of)f(neigh)o(b)q(ors)h(of)f(no)q(de)h FB(i)p FD(,)f FB(i)g Fw(>)h FB(0)p FD(,)f(is)h(stored)f(in)75 419 y FB(edges[j])p FD(,)c FB(index)p FD([)p FB(i)9 b FA(\000)h FB(1)p FD(])i FA(\024)h FB(j)g FA(\024)g FB(index)p FD([)p FB(i)p FD(])8 b FA(\000)j FB(1)p FD(.)166 477 y(In)17 b(F)l(ortran,)f FB(index[1])f FD(is)j(the)e(degree)i(of)e(no)q (de)h(zero,)g(and)g FB(index[i+1])22 b(-)i(index[i])16 b FD(is)h(the)75 533 y(degree)22 b(of)g(no)q(de)g FB(i,)i(i=1,)f Fw(:)8 b(:)g(:)e FB(,)23 b(nnodes-1)p FD(;)h(the)e(list)h(of)e(neigh)o (b)q(ors)i(of)e(no)q(de)i(zero)e(is)i(stored)e(in)75 590 y FB(edges[j])p FD(,)h FB(1)i FA(\024)g FB(j)g FA(\024)g FB(index)p FD([)p FB(1)p FD(])c(and)i(the)g(list)h(of)f(neigh)o(b)q (ors)g(of)g(no)q(de)g FB(i)p FD(,)h FB(i)h Fw(>)g FB(0)p FD(,)f(is)g(stored)e(in)75 646 y FB(edges[j])p FD(,)14 b FB(index)p FD([)p FB(i)p FD(])8 b(+)j FB(1)h FA(\024)h FB(j)g FA(\024)g FB(index)p FD([)p FB(i)c FD(+)h FB(1)p FD(].)166 704 y(The)k(total)f(n)o(um)o(b)q(er)h(of)g(en)o(tries)g(in)h Fs(index)f FD(is)h FB(nnodes)e FD(and)h(the)f(total)h(n)o(um)o(b)q(er)g (of)f(en)o(tries)h(in)h Fs(edges)75 761 y FD(is)h(equal)g(to)e(the)h(n) o(um)o(b)q(er)h(of)f(graph)g(edges.)166 901 y Fl(Discussion:)33 b Ft(W)m(e)14 b(opted)g(to)f(ha)o(v)o(e)g(no)q(des)i(n)o(um)o(b)q(ered) e(from)f Fb(0)h Ft(to)h Fb(nnodes-1)p Ft(,)d(b)q(oth)j(in)f(C)g(and)h (F)m(ortran,)75 958 y(so)f(that)h(the)g(same)e Fb(index)g Ft(and)h Fb(edges)g Ft(lists)g(represen)o(t)i(a)e(graph)h(in)e(C)i(or)f (in)g(F)m(ortran.)k(Ho)o(w)o(ev)o(er,)d(since)g(arra)o(ys)75 1014 y(start)f(at)f(zero)h(in)e(C)i(and)f(one)g(in)g(F)m(ortran,)f (this)i(means)e(that)h(the)h(corresp)q(ondence)i(b)q(et)o(w)o(een)f (arra)o(y)e(indices)g(and)75 1071 y(no)q(de)i(n)o(um)o(b)q(ers)g(is)g (di\013eren)o(t)g(in)g(C)g(and)f(F)m(ortran.)166 1287 y Fl(Implemen)o(tati)o(on)k(note:)54 b Ft(The)19 b(follo)o(wing)c(top)q (ology)i(information)f(is)i(lik)o(ely)f(to)h(b)q(e)h(stored)g(with)f(a) 75 1337 y(comm)o(unicator:)134 1417 y(1.)k(T)o(yp)q(e)14 b(of)f(top)q(ology)g(\(Cartesian/graph\))134 1497 y(2.)22 b(F)m(or)13 b(a)h(Cartesian)g(top)q(ology:)245 1585 y Fj(\017)23 b Ft(ndims)12 b(\(n)o(um)o(b)q(er)h(of)h(dimensions\),)245 1654 y Fj(\017)23 b Ft(dims)12 b(\(n)o(um)o(b)q(ers)i(of)f(pro)q (cesses)k(p)q(er)d(co)q(ordinate)h(direction\),)245 1724 y Fj(\017)23 b Ft(p)q(erio)q(ds)14 b(\(p)q(erio)q(dicit)o(y)g (information\),)245 1793 y Fj(\017)23 b Ft(o)o(wn)p 364 1793 13 2 v 14 w(p)q(osition)13 b(\(o)o(wn)h(p)q(osition)f(in)h(grid,)f (could)g(also)h(b)q(e)g(computed)g(from)e(rank)h(and)h(dims\))134 1880 y(3.)22 b(F)m(or)13 b(a)h(graph)f(top)q(ology:)245 1967 y Fj(\017)23 b Ft(index,)13 b(edges)i(\(v)o(ectors)g(de\014ning)f (graph)g(structure\))166 2061 y(F)m(or)h(a)h(graph)f(structure)j(the)e (n)o(um)o(b)q(er)f(of)g(no)q(des)i(is)e(equal)h(to)f(the)i(n)o(um)o(b)q (er)e(of)g(pro)q(cesses)j(in)d(the)i(group.)75 2118 y(Therefore,)h(it)e (do)q(es)h(not)f(ha)o(v)o(e)g(to)g(b)q(e)h(stored)h(explicitly)m(.)23 b(An)17 b(additional)d(zero)k(en)o(try)e(at)h(the)g(start)g(of)e(arra)o (y)75 2174 y Fv(index)g Ft(will)e(simplify)e(access)k(to)f(the)h(top)q (ology)d(information.)75 2387 y Fn(6.5.1)49 b(T)l(op)q(ology)19 b(inquiry)e(functions)75 2476 y FD(If)h(a)f(top)q(ology)g(has)h(b)q (een)h(de\014ned)g(with)e(one)h(of)f(the)h(ab)q(o)o(v)o(e)f(functions,) i(the)f(top)q(ology)f(information)75 2533 y(can)g(b)q(e)g(lo)q(ok)o(ed) g(up)g(using)g(inquiry)h(functions.)24 b(They)17 b(all)h(are)e(lo)q (cal)h(calls,)h(and)e(do)h(not)f(require)h(an)o(y)75 2589 y(comm)o(unication.)166 2647 y(The)f(function)g Fs(MPI)p 523 2647 14 2 v 16 w(TOPO)p 665 2647 V 17 w(ST)l(A)l(TUS)h FD(returns)f(the)f(t)o(yp)q(e)h(of)f(top)q(ology)h(whic)o(h)g(is)g (assigned)g(to)f(a)75 2704 y(comm)o(unicator.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 169 176 169 175 bop 75 -100 a Fp(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FD(169)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(TOPO)p 302 45 V 17 w(ST)l(A)l(TUS\(comm)n(,)12 b(status\))117 125 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)11 b(\(handle\))117 205 y(OUT)124 b Fs(status)460 b Ft(top)q(ology)13 b(t)o(yp)q(e)h(of)g (comm)o(uni)o(cator)e Fb(comm)h Ft(\(c)o(hoice\))75 332 y FB(int)23 b(MPI)p 245 332 15 2 v 17 w(Topo)p 358 332 V 17 w(status\(MPI)p 615 332 V 16 w(comm)g(comm,)g(int)h(*status\))75 422 y(MPI)p 150 422 V 17 w(TOPO)p 263 422 V 16 w(STATUS\(COMM,)f (STATUS,)g(IERROR\))170 478 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)166 567 y FD(The)15 b(output)g(v)m(alue)i FB(status)d FD(is)i(one)f(of)g (the)g(follo)o(wing:)117 671 y Fv(MPI)p 194 671 13 2 v 14 w(GRAPH)602 b FD(graph)15 b(top)q(ology)117 730 y Fv(MPI)p 194 730 V 14 w(CART)631 b FD(Cartesian)15 b(top)q(ology)117 790 y Fv(MPI)p 194 790 V 14 w(UNDEFINED)508 b FD(no)15 b(top)q(ology)166 876 y(F)l(unctions)k Fs(MPI)p 462 876 14 2 v 16 w(GET)p 566 876 V 16 w(GRAPHDIMS)g FD(and)g Fs(MPI)p 1036 876 V 16 w(GET)p 1140 876 V 16 w(GRAPH)h FD(retriev)o(e)e(the)g(graph)g(top)q(ology)75 932 y(information)d(that)g(w)o(as)f(asso)q(ciated)i(with)f(a)g(comm)o (unicator)g(b)o(y)g Fs(MPI)p 1310 932 V 16 w(MAKE)p 1455 932 V 16 w(GRAPH)p FD(.)75 1038 y Fs(MPI)p 160 1038 V 16 w(GET)p 264 1038 V 17 w(GRAPHDIMS\(comm)m(,)d(nno)q(des,)17 b(nedges\))117 1118 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)7 b(for)i(group)h(with)f(graph)h(structure)i(\(handle\))117 1199 y(OUT)124 b Fs(nno)q(des)441 b Ft(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(graph)g(\(in)o(teger\))117 1279 y(OUT)124 b Fs(nedges)445 b Ft(n)o(um)o(b)q(er)13 b(of)h(edges)h(in)e (graph)h(\(in)o(teger\))75 1406 y FB(int)23 b(MPI)p 245 1406 15 2 v 17 w(Get)p 334 1406 V 17 w(graphdims\(MPI)p 663 1406 V 15 w(comm)h(comm,)f(int)g(*nnodes,)g(int)h(*nedges\))75 1495 y(MPI)p 150 1495 V 17 w(GET)p 239 1495 V 17 w(GRAPHDIMS\(COMM,)d (NNODES,)i(NEDGES,)g(IERROR\))170 1552 y(INTEGER)g(COMM,)h(NNODES,)e (NEDGES,)h(IERROR)166 1641 y FD(The)e(information)g(pro)o(vided)h(b)o (y)f Fs(MPI)p 864 1641 14 2 v 16 w(GET)p 968 1641 V 16 w(GRAPHDIMS)h FD(can)f(b)q(e)g(used)h(to)e(dimension)j(the)75 1697 y(v)o(ectors)14 b FB(index)h FD(and)g FB(edges)g FD(long)g(enough)h(in)g(the)f(follo)o(wing)h(call)g(to)f Fs(MPI)p 1389 1697 V 16 w(GET)p 1493 1697 V 16 w(GRAPH)p FD(.)75 1804 y Fs(MPI)p 160 1804 V 16 w(GET)p 264 1804 V 17 w(GRAPH\(comm)n(,)c(maxindex,)i(maxedges,)g(index,)j(edges\))117 1883 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)7 b(for)i(group)h(with)f(graph)h(structure)i(\(handle\))117 1964 y(IN)171 b Fs(maxi)o(ndex)392 b Ft(length)14 b(of)f(v)o(ector)i Fv(index)g Ft(in)f(the)g(calling)f(program)905 2020 y(\(in)o(teger\)) 117 2101 y(IN)171 b Fs(maxedges)386 b Ft(length)14 b(of)f(v)o(ector)i Fv(edges)g Ft(in)e(the)i(calling)d(program)905 2157 y(\(in)o(teger\)) 117 2238 y(OUT)124 b Fs(index)473 b Ft(arra)o(y)17 b(of)g(in)o(tegers)i (con)o(taining)d(the)i(graph)g(structure)h(\(for)905 2294 y(details)14 b(see)h(the)g(de\014nition)e(of)g Fv(MPI)p 1484 2294 13 2 v 15 w(MAKE)p 1617 2294 V 14 w(GRAPH)p Ft(\))117 2374 y(OUT)124 b Fs(edges)468 b Ft(arra)o(y)14 b(of)f(in)o(tegers)i(con)o(taining)e(the)h(graph)g(structure)75 2502 y FB(int)23 b(MPI)p 245 2502 15 2 v 17 w(Get)p 334 2502 V 17 w(graph\(MPI)p 567 2502 V 16 w(comm)g(comm,)g(int)h (maxindex,)f(int)g(maxedges,)g(int)g(*index,)393 2558 y(int)h(*edges\))75 2647 y(MPI)p 150 2647 V 17 w(GET)p 239 2647 V 17 w(GRAPH\(COMM,)e(MAXINDEX,)h(MAXEDGES,)f(INDEX,)h(EDGES,) g(IERROR\))170 2704 y(INTEGER)g(COMM,)h(MAXINDEX,)e(MAXEDGES,)h (INDEX\(*\),)f(EDGES\(*\),)h(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 170 177 170 176 bop 75 -100 a FD(170)882 b Fp(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)166 45 y FD(The)e(functions)g Fs(MPI)p 535 45 14 2 v 15 w(GET)p 638 45 V 17 w(CARTDIM)f FD(and)h Fs(MPI)p 1040 45 V 16 w(GET)p 1144 45 V 16 w(CART)g FD(return)g(the)f(Cartesian)g(top)q(ology)75 102 y(information)j(that)g (w)o(as)f(asso)q(ciated)i(with)f(a)g(comm)o(unicator)g(b)o(y)g(the)g (function)h Fs(MPI)p 1566 102 V 16 w(MAKE)p 1711 102 V 16 w(CART)p FD(.)75 221 y Fs(MPI)p 160 221 V 16 w(GET)p 264 221 V 17 w(CARTDIM\(com)o(m)m(,)c(ndims\))117 314 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)7 b(for)i(group)g(with)g (Cartesian)g(structure)j(\(han-)905 370 y(dle\))117 477 y(OUT)124 b Fs(ndims)458 b Ft(n)o(um)o(b)q(er)15 b(of)g(dimensions)f (of)g(the)i(Cartesian)g(structure)h(\(in-)905 533 y(teger\))75 673 y FB(int)23 b(MPI)p 245 673 15 2 v 17 w(Get)p 334 673 V 17 w(cartdim\(MPI)p 615 673 V 16 w(comm)g(comm,)g(int)h(*ndims\)) 75 776 y(MPI)p 150 776 V 17 w(GET)p 239 776 V 17 w(CARTDIM\(COMM,)e (NDIMS,)h(IERROR\))170 832 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)75 982 y Fs(MPI)p 160 982 14 2 v 16 w(GET)p 264 982 V 17 w(CART\(comm)m(,)12 b(maxdim)m(s,)g(dims,)h(p)q(erio)q(ds,)j(co)q(o)o (rds\))117 1074 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j(\(han-)905 1131 y(dle\))117 1237 y(IN)171 b Fs(maxdim)m(s)402 b Ft(length)16 b(of)f(v)o(ectors)i Fb(dims,)k(periods)p Ft(,)15 b(and)g Fb(coords)g Ft(in)g(the)905 1294 y(calling)e(program)f (\(in)o(teger\))117 1400 y(OUT)124 b Fs(dims)481 b Ft(n)o(um)o(b)q(er) 11 b(of)g(pro)q(cesses)k(for)c(eac)o(h)h(Cartesian)g(dimension)e(\(ar-) 905 1457 y(ra)o(y)k(of)f(in)o(teger\))117 1563 y(OUT)124 b Fs(p)q(erio)q(ds)436 b Ft(p)q(erio)q(dicit)o(y)18 b(\()p Fv(true)p Ft(/)p Fv(false)p Ft(\))i(for)e(eac)o(h)h(Cartesian)g (dimension)905 1620 y(\(arra)o(y)14 b(of)f(logical\))117 1726 y(OUT)124 b Fs(co)q(o)o(rds)449 b Ft(co)q(ordinates)19 b(of)f(calling)f(pro)q(cess)j(in)d(Cartesian)i(structure)905 1783 y(\(arra)o(y)14 b(of)f(in)o(teger\))75 1923 y FB(int)23 b(MPI)p 245 1923 15 2 v 17 w(Get)p 334 1923 V 17 w(cart\(MPI)p 543 1923 V 16 w(comm)g(comm,)h(int)f(maxdims,)g(int)g(*dims,)g(int)h (*periods,)393 1979 y(int)g(*coords\))75 2082 y(MPI)p 150 2082 V 17 w(GET)p 239 2082 V 17 w(CART\(COMM,)e(MAXDIMS,)h(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 2138 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(PERIODS\(*\),)f(COORDS\(*\),)h(IERROR)166 2323 y Fl(Discussion:)34 b Ft(The)15 b(inquiries)e(for)h(general)g (graphs)g(and)g(for)g(Cartesian)g(top)q(ology)f(information)e(is)j(th)o (us)75 2379 y(split)h(up)h(in)o(to)f(t)o(w)o(o)h(calls.)23 b(The)16 b(reason)g(is)g(that)g(otherwise)h(the)f(user)h(migh)o(t)c (not)j(kno)o(w)f(a)h(safe)g(v)n(alue)f(for)g(the)75 2436 y(dimension)d(of)i(v)o(ectors)h(that)e(will)g(store)i(the)f(graph)g(or) g(Cartesian)g(grid)f(description.)166 2591 y FD(F)l(or)e(a)g(pro)q (cess)h(group)f(with)h(Cartesian)g(structure,)f(the)h(function)h Fs(MPI)p 1399 2591 14 2 v 15 w(CART)p 1533 2591 V 17 w(RANK)f FD(translates)75 2647 y(the)21 b(logical)h(pro)q(cess)f(co)q (ordinates)g(to)f(pro)q(cess)h(ranks)f(as)g(they)h(are)f(used)i(b)o(y)e (the)h(p)q(oin)o(t{to{p)q(oin)o(t)75 2704 y(routines:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 171 178 171 177 bop 75 -100 a Fp(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FD(171)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(CART)p 295 45 V 17 w(RANK\(comm)m(,)12 b(co)q(o)o(rds,)j(rank\))117 130 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j(\(han-)905 186 y(dle\))117 276 y(IN)171 b Fs(co)q(o)o(rds)449 b Ft(in)o(teger)15 b(arra)o(y)g(\(of)f(size)h Fb(ndims)p Ft(\))f(sp)q(ecifying)g(the)i(Cartesian)905 333 y(co)q(ordinates)f(of)e (a)h(pro)q(cess)117 423 y(OUT)124 b Fs(rank)488 b Ft(rank)14 b(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 555 y FB(int)23 b(MPI)p 245 555 15 2 v 17 w(Cart)p 358 555 V 17 w(rank\(MPI)p 567 555 V 16 w(comm)g(comm,)g(int)h(*coords,)f (int)g(*rank\))75 649 y(MPI)p 150 649 V 17 w(CART)p 263 649 V 16 w(RANK\(COMM,)g(COORDS,)g(RANK,)g(IERROR\))170 705 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h(IERROR)166 799 y FD(If)14 b(for)f(dimension)j FB(i)e FD(with)g FB(periods\(i\))22 b(=)i(true)13 b FD(the)h(co)q(ordinate)h FB(coords\(i\))d FD(is)j(out)e(of)h(range,)75 856 y(i.e.,)39 b FB(coords\(i\))22 b Fw(<)j FB(0)34 b FD(or)g FB(coords\(i\))22 b FA(\025)i FB(dims\(i\))p FD(,)38 b(it)d(is)g(shifted)g(bac)o(k)f(to)f(the)i(in)o (terv)m(al)75 912 y FB(0)24 b FA(\024)g FB(coords\(i\))f FA(\024)h FB(dims\(i\))c FD(automatically)l(.)40 b(Out{of{range)21 b(co)q(ordinates)h(are)f(erroneous)h(for)75 969 y(ap)q(erio)q(dic)17 b(dimensions.)166 1033 y(The)e(in)o(v)o(erse)h(rank-to-co)q(ordinates)f (translation)g(is)h(pro)o(vided)g(b)o(y)f(the)g(function:)75 1144 y Fs(MPI)p 160 1144 14 2 v 16 w(CART)p 295 1144 V 17 w(COORDS\(comm)n(,)d(rank,)j(maxdim)m(s,)d(co)q(o)o(rds\))117 1229 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j(\(han-)905 1285 y(dle\))117 1375 y(IN)171 b Fs(rank)488 b Ft(rank)14 b(of)f(a)h(pro)q(cess)i(within)d(group)g(of)h Fv(comm)8 b Ft(\(in)o(teger\))117 1465 y(IN)171 b Fs(maxdim)m(s)402 b Ft(length)11 b(of)f(v)o(ector)i Fb(coord)d Ft(in)h(the)i(calling)d (program)g(\(in)o(teger\))117 1556 y(OUT)124 b Fs(co)q(o)o(rds)449 b Ft(in)o(teger)14 b(arra)o(y)f(\(of)g(size)h Fb(ndims)p Ft(\))e(con)o(taining)h(the)h(Cartesian)905 1612 y(co)q(ordinates)h(of) e(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 1744 y FB(int)23 b(MPI)p 245 1744 15 2 v 17 w(Cart)p 358 1744 V 17 w(coords\(MPI)p 615 1744 V 16 w(comm)g(comm,)g(int)h(rank,)f(int)g (maxdims,)g(int)g(*coords\))75 1838 y(MPI)p 150 1838 V 17 w(CART)p 263 1838 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g (COORDS,)f(IERROR\))170 1895 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f (COORDS\(*\),)h(IERROR)166 1988 y FD(The)14 b(follo)o(wing)h(functions) h(pro)o(vide)e(\\adjacency")h(information)f(for)g(a)g(general)g(graph)g (top)q(ology)l(.)75 2100 y Fs(MPI)p 160 2100 14 2 v 16 w(GRAPH)p 326 2100 V 17 w(NEIGHBORS)p 594 2100 V 19 w(COUNT\(comm)m(,)e (rank,)j(nneighb)q(o)o(rs\))117 2184 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)12 b(for)i(a)g(group)g(with)g(graph)g(top)q(ology) f(\(han-)905 2241 y(dle\))117 2331 y(IN)171 b Fs(rank)488 b Ft(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(\(in)o(teger\))117 2421 y(OUT)124 b Fs(nneighb)q(o)o(rs)369 b Ft(n)o(um)o(b)q(er)13 b(of)h(neigh)o(b)q(ors)g(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o (teger\))75 2553 y FB(int)23 b(MPI)p 245 2553 15 2 v 17 w(Graph)p 382 2553 V 17 w(neighbors)p 615 2553 V 16 w(count\(MPI)p 847 2553 V 16 w(comm)g(comm,)g(int)h(rank,)f(int)g (*nneighbors\))75 2647 y(MPI)p 150 2647 V 17 w(GRAPH)p 287 2647 V 16 w(NEIGHBORS)p 519 2647 V 16 w(COUNT\(COMM,)g(RANK,)g (NNEIGHBORS,)f(IERROR\))170 2704 y(INTEGER)h(COMM,)h(RANK,)f (NNEIGHBORS,)f(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 172 179 172 178 bop 75 -100 a FD(172)882 b Fp(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(NEIGHBORS\(comm)o(,)d(rank,)j (maxneighb)q(o)o(rs,)d(neighb)q(o)o(rs\))117 135 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)12 b(for)i(a)g(group)g(with)g(graph) g(top)q(ology)f(\(han-)905 192 y(dle\))117 292 y(IN)171 b Fs(rank)488 b Ft(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(\(in)o (teger\))117 393 y(OUT)124 b Fs(maxneighb)q(o)o(rs)310 b Ft(size)15 b(of)e(arra)o(y)h Fv(neighb)q(o)o(rs)h Ft(\(in)o(teger\)) 117 494 y(OUT)124 b Fs(neighb)q(o)o(rs)392 b Ft(ranks)13 b(of)g(pro)q(cesses)i(that)e(are)h(neigh)o(b)q(ors)f(to)g(sp)q (eci\014ed)h(pro-)905 550 y(cess)i(\(arra)o(y)d(of)h(in)o(teger\))75 688 y FB(int)23 b(MPI)p 245 688 15 2 v 17 w(Graph)p 382 688 V 17 w(neighbors\(MPI)p 711 688 V 15 w(comm)g(comm,)h(int)f(rank,)g (int)h(*maxneighbors,)393 744 y(int)g(*neighbors\))75 843 y(MPI)p 150 843 V 17 w(GRAPH)p 287 843 V 16 w(NEIGHBORS\(COMM,)e (RANK,)h(MAXNEIGHBORS,)f(NEIGHBORS,)h(IERROR\))170 900 y(INTEGER)g(COMM,)h(RANK,)f(MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR) 166 999 y FD(Example:)k(Supp)q(ose)19 b(that)f FB(comm)f FD(is)i(a)e(comm)o(unicator)h(for)g(a)f(group)h(with)h(a)e(sh)o (u\017e-exc)o(hange)75 1056 y(top)q(ology:)h(The)13 b(group)f(has)g(2) 596 1039 y Ff(n)631 1056 y FD(mem)o(b)q(ers.)19 b(Eac)o(h)12 b(pro)q(cess)h Fw(a)1130 1063 y Fi(1)1149 1056 y Fw(;)8 b(:)g(:)g(:)d(;)j(a)1275 1063 y Ff(n)1310 1056 y FD(\()p Fw(a)1352 1063 y Ff(i)1378 1056 y FA(2)13 b(f)p FD(0)p Fw(;)8 b FD(1)p FA(g)p FD(\))i(has)i(three)h(neigh-)75 1112 y(b)q(ors:)k(exc)o(hange\()p Fw(a)411 1119 y Fi(1)431 1112 y Fw(;)8 b(:)g(:)g(:)d(;)j(a)557 1119 y Ff(n)580 1112 y FD(\))k(=)h Fw(a)682 1119 y Fi(1)702 1112 y Fw(;)8 b(:)g(:)g(:)t(;)g(a)827 1119 y Ff(n)p Fg(\000)p Fi(1)895 1112 y Fw(;)g FD(\026)-23 b Fw(a)940 1119 y Ff(n)973 1112 y FD(\()q(\026)f Fw(a)12 b FD(=)h(1)p FA(\000)p Fw(a)p FD(\),)e(sh)o(u\017e\()p Fw(a)1366 1119 y Fi(1)1385 1112 y Fw(;)d(:)g(:)g(:)d(;)j(a)1511 1119 y Ff(n)1534 1112 y FD(\))k(=)h Fw(a)1636 1119 y Fi(2)1656 1112 y Fw(;)8 b(:)g(:)g(:)d(;)j(a)1782 1119 y Ff(n)1805 1112 y Fw(;)g(a)1850 1119 y Fi(1)1868 1112 y FD(,)75 1169 y(and)17 b(unsh)o(u\017e\()p Fw(a)382 1176 y Fi(1)402 1169 y Fw(;)8 b(:)g(:)g(:)d(;)j(a)528 1176 y Ff(n)551 1169 y FD(\))15 b(=)h Fw(a)659 1176 y Ff(n)683 1169 y Fw(;)8 b(a)728 1176 y Fi(1)747 1169 y Fw(;)g(:)g(:)g(:)d(;)j(a)873 1176 y Ff(n)p Fg(\000)p Fi(1)941 1169 y FD(.)25 b(The)17 b(graph)g(adjacency)g(list)h(is)g(illustrated)g(b)q(elo)o(w)75 1225 y(for)d Fw(n)d FD(=)h(3)p 75 1289 1119 2 v 74 1346 2 57 v 133 1329 a Fx(no)q(de)p 300 1346 V 104 w(exc)o(hange)122 b(sh)o(u\017e)h(unsh)o(u\017e)p 1193 1346 V 74 1402 V 300 1402 V 326 1385 a FD(neigh)o(b)q(ors\(1\))49 b(neigh)o(b)q (ors\(2\))h(neigh)o(b)q(ors\(3\))p 1193 1402 V 75 1404 1119 2 v 74 1461 2 57 v 100 1444 a(0)f(\(000\))p 300 1461 V 161 w(1)275 b(0)g(0)p 1193 1461 V 74 1517 V 100 1500 a(1)49 b(\(001\))p 300 1517 V 161 w(0)275 b(2)g(4)p 1193 1517 V 74 1573 V 100 1556 a(2)49 b(\(010\))p 300 1573 V 161 w(3)275 b(4)g(1)p 1193 1573 V 74 1630 V 100 1613 a(3)49 b(\(011\))p 300 1630 V 161 w(2)275 b(6)g(5)p 1193 1630 V 74 1686 V 100 1669 a(4)49 b(\(100\))p 300 1686 V 161 w(5)275 b(1)g(2)p 1193 1686 V 74 1743 V 100 1726 a(5)49 b(\(101\))p 300 1743 V 161 w(4)275 b(3)g(6)p 1193 1743 V 74 1799 V 100 1782 a(6)49 b(\(110\))p 300 1799 V 161 w(7)275 b(5)g(3)p 1193 1799 V 74 1856 V 100 1839 a(7)49 b(\(111\))p 300 1856 V 161 w(6)275 b(7)g(7)p 1193 1856 V 75 1857 1119 2 v 75 2016 a FB(C)48 b(assume:)22 b(each)i(process)f(has)g(stored)g(a)h(real)f(number)g(A.)75 2072 y(C)48 b(extract)22 b(neighborhood)h(information)218 2129 y(CALL)g(MPI_COMM_RANK\(comm,)f(myrank,)h(ierr\))218 2185 y(CALL)g(MPI_GRAPH_NEIGHBORS\(comm,)e(myrank,)i(3,)g(neighbors,)g (ierr\))75 2242 y(C)48 b(perform)22 b(exchange)h(permutation)218 2298 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e (neighbors\(1\),)h(0,)194 2355 y(+)119 b(neighbors\(1\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 2411 y(C)48 b(perform)22 b(shuffle)h(permutation)218 2468 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e (1,)j(MPI_REAL,)e(neighbors\(2\),)h(0,)194 2524 y(+)119 b(neighbors\(3\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 2581 y(C)48 b(perform)22 b(unshuffle)h(permutation)218 2637 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e (neighbors\(3\),)h(0,)194 2693 y(+)119 b(neighbors\(2\),)23 b(0,)g(comm,)g(status,)g(ierr\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 173 180 173 179 bop 75 -100 a Fp(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FD(173)75 45 y Fn(6.5.2)49 b(Ca)o(rtesian)16 b(shift)g(co)q(o)o (rdinates)75 131 y FD(If)i(the)f(pro)q(cess)h(top)q(ology)f(is)h(a)f (Cartesian)g(structure,)h(a)f(collectiv)o(e)i(shift)f(op)q(eration)g (is)g(most)e(lik)o(ely)75 187 y(to)g(b)q(e)g(used)h(along)f(a)g(co)q (ordinate)h(direction.)24 b(In)17 b(MPI)f(shift)g(op)q(erations)h(are)e (handled)j(b)o(y)e(the)g(more)75 244 y(general)k(send{receiv)o(e)i (functions.)35 b(As)20 b(input)h(they)f(tak)o(e)f(the)h(rank)g(of)f(a)h (source)g(pro)q(cess)g(for)g(the)75 300 y(receiv)o(e,)c(and)f(the)g (rank)g(of)g(a)g(destination)h(pro)q(cess)g(for)e(the)h(send.)166 357 y(If)j(the)g(function)h Fs(MPI)p 561 357 14 2 v 16 w(CART)p 696 357 V 17 w(SHIFT)f FD(is)g(called)i(for)d(a)h(Cartesian)g (pro)q(cess)g(group,)g(it)g(pro)o(vides)75 413 y(the)c(calling)i(pro)q (cess)e(with)h(the)f(ab)q(o)o(v)o(e)g(iden)o(ti\014ers,)h(whic)o(h)g (then)g(can)f(b)q(e)h(passed)f(to)f(the)i(send{receiv)o(e)75 470 y(function.)21 b(The)15 b(user)g(sp)q(eci\014es)i(the)e(co)q (ordinate)h(direction)g(for)f(the)g(shift)h(and)f(the)g(step)g(length.) 21 b(The)75 526 y(function)16 b(is)g(lo)q(cal.)75 630 y Fs(MPI)p 160 630 V 16 w(CART)p 295 630 V 17 w(SHIFT\(com)o(m)n(,)11 b(direction,)16 b(disp,)g(rank)p 977 630 V 16 w(source,)g(rank)p 1224 630 V 16 w(dest\))117 707 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j (\(han-)905 763 y(dle\))117 835 y(IN)171 b Fs(direction)408 b Ft(co)q(ordinate)14 b(dimension)f(of)g(shift)h(\(in)o(teger\))117 907 y(IN)171 b Fs(disp)497 b Ft(displacemen)o(t)18 b(\()p Fu(>)h Ft(0:)27 b(up)o(w)o(ards)18 b(shift,)h Fu(<)g Ft(0:)27 b(do)o(wn)o(w)o(ards)905 964 y(shift\))14 b(\(in)o(teger\))117 1036 y(OUT)124 b Fs(rank)p 420 1036 V 16 w(source)353 b Ft(rank)14 b(of)f(source)j(pro)q(cess)f(\(in)o(teger\))117 1108 y(OUT)124 b Fs(rank)p 420 1108 V 16 w(dest)396 b Ft(rank)14 b(of)f(destination)h(pro)q(cess)i(\(in)o(teger\))75 1232 y FB(int)23 b(MPI)p 245 1232 15 2 v 17 w(Cart)p 358 1232 V 17 w(shift\(MPI)p 591 1232 V 16 w(comm)g(comm,)g(int)h (direction,)e(int)i(disp,)f(int)g(*rank)p 1681 1232 V 17 w(source,)393 1289 y(int)h(*rank)p 612 1289 V 16 w(dest\))75 1375 y(MPI)p 150 1375 V 17 w(CART)p 263 1375 V 16 w(SHIFT\(COMM,)f (DIRECTION,)f(DISP,)h(RANK)p 1067 1375 V 17 w(SOURCE,)g(RANK)p 1371 1375 V 17 w(DEST,)g(IERROR\))170 1431 y(INTEGER)g(COMM,)h (DIRECTION,)e(DISP,)h(RANK)p 1009 1431 V 17 w(SOURCE,)g(RANK)p 1313 1431 V 16 w(DEST,)h(IERROR)166 1518 y FD(Dep)q(ending)17 b(on)e(the)g(p)q(erio)q(dicit)o(y)i(of)e(the)g(Cartesian)g(group)g(in)h (the)f(sp)q(eci\014ed)i(co)q(ordinate)f(direc-)75 1574 y(tion,)d Fs(MPI)p 264 1574 14 2 v 16 w(CART)p 399 1574 V 17 w(SHIFT)f FD(pro)o(vides)h(the)g(iden)o(ti\014ers)h(for)e(a)h (circular)h(or)e(an)g(end{o\013)h(shift.)19 b(In)13 b(the)g(case)75 1631 y(of)j(an)h(end{o\013)g(shift,)g(the)g(v)m(alue)h Fv(MPI)p 746 1631 13 2 v 14 w(PROCNULL)e FD(ma)o(y)g(b)q(e)i(returned)f (in)g Fs(rank)p 1478 1631 14 2 v 17 w(source)g FD(or)f Fs(rank)p 1771 1631 V 17 w(dest)p FD(,)75 1687 y(indicating)h(that)e (the)g(source)g(or)g(the)g(destination)h(for)f(the)g(shift)h(is)f(out)g (of)g(range.)166 1744 y(Example:)37 b(The)24 b(comm)o(unicator)f Fs(comm)18 b FD(is)24 b(asso)q(ciated)f(with)h(a)f(t)o(w)o (o-dimensional)i(p)q(erio)q(dic)75 1800 y(Cartesian)14 b(top)q(ology)l(.)19 b(A)14 b(t)o(w)o(o-dimensional)i(arra)o(y)c(of)i FB(REAL)p FD(s)g(is)g(stored)g(one)g(elemen)o(t)h(p)q(er)g(pro)q(cess,) f(in)75 1857 y(v)m(ariable)j FB(A)p FD(.)d(One)i(wishes)g(to)f(sk)o(ew) f(this)i(arra)o(y)l(,)e(b)o(y)h(shifting)h(column)g Fw(i)f FD(b)o(y)g Fw(i)p FD(.)75 1935 y FB(....)75 1991 y(C)24 b(find)f(process)g(rank)218 2047 y(CALL)g(MPI_COMM_RANK\(comm,)f(rank,) h(ierr,)g(ierr\)\))75 2104 y(C)h(find)f(Cartesian)g(coordinates)218 2160 y(CALL)g(MPI_CART_COORD\(comm,)f(rank,)h(coords,)g(ierr\))75 2217 y(C)h(compute)f(shift)g(source)g(and)g(destination)218 2273 y(CALL)g(MPI_CART_SHIFT\(comm,)f(2,)h(coords\(2\),)g(source,)g (dest,)g(ierr\))75 2330 y(C)h(skew)f(array)218 2386 y(CALL)g (MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(dest,)i(0,)f(source,)g(0,) h(comm,)194 2443 y(+)621 b(status,)23 b(ierr\))75 2561 y Fn(6.5.3)49 b(P)o(a)o(rtitioning)17 b(of)g(Ca)o(rtesian)e(structures) 75 2647 y FD(If)40 b(a)f(Cartesian)g(top)q(ology)g(has)g(b)q(een)i (created)e(with)h Fs(MPI)p 1280 2647 V 16 w(MAKE)p 1425 2647 V 16 w(CART)p FD(,)g(the)f(function)75 2704 y Fs(MPI)p 160 2704 V 16 w(CART)p 295 2704 V 17 w(SUB)20 b FD(can)f(b)q(e)h(used)g (to)e(partition)i(the)f(comm)o(unicator)g(group)g(in)o(to)g(subgroups)g (whic)o(h)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 174 181 174 180 bop 75 -100 a FD(174)882 b Fp(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y FD(form)h(lo)o(w)o(er{dimensional) h(Cartesian)f(subgrids,)h(and)g(to)f(build)i(for)d(eac)o(h)i(subgroup)f (a)g(comm)o(unica-)75 102 y(tor)e(with)i(the)f(asso)q(ciated)h(subgrid) g(Cartesian)f(top)q(ology)l(.)75 206 y Fs(MPI)p 160 206 14 2 v 16 w(CART)p 295 206 V 17 w(SUB\(comm)m(,)d(remain)p 690 206 V 14 w(dims,)g(comm)p 937 206 V 11 w(new\))117 284 y Ft(IN)171 b Fs(comm)450 b Ft(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j(\(han-)905 341 y(dle\))117 418 y(IN)171 b Fs(rem)o(ain)p 467 418 V 14 w(dims)337 b Ft(the)22 b Fb(i)p Ft(-th)f(en)o(try)g(of)g Fv(remain)p 1371 418 13 2 v 13 w(dims)d Ft(sp)q(eci\014es)23 b(whether)f(the)905 474 y Fb(i)p Ft(-th)g(dimension)e(is)i(k)o(ept)g (in)f(the)h(subgrid)g(\()p Fb(true)p Ft(\))f(or)h(is)905 531 y(dropp)q(ed)15 b(\()p Fb(false)p Ft(\))e(\(logical)f(v)o(ector\)) 117 608 y(OUT)124 b Fs(comm)p 454 608 14 2 v 10 w(new)366 b Ft(comm)o(unicator)14 b(con)o(taining)h(the)i(subgrid)g(whic)o(h)f (includes)905 664 y(the)f(calling)d(pro)q(cess)k(\(handle\))75 789 y FB(int)23 b(MPI)p 245 789 15 2 v 17 w(Cart)p 358 789 V 17 w(sub\(MPI)p 543 789 V 16 w(comm)g(comm,)h(int)f(*remain)p 1085 789 V 16 w(dims,)g(MPI)p 1316 789 V 17 w(comm)h(*comm)p 1573 789 V 16 w(new\))75 877 y(MPI)p 150 877 V 17 w(CART)p 263 877 V 16 w(SUB\(COMM,)f(REMAIN)p 662 877 V 16 w(DIMS,)h(COMM)p 918 877 V 16 w(NEW,)g(IERROR\))170 933 y(INTEGER)f(COMM,)h(REMAIN)p 652 933 V 16 w(DIMS\(*\),)f(COMM)p 979 933 V 16 w(NEW,)h(IERROR)166 1021 y FD(Example:)19 b(Assume)12 b(that)f Fs(MPI)p 716 1021 14 2 v 16 w(MAKE)p 861 1021 V 28 w(CART\(...,)h(comm)m(\))c FD(has)k(de\014ned)i(a)d(\(2)t FA(\002)t FD(3)t FA(\002)t FD(4\))g(grid.)19 b(Let)75 1077 y FB(remain)p 222 1077 15 2 v 16 w(dims)24 b(=)f(\(true,)g(false,)g(true\))p FD(.)18 b(Then)11 b(a)e(call)j(to)d Fs(MPI)p 1247 1077 14 2 v 16 w(CART)p 1382 1077 V 17 w(SUB\(comm)n(,)e(remain)p 1773 1077 V 14 w(dims,)75 1134 y(comm)p 196 1134 V 11 w(new\))14 b FD(will)h(create)f(comm)o(unicators)f(for)h(three)g(new)g (groups)f(with)h(eigh)o(t)h(pro)q(cesses,)f(eac)o(h)g(with)75 1190 y(a)21 b(2)13 b FA(\002)h FD(4)21 b(Cartesian)f(top)q(ology)l(.)37 b(If)21 b FB(remain)p 874 1190 15 2 v 16 w(dims)j(=)g(\(false,)e (false,)h(true\))e FD(then)g(the)g(call)h(to)75 1247 y Fs(MPI)p 160 1247 14 2 v 16 w(CART)p 295 1247 V 17 w(SUB\(comm)m(,)g(remain)p 700 1247 V 14 w(dims,)g(comm)p 957 1247 V 11 w(new\))h FD(will)i(create)e(comm)o(unicators)g(for)f (six)i(new)75 1303 y(groups)15 b(with)g(four)g(pro)q(cesses,)g(eac)o(h) h(with)f(a)g(one-dimensional)j(Cartesian)d(top)q(ology)l(.)75 1430 y Fn(6.5.4)49 b(Lo)o(w{level)18 b(top)q(ology)g(functions)75 1518 y FD(The)11 b(t)o(w)o(o)e(new)h(functions)i(in)o(tro)q(duced)f(in) h(this)f(section)g(can)f(b)q(e)h(used)g(to)f(implemen)o(t)i(all)f (other)f(top)q(ology)75 1574 y(functions.)21 b(In)15 b(general)h(they)f(will)i(not)e(b)q(e)h(called)h(b)o(y)e(the)g(user)g (directly)l(.)75 1679 y Fs(MPI)p 160 1679 V 16 w(MAP)p 275 1679 V 16 w(CART\(comm)p 546 1679 V 11 w(old,)g(ndims,)e(dims,)g(p) q(erio)q(ds,)j(newrank\))117 1757 y Ft(IN)171 b Fs(comm)p 454 1757 V 10 w(old)383 b Ft(comm)o(unicator)11 b(for)j(paren)o(t)g (group)g(\(handle\))117 1834 y(IN)171 b Fs(ndims)458 b Ft(n)o(um)o(b)q(er)12 b(of)f(dimensions)h(of)f(Cartesian)i(structure) h(\(in)o(teger\))117 1911 y(IN)171 b Fs(dims)481 b Ft(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fb(ndims)e Ft(sp)q(ecifying)h(the)h(n)o(um)o (b)q(er)f(of)905 1967 y(pro)q(cesses)g(in)c(eac)o(h)i(co)q(ordinate)f (direction)117 2044 y(IN)171 b Fs(p)q(erio)q(ds)436 b Ft(logical)16 b(arra)o(y)h(of)g(size)h Fb(ndims)f Ft(sp)q(ecifying)g (the)h(p)q(erio)q(dicit)o(y)905 2101 y(sp)q(eci\014cation)d(in)e(eac)o (h)i(co)q(ordinate)f(direction)117 2178 y(OUT)124 b Fs(newrank)414 b Ft(reordered)11 b(rank)f(of)e(the)i(calling)e(pro)q(cess;)k Fv(MPI)p 1643 2178 13 2 v 15 w(UNDEFINED)905 2234 y Ft(if)h(calling)g (pro)q(cess)i(do)q(es)g(not)f(b)q(elong)f(to)h(grid)g(\(in)o(teger\))75 2359 y FB(int)23 b(MPI)p 245 2359 15 2 v 17 w(Map)p 334 2359 V 17 w(cart\(MPI)p 543 2359 V 16 w(comm)g(comm)p 774 2359 V 17 w(old,)g(int)h(ndims,)f(int)g(*dims,)g(int)h(*period,)393 2416 y(int)g(*newrank\))75 2503 y(MPI)p 150 2503 V 17 w(MAP)p 239 2503 V 17 w(CART\(COMM)p 472 2503 V 15 w(OLD,)g(NDIMS,)f (DIMS,)g(PERIOD,)g(NEWRANK,)g(IERROR\))170 2560 y(INTEGER)g(COMM)p 460 2560 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(PERIOD\(*\),)g(NEWRANK,)f (IERROR)166 2647 y FD(This)c(function)g(computes)g(an)g(optimal)g (placemen)o(t)g(for)f(the)g(calling)j(pro)q(cess)e(on)f(the)h(ph)o (ysical)75 2704 y(mac)o(hine.)31 b(A)19 b(p)q(ossible)i(implemen)o (tation)f(of)e(this)h(function)h(is)f(to)f(alw)o(a)o(ys)g(return)h(the) g(rank)g(of)f(the)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 175 182 175 181 bop 75 -100 a Fp(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FD(175)75 45 y(calling)17 b(pro)q(cess,)e(i.e.,)g(not)g(to)f(p)q (erform)h(an)o(y)g(reordering.)166 114 y(The)k(function)g Fs(MPI)p 529 114 14 2 v 16 w(MAKE)p 674 114 V 16 w(CART\(comm)p 945 114 V 11 w(old,)g(ndims,)e(dims,)g(p)q(erio)q(ds,)k(reo)o(rder,)c (comm)p 1761 114 V 11 w(ca)o(rt\))p FD(,)75 170 y(with)j FB(reorder)j(=)g(true)c FD(can)h(b)q(e)g(implemen)o(ted)h(b)o(y)f (calling)h Fs(MPI)p 1264 170 V 16 w(MAP)p 1379 170 V 16 w(CART\(comm)p 1650 170 V 11 w(old,)f(ndims,)75 227 y(dims,)11 b(p)q(erio)q(ds,)k(newrank\))p FD(,)f(then)g(calling)h Fs(MPI)p 883 227 V 16 w(COMM)p 1041 227 V 16 w(SPLIT\(comm)m(,)10 b(colo)o(r,)i(k)o(ey)l(,)i(comm)p 1659 227 V 11 w(ca)o(rt\))p FD(,)f(with)75 283 y FB(color)23 b(=)h(0)17 b FD(if)h FB(newrank)23 b FA(6)p FD(=)h FB(MPI)p 676 283 15 2 v 17 w(UNDEFINED)p FD(,)16 b FB(color)23 b(=)h(MPI)p 1201 283 V 17 w(UNDEFINED)16 b FD(otherwise,)i(and)g FB(key)23 b(=)75 340 y(newrank)p FD(.)166 409 y(The)15 b(function)g Fs(MPI)p 521 409 14 2 v 16 w(CART)p 656 409 V 17 w(SUB\(comm)n(,)c (rema)o(in)p 1050 409 V 14 w(dims,)h(comm)p 1297 409 V 11 w(new\))j FD(can)f(b)q(e)i(implemen)o(ted)g(b)o(y)75 465 y(a)f(call)i(to)e Fs(MPI)p 338 465 V 15 w(COMM)p 495 465 V 17 w(SPLIT\(com)o(m)m(,)d(colo)o(r,)i(k)o(ey)l(,)i(comm)p 1118 465 V 10 w(new\))p FD(,)g(using)g(a)f(single)i(n)o(um)o(b)q(er)e (enco)q(ding)75 522 y(of)g(the)g(lost)g(dimensions)i(as)e FB(color)g FD(and)g(a)g(single)i(n)o(um)o(b)q(er)f(enco)q(ding)g(of)f (the)h(preserv)o(ed)f(dimensions)75 578 y(as)g FB(key)p FD(.)166 647 y(All)h(other)e(Cartesian)g(top)q(ology)g(functions)h(can) g(b)q(e)g(implemen)o(ted)h(lo)q(cally)l(,)h(using)e(the)f(top)q(ology) 75 703 y(information)h(that)g(is)h(cac)o(hed)f(with)h(the)f(comm)o (unicator.)166 772 y(The)g(corresp)q(onding)i(new)e(function)h(for)f (general)g(graph)g(structures)g(is)75 888 y Fs(MPI)p 160 888 V 16 w(MAP)p 275 888 V 16 w(GRAPH\(comm)p 578 888 V 12 w(old,)g(nno)q(des,)h(index,)g(edges,)g(newrank\))117 978 y Ft(IN)171 b Fs(comm)p 454 978 V 10 w(old)383 b Ft(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g(\(handle\))117 1078 y(IN)171 b Fs(nno)q(des)441 b Ft(n)o(um)o(b)q(er)13 b(of)h(graph)f(no)q(des)i(\(in)o(teger\))117 1178 y(IN)171 b Fs(index)473 b Ft(in)o(teger)30 b(arra)o(y)g(sp)q(ecifying)f(the)i (graph)e(structure,)36 b(see)905 1234 y Fv(MPI)p 982 1234 13 2 v 15 w(MAKE)p 1115 1234 V 14 w(GRAPH)117 1334 y Ft(IN)171 b Fs(edges)468 b Ft(in)o(teger)15 b(arra)o(y)e(sp)q (ecifying)h(the)g(graph)g(structure)117 1434 y(OUT)124 b Fs(newrank)414 b Ft(reordered)11 b(rank)f(of)e(the)i(calling)e(pro)q (cess;)k Fv(MPI)p 1643 1434 V 15 w(UNDEFINED)905 1491 y Ft(if)j(the)i(calling)d(pro)q(cess)k(do)q(es)e(not)g(b)q(elong)g(to)f (graph)h(\(in)o(te-)905 1547 y(ger\))75 1684 y FB(int)23 b(MPI)p 245 1684 15 2 v 17 w(Map)p 334 1684 V 17 w(graph\(MPI)p 567 1684 V 16 w(comm)g(comm)p 798 1684 V 17 w(old,)g(int)h(nnodes,)f (int)g(*index,)g(int)g(*edges,)393 1741 y(int)h(*newrank\))75 1840 y(MPI)p 150 1840 V 17 w(MAP)p 239 1840 V 17 w(GRAPH\(COMM)p 496 1840 V 15 w(OLD,)g(NNODES,)f(INDEX,)g(EDGES,)g(NEWRANK,)f(IERROR\)) 170 1896 y(INTEGER)h(COMM)p 460 1896 V 17 w(OLD,)g(NNODES,)g (INDEX\(*\),)g(EDGES\(*\),)f(NEWRANK,)h(IERROR)166 1995 y FD(The)10 b(function)h Fs(MPI)p 512 1995 14 2 v 16 w(MAKE)p 657 1995 V 16 w(GRAPH\(comm)p 960 1995 V 12 w(old,)f(nno)q(des,)j(index,)f(edges,)g(reo)o(rder,)d(comm)p 1734 1995 V 11 w(graph\))p FD(,)75 2051 y(with)k FB(reorder)23 b(=)h(true)12 b FD(can)h(b)q(e)h(implemen)o(ted)h(b)o(y)e(calling)i Fs(MPI)p 1219 2051 V 15 w(MAP)p 1333 2051 V 17 w(GRAPH\(comm)p 1637 2051 V 11 w(old,)e(nno)q(des,)75 2108 y(index,)g(edges,)g (newrank\))p FD(,)g(then)f(calling)h Fs(MPI)p 854 2108 V 16 w(COMM)p 1012 2108 V 17 w(SPLIT\(com)o(m)m(,)c(colo)o(r,)i(k)o(ey) l(,)h(comm)p 1626 2108 V 11 w(graph\))p FD(,)g(with)75 2164 y FB(color)23 b(=)h(0)17 b FD(if)h FB(newrank)23 b FA(6)p FD(=)h FB(MPI)p 676 2164 15 2 v 17 w(UNDEFINED)p FD(,)16 b FB(color)23 b(=)h(MPI)p 1201 2164 V 17 w(UNDEFINED)16 b FD(otherwise,)i(and)g FB(key)23 b(=)75 2221 y(newrank)p FD(.)166 2290 y(All)f(other)f(graph)g(top)q(ology)g(functions)h(can)g (b)q(e)g(implemen)o(ted)h(lo)q(cally)l(,)h(using)e(the)f(top)q(ology)75 2346 y(information)15 b(that)g(is)h(cac)o(hed)f(with)h(the)f(comm)o (unicator.)166 2498 y Fl(Discussion:)54 b Ft(An)19 b(alternativ)o(e)g (implemen)o(tation)d(strategy)j(is)g(to)g(use)h(functions)f Fv(MPI)p 1648 2498 13 2 v 14 w(MAP)p 1753 2498 V 15 w(CART)75 2554 y Ft(and)14 b Fv(MPI)p 233 2554 V 14 w(MAP)p 338 2554 V 14 w(GRAPH)f Ft(that)h(return)h(a)e(group)h(structure)i(\(i.e.,) c(the)j(new)f(ranks)g(of)f(all)g(pro)q(cesses\),)j(next)e(use)75 2611 y Fv(MPI)p 152 2611 V 14 w(COMM)p 296 2611 V 15 w(MAKE\(\))g Ft(to)f(generate)j(the)e(new)g(comm)o(unicator.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 176 183 176 182 bop 75 -100 a FD(176)882 b Fp(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fq(6.6)59 b(A)20 b(simple)f(example)75 147 y FD(The)12 b(example)h(in)g(\014gure)f(6.1)f (sho)o(ws)g(ho)o(w)h(the)g(grid)g(de\014nition)i(and)e(inquiry)i (functions)f(can)f(b)q(e)h(used)f(in)75 203 y(an)k(application)h (program.)k(A)16 b(partial)g(di\013eren)o(tial)h(equation,)f(for)f (example)h(the)g(P)o(oisson)g(equation,)75 259 y(is)g(to)f(b)q(e)h (solv)o(ed)f(on)h(a)f(rectangular)g(domain.)21 b(First)15 b(the)g(pro)q(cesses)h(organize)f(themselv)o(es)h(in)h(a)e(t)o(w)o(o{) 75 316 y(dimensional)h(structure.)k(Eac)o(h)14 b(pro)q(cess)h(then)f (inquires)j(for)c(the)i(ranks)f(of)g(its)g(neigh)o(b)q(ors)i(in)f(the)f (four)75 372 y(directions.)29 b(The)18 b(n)o(umerical)h(problem)g(is)f (solv)o(ed)h(b)o(y)f(an)f(iterativ)o(e)h(metho)q(d,)h(the)f(details)h (of)e(whic)o(h)75 429 y(are)e(hidden)i(in)f(the)f(subroutine)h FB(relax)p FD(.)166 485 y(In)e(eac)o(h)f(relaxation)g(step)g(eac)o(h)g (pro)q(cess)h(computes)f(new)g(v)m(alues)h(for)f(the)g(solution)h(grid) f(function)75 542 y(at)i(all)i(p)q(oin)o(ts)f(o)o(wned)g(b)o(y)f(the)h (pro)q(cess.)22 b(Then)16 b(the)g(v)m(alues)h(at)e(in)o(terpro)q(cess)i (b)q(oundaries)g(ha)o(v)o(e)e(to)g(b)q(e)75 598 y(exc)o(hanged)d(with)h (neigh)o(b)q(oring)g(pro)q(cesses.)19 b(F)l(or)11 b(example,)i(the)f (exc)o(hange)g(subroutine)h(migh)o(t)f(con)o(tain)75 655 y(a)i(call)h(lik)o(e)g Fs(MPI)p 361 655 14 2 v 15 w(SEND\(...,neigh)p 663 655 V 16 w(rank\(1\),...\))i FD(to)c(send)i(up)q(dated)g(v)m(alues)g(to)e(the)h(left{hand)g(neigh)o (b)q(or)75 711 y(\(i-1,j\).)19 b(Note)c(that)g(there)g(is)h(only)f(lo)q (cal)i(comm)o(unication)f(in)g(this)f(exc)o(hange)h(op)q(eration.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 177 184 177 183 bop 75 -100 a Fp(6.6.)34 b(A)15 b(SIMPLE)h(EXAMPLE)1157 b FD(177)194 163 y FB(integer)23 b(ndims,)g(num)p 627 163 15 2 v 17 w(neigh)194 219 y(logical)g(reorder)194 276 y(parameter)g(\(ndims=2,)g(num)p 747 276 V 16 w(neigh=4,)g (reorder=.true.\))194 332 y(integer)g(comm,)g(comm)p 627 332 V 17 w(cart,)g(dims\(ndims\),)f(neigh)p 1217 332 V 17 w(def\(ndims\))194 388 y(integer)h(neigh)p 508 388 V 17 w(rank\(num)p 717 388 V 16 w(neigh\),)g(own)p 996 388 V 16 w(position\(ndims\),)f(i,)i(j)194 445 y(logical)f (periods\(ndims\))194 501 y(real)p FA(\003)p FB(8)g(u\(0:101,0:101\),)f (f\(0:101,0:101\))194 558 y(data)i(dims)f(/)h(ndims)f FA(\003)g FB(0)h(/)194 614 y(comm)g(=)f(MPI)p 436 614 V 17 w(COMM)p 549 614 V 17 w(WORLD)75 671 y(C)107 b(Set)23 b(process)g(grid)g(size)h(and)f(periodicity)194 727 y(call)h(MPI)p 389 727 V 16 w(MAKDIM\(comm,)f(ndims,)g(dims\))194 784 y(periods\(1\))g(=)h(.TRUE.)194 840 y(periods\(2\))f(=)h(.TRUE.)75 897 y(C)107 b(Create)23 b(a)g(grid)h(structure)e(in)i(WORLD)f(group)g (and)h(inquire)f(about)g(own)g(position)194 953 y(call)h(MPI)p 389 953 V 16 w(MAKE)p 501 953 V 17 w(CART)f(\(comm,)g(ndims,)g(dims,)h (periods,)e(reorder,)h(comm)p 1640 953 V 17 w(cart\))194 1009 y(call)h(MPI)p 389 1009 V 16 w(GET)p 477 1009 V 17 w(CART)f(\(comm)p 733 1009 V 17 w(cart,)g(ndims,)g(dims,)g(periods,) g(own)p 1490 1009 V 17 w(position\))75 1066 y(C)107 b(Look)23 b(up)h(the)f(ranks)g(for)h(the)f(neighbors.)46 b(Own)24 b(process)f(coordinates)f(are)i(\(i,j\).)75 1122 y(C)107 b(Neighbors)22 b(are)i(\(i-1,j\),)f(\(i+1,j\),)f(\(i,j-1\),)h (\(i,j+1\))194 1179 y(i)h(=)g(own)p 365 1179 V 17 w(position\(1\))194 1235 y(j)g(=)g(own)p 365 1235 V 17 w(position\(2\))194 1292 y(neigh)p 317 1292 V 17 w(def\(1\))f(=)h(i-1)194 1348 y(neigh)p 317 1348 V 17 w(def\(2\))f(=)h(j)194 1405 y(call)g(MPI)p 389 1405 V 16 w(CART)p 501 1405 V 17 w(RANK)f(\(comm)p 757 1405 V 17 w(cart,)g(neigh)p 1037 1405 V 17 w(def,)g(neigh)p 1293 1405 V 16 w(rank\(1\)\))194 1461 y(neigh)p 317 1461 V 17 w(def\(1\))g(=)h(i+1)194 1518 y(neigh)p 317 1518 V 17 w(def\(2\))f(=)h(j)194 1574 y(call)g(MPI)p 389 1574 V 16 w(CART)p 501 1574 V 17 w(RANK)f(\(comm)p 757 1574 V 17 w(cart,)g(neigh)p 1037 1574 V 17 w(def,)g(neigh)p 1293 1574 V 16 w(rank\(2\)\))194 1630 y(neigh)p 317 1630 V 17 w(def\(1\))g(=)h(i)194 1687 y(neigh)p 317 1687 V 17 w(def\(2\))f(=)h(j-1)194 1743 y(call)g(MPI)p 389 1743 V 16 w(CART)p 501 1743 V 17 w(RANK)f(\(comm)p 757 1743 V 17 w(cart,)g(neigh)p 1037 1743 V 17 w(def,)g(neigh)p 1293 1743 V 16 w(rank\(3\)\))194 1800 y(neigh)p 317 1800 V 17 w(def\(1\))g(=)h(i)194 1856 y(neigh)p 317 1856 V 17 w(def\(2\))f(=)h(j+1)194 1913 y(call)g(MPI)p 389 1913 V 16 w(CART)p 501 1913 V 17 w(RANK)f(\(comm)p 757 1913 V 17 w(cart,)g(neigh)p 1037 1913 V 17 w(def,)g(neigh)p 1293 1913 V 16 w(rank\(4\)\))75 1969 y(C)107 b(Initialize)22 b(the)i(grid)f(functions)g(and)g(start)g(the)h(iteration)194 2026 y(call)g(init)f(\(u,)g(f\))194 2082 y(do)h(10)f(it=1,100)242 2139 y(call)g(relax)h(\(u,)f(f\))75 2195 y(C)107 b(Exchange)22 b(data)i(with)f(neighbor)g(processes)242 2251 y(call)g(exchange)g(\(u,) h(comm)p 771 2251 V 16 w(cart,)f(neigh)p 1050 2251 V 17 w(rank,)g(num)p 1282 2251 V 17 w(neigh\))75 2308 y(10)71 b(continue)194 2364 y(call)24 b(output)f(\(u\))194 2421 y(end)147 2625 y FD(Figure)15 b(6.1:)k(Set{up)d(of)e(pro)q(cess)i (structure)f(for)f(t)o(w)o(o{dimensional)i(parallel)h(P)o(oisson)e (solv)o(er.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 178 185 178 184 bop 75 356 a Fr(Chapter)34 b(7)75 564 y Fy(MPI)40 b(En)m(vironmen)m(tal)h(Managemen)m(t)75 786 y FD(This)15 b(c)o(hapter)e(discusses)j(routines)e(for)g(getting)f(and,)i(where)f (appropriate,)g(setting)g(v)m(arious)g(parame-)75 843 y(ters)h(that)g(relate)g(to)g(the)g(MPI)h(implemen)o(tation)g(and)g (the)f(execution)i(en)o(vironmen)o(t.)j(\(suc)o(h)c(as)f(error)75 899 y(handling)i(and)f(comm)o(unication)g(bu\013ering\).)21 b(The)16 b(pro)q(cedures)g(for)f(en)o(tering)h(and)g(lea)o(ving)g(the)g (MPI)75 956 y(execution)g(en)o(vironmen)o(t)g(are)f(also)g(dexcrib)q (ed)i(here.)75 1099 y Fq(7.1)59 b(Implementation)17 b(info)n(rmation)75 1202 y Fn(7.1.1)49 b(Environmental)15 b(Inquiries)75 1288 y FD(The)g(follo)o(wing)h(routines)g(allo)o(w)f(to)g(inquire)i(on) e(the)g(execution)i(en)o(vironmen)o(t)e(of)g(an)g(MPI)g(program.)75 1392 y Fs(MPI)p 160 1392 14 2 v 16 w(GET)p 264 1392 V 17 w(V)l(ALID)p 408 1392 V 16 w(T)l(A)o(G)p 510 1392 V 17 w(RANGE)h(\()f(lo)o(w,)f(high)i(\))117 1469 y Ft(OUT)108 b Fs(lo)o(w)523 b Ft(Smallest)12 b(v)n(alid)h(tag)g(v)n(alue)h(\(in)o (teger\))117 1544 y(OUT)108 b Fs(high)507 b Ft(Largest)15 b(v)n(alid)d(tag)i(v)n(alue)f(\(in)o(teger\))75 1669 y FB(int)23 b(MPI)p 245 1669 15 2 v 17 w(Get)p 334 1669 V 17 w(valid)p 471 1669 V 16 w(tag)p 559 1669 V 17 w(range\(int)g (*low,)g(int)h(*high\))75 1755 y(MPI)p 150 1755 V 17 w(GET)p 239 1755 V 17 w(VALID)p 376 1755 V 16 w(TAG)p 464 1755 V 17 w(RANGE\(LOW,)e(HIGH,)h(IERROR\))170 1811 y(INTEGER)g(LOW,)h(HIGH,)f(IERROR)75 1898 y FD(This)16 b(routine)g(gets)f(the)g(range)g(of)g(v)m(alid)i(tags;)e(that)f(is,)i (the)f(user)h(ma)o(y)f(use)h(an)o(y)f(tag)f(v)m(alue)j(from)d FB(low)75 1954 y FD(to)j FB(high)g FD(inclusiv)o(e.)29 b(These)18 b(v)m(alues)g(are)f(guaran)o(teed)g(to)g(b)q(e)h(unc)o (hanging)h(during)f(the)g(execution)g(of)75 2011 y(an)d(MPI)g(program.) -668 b Fm(CHANGE)257 2143 y Fl(Discussion:)17 b Ft(After)e(MPI)g(in)g (Europ)q(e,)g(there)h(w)o(as)f(considerable)h(demand)d(for)i(the)g(t)o (w)o(o)f(fea-)166 2193 y(tures:)134 2268 y(1.)22 b(zero)14 b(is)g(guaran)o(teed)h(to)e(b)q(e)i(v)n(alid)d(\(lo)o(w)h Fj(\024)f Ft(0)f Fj(\024)j Ft(high\).)134 2343 y(2.)22 b(high)10 b(is)g(guaran)o(teed)i(to)e(b)q(e)i(at)e(least)h(127)f(\(giv) o(es)h(a)g(minim)n(um)6 b(range)11 b(that)g(users)h(can)f(coun)o(t)g (on;)g(virtually)189 2392 y(ev)o(ery)o(one)j(exp)q(ects)i(a)e(larger)g (range\).)166 2467 y(An)g(alternate)g(prop)q(osal)g(is)g(to)f (eliminate)f Fb(low)p Ft(,)h(requiring)h(that)g Fb(0)f Ft(b)q(e)i(the)f(minim)n(um)c(v)n(alue.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 179 186 179 185 bop 75 -100 a Fp(7.1.)34 b(IMPLEMENT)l(A)l(TION)16 b(INF)o(ORMA)l(TION)841 b FD(179)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(GET)p 264 45 V 17 w(PROCESSOR)p 540 45 V 18 w(NAME\()16 b(len,)f(name)e(\))117 122 y Ft(IN)155 b Fs(len)533 b Ft(Length)14 b(\(in)g(c)o(haracters\))i(of)d(the)i (space)g(in)e Fb(name)117 197 y Ft(OUT)108 b Fs(name)483 b Ft(A)17 b(unique)g(sp)q(eci\014er)i(for)e(the)g(actual)g(\(as)g(opp)q (osed)h(to)e(vir-)905 254 y(tual\))e(no)q(de.)75 378 y FB(int)23 b(MPI)p 245 378 15 2 v 17 w(Get)p 334 378 V 17 w(processor)p 567 378 V 16 w(name\(int)g(len,)g(char)g(*name\))75 465 y(MPI)p 150 465 V 17 w(GET)p 239 465 V 17 w(PROCESSOR)p 472 465 V 15 w(NAME\(NAME,)g(IERROR\))170 521 y(CHARACTER*\(*\))g(NAME) 170 578 y(INTEGER)g(IERROR)166 664 y FD(The)c(name)g(is)g(a)f(c)o (haracter)g(string)h(for)f(maxim)o(um)h(\015exibilit)o(y)l(.)33 b(F)l(rom)18 b(this)h(v)m(alue)h(it)f(m)o(ust)f(b)q(e)75 721 y(p)q(ossible)23 b(to)d(iden)o(tify)i(a)e(sp)q(eci\014c)j(piece)f (of)e(hardw)o(are;)j(p)q(ossible)f(v)m(alues)g(include)h(\\pro)q (cessor)e(9)f(in)75 777 y(rac)o(k)f(4)h(of)f(mpp.cs.org")g(and)h (\\231")e(\(where)i(231)f(is)h(the)g(actual)g(pro)q(cessor)g(n)o(um)o (b)q(er)g(in)h(the)f(run-)75 834 y(ning)f(homogeneous)f(system\).)29 b(A)18 b(error)g(co)q(de)h(of)e Fv(MPI)p 1062 834 13 2 v 15 w(UNDEFINED)g FD(is)i(allo)o(w)o(ed)g(but)f(strongly)g(dis-)75 890 y(couraged.)31 b(The)19 b(v)m(alue)h(of)e Fv(MPI)p 646 890 V 14 w(MAX)p 752 890 V 15 w(PROCESSOR)p 1008 890 V 13 w(NAME)h FD(is)g(the)g(maxim)o(um)g(length)g(required)h(b)o(y) 75 946 y Fs(MPI)p 160 946 14 2 v 16 w(GET)p 264 946 V 17 w(PROCESSOR)p 540 946 V 18 w(NAME)c FD(\(including,)h(in)f(the)f(C)g (binding,)i(the)e(terminating)h(n)o(ull\).)257 b Fm(CHANGE)166 1003 y FD(This)17 b(routine)f(returns)g(the)g(name)h(of)e(the)i(pro)q (cessor)e(on)i(whic)o(h)g(it)f(w)o(as)f(called)j(at)d(the)i(momen)o(t) 75 1059 y(of)g(the)g(call.)27 b(This)17 b(is)h(to)f(allo)o(w)g(MPI)g (implemen)o(tation)h(that)f(do)g(pro)q(cess)g(migration)g(to)g(return)g (the)75 1116 y(curren)o(t)j(pro)q(cessor.)34 b(Note)20 b(that)f(nothing)i(in)g(MPI)f Fo(r)n(e)n(quir)n(es)f FD(or)g(de\014nes)i(pro)q(cess)f(migration;)j(this)75 1172 y(de\014nition)17 b(of)e Fs(MPI)p 414 1172 V 16 w(GET)p 518 1172 V 16 w(PROCESSOR)p 793 1172 V 19 w(NAME)h FD(simply)g(allo)o(ws)f(suc)o(h)h(an)f(implemen)o(tation.)75 1276 y Fs(MPI)p 160 1276 V 16 w(GET)p 264 1276 V 17 w(HOST\()g(comm)m (,)d(rank\))117 1353 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(\(handle\))117 1428 y(OUT)108 b Fs(rank)504 b Ft(rank)21 b(of)f Fb(HOST)g Ft(in)h(group)f(of)g Fb(comm)p Ft(;)j Fv(MPI)p 1604 1428 13 2 v 15 w(PROCNULL)d Ft(if)905 1485 y(there)15 b(is)f(no)g Fb(HOST)f Ft(\(in)o(teger\))75 1609 y FB(int)23 b(MPI)p 245 1609 15 2 v 17 w(Get)p 334 1609 V 17 w(host\(MPI)p 543 1609 V 16 w(Comm)g(comm,)h(int)f(*rank\))75 1696 y(MPI)p 150 1696 V 17 w(GET)p 239 1696 V 17 w(HOST\(COMM,)f(RANK,) h(IERROR\))170 1752 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)166 1839 y FD(This)15 b(routine)h(gets)e(the)h(rank)g(of)g(the)g FB(HOST)f FD(pro)q(cess)h(in)h(the)f(group)g(asso)q(ciated)g(with)g (comm)o(uni-)75 1895 y(cator)e FB(comm)p FD(,)g(if)h(there)f(is)i(suc)o (h.)k(MPI)14 b(do)q(es)g(not)f(sp)q(ecify)i(what)e(it)h(means)f(for)g (a)h(pro)q(cess)f(to)g(b)q(e)i(a)e FB(HOST)p FD(,)75 1952 y(nor)i(do)q(es)g(it)h(requires)g(that)e(a)h FB(HOST)g FD(exists.)75 2072 y Fs(Alternative)1970 2101 y Fm(CHANGE)75 2157 y FD(A)10 b(set)g(of)g(prede\014ned)i(in)o(teger-v)m(alued)g (attributes)e(is)g(asso)q(ciated)h(with)f(the)g(comm)o(unicator)g Fv(MPI)p 1732 2157 13 2 v 14 w(COMM)p 1876 2157 V 15 w(W)o(ORLD)75 2214 y FD(at)h(initialization)k(time.)k(These)13 b(de\014ne)g(the)f(execution)h(en)o(vironmen)o(t)f(of)f(MPI)h (programs.)18 b(The)12 b(v)m(alues)75 2270 y(of)i(these)h(attributes)g (can)g(b)q(e)g(retriev)o(ed)g(b)o(y)g(calling)h Fs(MPI)p 1076 2270 14 2 v 16 w(GET)p 1180 2270 V 17 w(A)l(TTR)p FD(.)f(These)g(attributes)g(cannot)f(b)q(e)75 2327 y(deleted.)21 b(The)16 b(list)g(of)e(prede\014ned)j(attributes)e(k)o(eys)g(include)75 2421 y Fv(MPI)p 152 2421 13 2 v 14 w(T)m(A)o(G)p 246 2421 V 14 w(UB)22 b FD(Upp)q(er)16 b(b)q(ound)g(for)f(tag)f(v)m(alue)75 2514 y Fv(MPI)p 152 2514 V 14 w(T)m(A)o(G)p 246 2514 V 14 w(LB)23 b FD(Lo)o(w)o(er)15 b(b)q(ound)h(for)e(tag)h(v)m(alue)h (\(if)g(it)f(is)h(not)e(set)h(to)g(b)q(e)h(zero\))75 2608 y Fv(MPI)p 152 2608 V 14 w(HOST)23 b FD(Host)15 b(pro)q(cess)g(rank,)g(if)h(suc)o(h)f(exists,)g Fv(MPI)p 1041 2608 V 15 w(UNDEFINED)p FD(,)e(otherwise.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 180 187 180 186 bop 75 -100 a FD(180)532 b Fp(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fv(MPI)p 152 45 13 2 v 14 w(SERVER)22 b FD(Serv)o(er)13 b(rank,)f(or)h(0/1)f(\015ag,)g(indicating)j(whether)e(a)g(serv)o(er)f (exists)i(\(dep)q(ending)g(on)f(\014nal)189 102 y(decision)k(on)e(in)o (tercomm)o(unicator)g(name)g(serv)o(ers\))75 195 y Fv(MPI)p 152 195 V 14 w(IO)22 b FD(rank)14 b(of)f(a)g(no)q(de)h(that)f(has)h (regular)g(I/O)g(facilities)i(\(p)q(ossibly)f(m)o(yrank\).)j(No)q(des)c (in)h(the)e(same)189 252 y(comm)o(unicator)h(ma)o(y)h(return)g (di\013eren)o(t)h(v)m(alues)g(for)e(this)i(attribute.)75 346 y Fv(MPI)p 152 346 V 14 w(ERROR)22 b FD(error)14 b(handler)j(t)o(yp)q(e:)j(fatal,)14 b(no)q(error,)g(or)h(user)g (de\014ned.)166 439 y(V)l(endors)g(ma)o(y)e(add)i(implemen)o(tation)h (sp)q(eci\014c)g(attributes)e(\(suc)o(h)h(as)f(no)q(de)h(n)o(um)o(b)q (er,)f(real)h(mem-)75 496 y(ory)g(size,)g(virtual)h(memory)f(size,)h (etc.\))189 602 y Fo(R)n(ationale.)40 b FD(\(1\))14 b(Wh)o(y)h(ha)o(v)o (e)g(a)g(slew)h(of)f(inquiry)i(functions)f(if)f(w)o(e)g(already)h(ha)o (v)o(e)f(an)g(attribute)189 659 y(mec)o(hanism?)189 734 y(\(2\))g(The)h(op)q(en)h(ended)g(form)e(of)h(this)g(mec)o(hanism)h (allo)o(ws)f(to)g(easily)h(add)f(attributes.)22 b(\()p Fo(End)17 b(of)189 790 y(r)n(ationale.)p FD(\))166 896 y(W)l(e)e(ma)o(y)g(still)h(w)o(an)o(t)e(the)i Fs(MPI)p 706 896 14 2 v 16 w(GET)p 810 896 V 16 w(PROCESSOR)p 1085 896 V 19 w(NAME)g FD(call,)g(whic)o(h)g(returns)f(a)g(string.)75 1018 y Fn(7.1.2)49 b(Bu\013ering)75 1104 y FD(While)15 b(it)g(is)f(p)q(ossible)i(to)d(design)i(algorithms)f(that)f(do)h(not)f (require)i(that)e(an)o(y)h(bu\013ering)h(b)q(e)f(pro)o(vided)75 1160 y(for)e(sending)i(and)e(receiving)i(\\blo)q(c)o(king")g(messages,) e(man)o(y)g(practitioners)h(exp)q(ect)g(that)f(the)g(message-)75 1217 y(passing)k(system)e(will)j(pro)o(vide)f(some)f(bu\013ering.)21 b(That)15 b(is,)g(there)g(is)h(often)f(an)g(exp)q(ectation)h(that)f (the)75 1273 y(program)f(fragmen)o(t,)g(executed)i(b)o(y)f(all)h (tasks,)75 1367 y FB(MPI_send\()23 b(buf,)g(size,)g(datatype,)g (destination,)f(tag,)h(comm)h(\))75 1424 y(MPI_recv\()f(buf,)g(size,)g (datatype,)g(destination,)f(tag,)h(comm)h(\))75 1517 y FD(will)16 b(execute)f(for)e(messages)h(whose)g Fv(size)h FD(is)g(not)e(to)q(o)h(large.)20 b(In)15 b(fact,)e(this)i(co)q(de)g(ma) o(y)e(deadlo)q(c)o(k)i(at)f(the)75 1574 y(\014rst)f FB(MPI)p 242 1574 15 2 v 16 w(send)p FD(.)19 b(The)13 b(purp)q(ose)h(of)f(this)g (section)h(is)f(to)g(giv)o(e)g(the)g(user)g(a)g(w)o(a)o(y)f(to)g (determine)i(what)f(\\to)q(o)75 1630 y(large")i(is)h(and)f(to)g(help)h (the)f(implemen)o(tation)i(pro)o(vide)f(enough)f(bu\013ering)h(for)f (the)g(user's)g(need.)166 1687 y(Note)f(that)g(nothing)h(in)g(this)g (section)g Fo(r)n(e)n(quir)n(es)i FD(an)e(implemen)o(tation)h(to)e(pro) o(vide)h(an)o(y)f(bu\013ering;)75 1743 y(that)f(is,)h(an)g(implemen)o (tation)h(ma)o(y)f(de\014ne)h(the)f(ab)q(o)o(v)o(e)f(program)g(fragmen) o(t)g(to)g(b)q(e)h(erroneous)g(for)f(an)o(y)75 1800 y(p)q(ositiv)o(e)j (v)m(alue)h(of)d Fv(size)p FD(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 181 188 181 187 bop 75 -100 a Fp(7.1.)34 b(IMPLEMENT)l(A)l(TION)16 b(INF)o(ORMA)l(TION)841 b FD(181)75 45 y Fs(General)16 b(bu\013ering)f(pa)o(rameters)75 178 y(MPI)p 160 178 14 2 v 16 w(GET)p 264 178 V 17 w(BUFFER)p 450 178 V 17 w(P)l(ARAMS\()i(numsend,)e(sizesend,)j(numrecv,)c(sizerecv,)i (totalnumsend,)f(totalsize-)75 235 y(send,)h(totalnumrecv,)d (totalsizerecv\))117 312 y Ft(OUT)108 b Fs(numsend)419 b Ft(Bound)10 b(on)g(the)g(n)o(um)o(b)q(er)f(of)g(p)q(ending)h (messages)g(with)g(source)905 368 y(at)k(calling)e(pro)q(cess)k(and)e (same)f(destination)117 442 y(OUT)108 b Fs(sizesend)436 b Ft(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 498 y(with)f(source)h(at)f(calling)e(pro)q(cess) k(and)e(same)f(destination)117 571 y(OUT)108 b Fs(numrecv)425 b Ft(Bound)16 b(on)e(the)i(n)o(um)o(b)q(er)e(of)g(p)q(ending)i (messages)f(with)f(des-)905 628 y(tination)f(at)h(calling)e(pro)q(cess) k(and)e(same)f(source)117 701 y(OUT)108 b Fs(sizerecv)442 b Ft(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 758 y(with)f(destination)f(at)h(calling)f(pro)q (cess)i(and)f(same)f(source)117 831 y(OUT)108 b Fs(totalnumsend)331 b Ft(Bound)10 b(on)g(the)g(n)o(um)o(b)q(er)f(of)g(p)q(ending)h (messages)g(with)g(source)905 888 y(at)k(the)g(calling)f(pro)q(cess)j (and)d(an)o(y)h(destination)117 961 y(OUT)108 b Fs(totalsizesend)348 b Ft(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 1017 y(with)d(source)i(at)f(the)g(calling)e(pro) q(cess)k(and)d(an)o(y)g(destination)117 1091 y(OUT)108 b Fs(totalnumrecv)337 b Ft(Bound)16 b(on)f(the)h(n)o(um)o(b)q(er)f(of)g (p)q(ending)g(messages)h(from)d(an)o(y)905 1147 y(source)i(with)f(the)g (calling)f(pro)q(cess)j(as)e(destination)117 1221 y(OUT)108 b Fs(totalsizerecv)354 b Ft(Bound)15 b(on)f(the)h(total)f(size)h(in)f (b)o(ytes)h(of)f(p)q(ending)h(messages)905 1277 y(from)9 b(an)o(y)h(source)i(with)e(destination)g(at)g(the)h(calling)e(pro)q (cess)75 1402 y FB(int)23 b(MPI)p 245 1402 15 2 v 17 w(Get)p 334 1402 V 17 w(buffer)p 495 1402 V 16 w(params\(int)g (*numsend,)f(int)i(*sizesend,)e(int)i(*numrecv,)393 1458 y(int)g(*sizerecv,)e(int)i(*totalnumsend,)e(int)h(*totalsizesend,)393 1514 y(int)h(*totalnumrecv,)e(int)h(*totalsizerecv\))75 1601 y(MPI)p 150 1601 V 17 w(GET)p 239 1601 V 17 w(BUFFER)p 400 1601 V 16 w(PARAMS\(NUMSEND,)f(SIZESEND,)g(NUMRECV,)h(SIZERECV,)g (TOTALNUMSEDND,)393 1657 y(TOTALSIZESEN,)f(TOTALNUMRECV,)g (TOTALSIZERECV,)g(IERROR\))170 1714 y(INTEGER)h(NUMSEND,)g(SIZESEND,)g (NUMRECV,)g(SIZERECV,)f(TOTALNUMSEDND,)170 1770 y(TOTALSIZESEN,)h (TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)166 1857 y FD(A)e(program)f (where)h(at)g(an)o(y)g(p)q(oin)o(t)g(in)h(time)g(no)f(pro)q(cess)g(has) g(more)g(then)g Fs(numsend)g FD(p)q(ending)75 1913 y(messages)13 b(sen)o(t)g(b)o(y)g(it)h(to)f(the)h(same)f(destination,)h(no)f(more)g (than)h Fs(sizesend)h FD(b)o(ytes)e(of)g(data)g(in)h(p)q(ending)75 1970 y(messages)h(sen)o(t)g(b)o(y)h(it)f(to)g(the)h(same)f (destination,)h(no)f(more)g(than)h Fs(totalnumsend)f FD(p)q(ending)i(messages)75 2026 y(sen)o(t)d(b)o(y)h(it)g(to)e(an)o(y)i (destination,)g(no)f(more)g(than)h Fs(totalsizesend)h FD(b)o(ytes)f(of)f(data)g(in)h(p)q(ending)h(messages)75 2083 y(sen)o(t)g(b)o(y)g(it)h(to)e(an)o(y)h(destination,)h(no)f(more)g (than)g Fs(numrecv)f FD(p)q(ending)j(messages)d(sen)o(t)h(to)g(it)g (from)g(the)75 2139 y(same)k(source,)h(no)e(more)h(than)g Fs(sizerecv)g FD(b)o(ytes)g(in)g(p)q(ending)i(messages)e(sen)o(t)f(to)h (it)g(from)f(the)h(same)75 2195 y(source,)g(no)f(more)g(than)g Fs(totalnumrecv)e FD(p)q(ending)k(messages)e(sen)o(t)g(to)g(it)g(from)g (an)o(y)g(source,)h(and)f(no)75 2252 y(more)c(than)g Fs(totalsizerecv)h FD(b)o(ytes)f(in)h(p)q(ending)h(messages)e(sen)o(t)g (to)f(it)i(from)e(an)o(y)h(source)h(is)f(guaran)o(teed)75 2308 y(not)g(to)f(deadlo)q(c)o(k)i(b)q(ecause)h(of)d(the)i(lac)o(k)f (of)g(bu\013er)g(space)h(for)e(p)q(ending)j(messages.)166 2365 y(Examples:)166 2421 y(1.)30 b(Supp)q(ose)20 b(that)e(p)q(ending)i (messages)f(are)f(alw)o(a)o(ys)g(bu\013ered)h(at)f(the)h(sender,)h(all) g(bu\013ers)e(are)75 2478 y(shared,)d(total)f(bu\013er)h(size)h(at)f (the)g(pro)q(cess)g(is)h FB(B)p FD(,)e(eac)o(h)h(p)q(ending)i(message)e (tak)o(es)f FB(h)h FD(b)o(ytes)g(of)g(storage,)75 2534 y(in)j(addition)g(to)e(the)h(storage)f(for)g(its)h(data,)g(and)g(no)g (storage)e(is)j(lost)f(due)g(to)g(fragmen)o(tation.)23 b(Then)75 2591 y(p)q(ossible)14 b(return)e(v)m(alues)h(for)e Fs(MPI)p 662 2591 14 2 v 16 w(GET)p 766 2591 V 17 w(BUFFER)p 952 2591 V 17 w(P)l(ARAMS)i FD(are)f FB(numsend)23 b(=)g(totalnumsend)g (=)g(k,)75 2647 y(sizesend)g(=)g(totalsizesend)g(=)g(B-kh)p FD(,)11 b(and)f FB(numrecv)23 b(=)g(totalnumrecv)g(=)g(sizerecv)g(=)h (totalsizerecv)75 2704 y(=)g(INT)p 198 2704 15 2 v 16 w(MAX)p FD(,)15 b(for)f(some)h FB(0)e Fw(<)g FB(k)f Fw(<)h FB(B)p Fw(=)p FB(h)p FD(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 182 189 182 188 bop 75 -100 a FD(182)532 b Fp(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)166 45 y FD(2.)36 b(Supp)q(ose)22 b(that)e(messages)g(are)h(alw)o(a)o(ys)f (bu\013ered)h(at)f(the)h(receiv)o(er,)h(and)f(that)f(a)g(separate)75 102 y(bu\013er)15 b(with)h FB(C)e FD(b)o(ytes)h(is)h(allo)q(cated)g (for)e(eac)o(h)i(pair)f(of)g(comm)o(unicating)h(pro)q(cesses;)f(en)o (tries)g(for)g(p)q(end-)75 158 y(ing)j(messages)g(are)f(k)o(ept)h(in)g (a)g(seeparate)f(table,)h(with)h(ro)q(om)e(for)g(total)g(of)g FB(m)h FD(p)q(ending)h(messages)f(at)75 214 y(eac)o(h)13 b(receiving)i(pro)q(cess;)e(the)g(total)g(n)o(um)o(b)q(er)g(of)g(pro)q (cesses)g(is)h FB(p)p FD(,)e(and)i(no)f(storage)e(is)j(lost)f(due)g(to) g(frag-)75 271 y(men)o(tation.)23 b(Then)16 b(p)q(ossible)i(return)e(v) m(alues)i(for)d Fs(MPI)p 1035 271 14 2 v 16 w(GET)p 1139 271 V 17 w(BUFFER)p 1325 271 V 17 w(P)l(ARAMS)j FD(are)e FB(sendcount)22 b(=)75 327 y(totalsendcount)g(=)i(sendsize)e(=)i (totalsendsize)e(=)i(INT)p 1200 327 15 2 v 17 w(MAX,)f(recvcount)g(=)g (totalrecvcount)75 384 y(=)h(m,)f(recsize)g(=)h(C)15 b FD(and)g FB(totalrecvsize)22 b(=)i(\(p-1\))p FA(\002)p FB(C)p FD(.)166 440 y(The)19 b(call)g(ma)o(y)f(return)g(the)g(v)m(alue) i Fv(MPI)p 872 440 13 2 v 14 w(UNDEFINED)e FD(in)h(an)o(y)f(argumen)o (t.)29 b(If)18 b(this)h(is)g(the)f(case,)75 497 y(then)e(the)f(MPI)g (implemen)o(tation)h(mak)o(es)f(no)g(guaran)o(tee)g(ab)q(out)g (bu\013ering.)166 553 y(Some)k(implemen)o(tations)h(ma)o(y)f(b)q(e)g (able)h(to)f(adjust)f(their)i(bu\013ering.)32 b(In)20 b(addition,)h(since)f(the)75 610 y(parameters)13 b(returned)i(b)o(y)f Fs(MPI)p 640 610 14 2 v 16 w(GET)p 744 610 V 17 w(BUFFER)p 930 610 V 17 w(P)l(ARAMS)h FD(represen)o(t)f(a)g(guaran)o(tee)g(ab)q (out)g(a)g(partic-)75 666 y(ular)g(c)o(hoice)h(of)e(n)o(um)o(b)q(er)h (of)g(bu\013ers,)f(it)i(ma)o(y)e(not)g(guaran)o(tee)g(bu\013ering)i (that)e(it)h(can)g(in)g(fact)g(p)q(erform.)75 723 y(Th)o(us,)k(in)h (example)g(1)f(ab)q(o)o(v)o(e,)g(the)g(implemen)o(tation)h(pic)o(ks)g (an)f(arbitrary)f(v)m(alue)i(for)f FB(k)p FD(,)g(trading)g(o\013)75 779 y(n)o(um)o(b)q(er)h(of)f(messages)h(for)f(total)g(message)g(size,)j (rather)d(than)g(letting)i(the)f(user)g(mak)o(e)f(this)h(trade-)75 835 y(o\013.)k(In)18 b(order)e(to)g(allo)o(w)h(the)g(MPI)f(user)h(to)f (inquire)j(ab)q(out)d(a)g(sp)q(eci\014c)j(bu\013ering)e(need,)h(the)e (routine)75 892 y Fs(MPI)p 160 892 V 16 w(SUGGEST)p 375 892 V 18 w(BUFFER)p 562 892 V 17 w(P)l(ARAMS)g FD(is)g(pro)o(vided.)75 996 y Fs(MPI)p 160 996 V 16 w(SUGGEST)p 375 996 V 18 w(BUFFER)p 562 996 V 17 w(P)l(ARAMS\()h(numsend,)g(sizesend,)h (numrecv,)d(sizerecv,)i(totalnumsend,)g(to-)75 1052 y(talsizesend,)g (totalnumrecv,)c(totalsizerecv,)i(\015ag\))117 1129 y Ft(IN)155 b Fs(numsend)419 b Ft(Bound)13 b(on)f(n)o(um)o(b)q(er)g(of)f (p)q(ending)i(messages)f(with)g(source)i(at)905 1186 y(calling)f(pro)q(cess)i(and)f(same)f(destination)117 1261 y(IN)155 b Fs(sizesend)436 b Ft(Bound)12 b(on)g(total)f(size)i(in) e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1317 y(source)j(at)f(calling)f(pro)q(cess)i(and)f(same)f(destination)117 1392 y(IN)155 b Fs(numrecv)425 b Ft(Bound)16 b(on)e(the)i(n)o(um)o(b)q (er)e(of)g(p)q(ending)i(messages)f(with)f(des-)905 1449 y(tination)f(at)h(calling)e(pro)q(cess)k(and)e(same)f(source)117 1524 y(IN)155 b Fs(sizerecv)442 b Ft(Bound)12 b(on)g(total)f(size)i(in) e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1581 y(destination)i(at)g(calling)e(pro)q(cess)k(and)e(same)f(source)117 1656 y(IN)155 b Fs(totalnumsend)331 b Ft(Bound)10 b(on)g(the)g(n)o(um)o (b)q(er)f(of)g(p)q(ending)h(messages)g(with)g(source)905 1712 y(at)k(the)g(calling)f(pro)q(cess)j(and)d(an)o(y)h(destination)117 1787 y(IN)155 b Fs(totalsizesend)348 b Ft(Bound)12 b(on)g(total)f(size) i(in)e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1844 y(source)j(at)f(the)h(calling)d(pro)q(cess)k(and)e(an)o(y)f (destination)117 1919 y(IN)155 b Fs(totalnumrecv)337 b Ft(Bound)16 b(on)f(the)h(n)o(um)o(b)q(er)f(of)g(p)q(ending)g (messages)h(from)d(an)o(y)905 1975 y(source)i(with)f(the)g(calling)f (pro)q(cess)j(as)e(destination)117 2050 y(IN)155 b Fs(totalsizerecv)354 b Ft(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 2107 y(from)9 b(an)o(y)h(source)i(with)e (destination)g(at)g(the)h(calling)e(pro)q(cess)117 2182 y(OUT)108 b Fs(\015ag)518 b Fv(TRUE)17 b Ft(if)f(the)h(suggestion)h(w)o (as)f(accepted,)i Fv(F)m(ALSE)p Ft(,)c(other-)905 2238 y(wise)75 2363 y FB(int)23 b(MPI)p 245 2363 15 2 v 17 w(Suggest)p 430 2363 V 16 w(buffer)p 590 2363 V 17 w(params\(int)f (numsend,)h(int)h(sizesend,)e(int)i(numrecv,)393 2419 y(int)g(sizerecv,)e(int)i(totalnumsend,)e(int)h(totalsizesen,)393 2476 y(int)h(totalnumrecv,)e(int)h(totalsizerecv,)f(int)i(*flag\))75 2562 y(MPI)p 150 2562 V 17 w(SUGGEST)p 335 2562 V 16 w(BUFFER)p 495 2562 V 16 w(PARAMS\(NUMSEND,)e(SIZESEND,)h(NUMRECV,)g (SIZERECV,)393 2619 y(TOTALNUMSEND,)f(TOTALSIZESEN,)g(TOTALNUMRECV,)g (TOTALSIZERECV,)g(FLAG,)393 2675 y(IERROR\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 183 190 183 189 bop 75 -100 a Fp(7.1.)34 b(IMPLEMENT)l(A)l(TION)16 b(INF)o(ORMA)l(TION)841 b FD(183)170 45 y FB(INTEGER)23 b(NUMSEND,)g(SIZESEND,)g(NUMRECV,)g(SIZERECV,)f(TOTALNUMSEND,)170 102 y(TOTALSIZESEN,)h(TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)170 158 y(LOGICAL)h(FLAG)166 244 y FD(If)17 b(a)f(v)m(alue)h(of)f(true)h (is)g(returned)f(in)i Fs(\015ag)e FD(then)h(it)f(is)h(as)f(if)h(the)g (v)m(alues)g(pro)o(vided)h(b)o(y)e(the)h(user)f(in)75 301 y Fs(MPI)p 160 301 14 2 v 16 w(SUGGEST)p 375 301 V 18 w(BUFFER)p 562 301 V 17 w(P)l(ARAMS)f FD(w)o(ere)f(returned)h(b)o (y)f(a)g(call)h(to)e Fs(MPI)p 1378 301 V 16 w(GET)p 1482 301 V 17 w(BUFFER)p 1668 301 V 17 w(P)l(ARAMS)p FD(:)75 357 y(the)i(user)h(is)g(guaran)o(teed)e(that)h(no)g(deadlo)q(c)o(k)i (will)g(o)q(ccur)e(due)h(to)f(lac)o(k)g(of)g(bu\013ers,)g(pro)o(vided)h (it)g(nev)o(er)75 414 y(exceeds)11 b(the)g(limitations)g(on)g(n)o(um)o (b)q(er)f(and)h(size)g(of)f(p)q(ending)i(messages)e(implied)i(b)o(y)f (these)f(parameters.)166 470 y(This)19 b(routines)g(allo)o(ws)f(the)h (user)f(to)g(tell)i(the)e(MPI)g(implemen)o(tation)i(that)e(a)g(certain) h(amoun)o(t)75 527 y(of)c(bu\013ering)i(w)o(ould)f(b)q(e)h(desireable,) g(The)f(MPI)g(implemen)o(tation)h(is)g(not)e(required)i(to)e(pro)o (vide)i(suc)o(h)75 583 y(bu\013ering.)166 640 y(It)e(is)h(exp)q(ected)g (that)f(a)g(common)f(use)i(of)f(these)g(routines)h(will)h(b)q(e)75 733 y FB(MPI_Get_buffer_params\()k(&sendnum,)h(&sendsize,)h(...)g(\);) 75 790 y(if)h(\(sendsize)e(<)i(buffer_needed\))e({)170 846 y(fprintf\()h(stderr,)g("Program)g(requires)g(more)g(buffering)g (than)g(is)h(available\\n")e(\);)170 903 y(MPI_Abort\()h(1)h(\);)170 959 y(})75 1016 y(else)170 1072 y(exploit)f(buffering)75 1166 y FD(This)d(allo)o(ws)f(a)g(program)f(to)h(fail)h(rather)f(than)g (deadlo)q(c)o(k.)33 b(It)19 b(is)h(also)f(p)q(ossible)i(for)e(a)f (program)h(to)75 1222 y(switc)o(h)c(to)g(a)g(di\013eren)o(t)g (algorithm)g(that)g(requires)h(less)g(or)e(no)i(bu\013ering.)166 1279 y(Another)j(use)g(is)h(to)f(inform)g(the)g(MPI)g(implemen)o (tation)i(of)d(the)i(bu\013er)f(space)g(desires)h(of)f(the)75 1335 y(application:)75 1429 y FB(MPI_Suggest_buffer_params\()h(1,)k (buffer_needed,)e(1,)h(buffer_needed,)f(0,)i(0,)g(0,)1578 1486 y(0,)g(&flag)f(\);)75 1542 y(if)h(\(flag\))f({)170 1599 y(exploit)g(buffering)170 1655 y(})75 1711 y(else)g({)170 1768 y(fprintf\()g(stderr,)g("Program)g(requires)g(more)g(buffering)g (than)g(is)h(available\\n")e(\);)170 1824 y(MPI_abort\()h(1)h(\);)170 1881 y(})166 2057 y Fl(Discussion:)17 b Ft(An)e(implemen)o(tation)c(is) k(free)h(to)e(accept)i(suggestions)g(only)e(after)h Fv(MPI)p 1575 2057 13 2 v 14 w(INIT)g Ft(and)f(b)q(efore)75 2114 y(an)o(y)9 b(MPI)g(call)g(that)g(p)q(erforms)g(an)o(y)g(comm)o (unicatio)o(n.)14 b(It)9 b(is)g(alw)o(a)o(ys)f(correct)j(to)f(call)e Fv(MPI)p 1477 2114 V 14 w(SUGGEST)p 1675 2114 V 14 w(BUFFER)p 1846 2114 V 13 w(P)m(ARAMS)75 2170 y Ft(after)14 b Fv(MPI)p 250 2170 V 14 w(INIT)p Ft(.)166 2385 y Fl(Discussion:)34 b Ft(These)15 b(functions)e(are)h(complex)f(\(eigh)o(t)g(parameters\),) h(and)f(still)g(do)g(not)h(pro)o(vide)g(an)f(ac-)75 2435 y(curate)g(information)8 b(on)k(an)f(implemen)o(tatio)o(n)e(\(do)i(non) o(blo)q(c)o(king)g(messages)g(share)i(bu\013er)f(space)h(with)e(blo)q (c)o(king)75 2485 y(ones?)18 b(Ho)o(w)13 b(man)o(y)e(messages)j(are)f (generated)i(b)o(y)d(collectiv)o(e)i(comm)o(uni)o(cations?)h(Or)f(comm) o(unicator)c(generat-)75 2535 y(ing)i(calls?\))18 b(Are)c(these)h (functions)e(really)g(useful,)f(as)i(a)f(\\no)q(op")f(implemen)o (tation)e(is)j(legitimate?)i(esp)q(ecially)f(if)75 2585 y(w)o(e)g(ha)o(v)o(e)g(pac)o(king/unpac)o(king?)166 2634 y(The)e(generic)g(attribute)h(mec)o(hanism)c(can)j(b)q(e)g(used)g(for)g (these)h(comm)o(uni)o(cator)c(parameters,)j(if)f(w)o(e)h(wish)f(to)75 2684 y(do)h(so.)18 b(I.e.,)12 b(add)g(eigh)o(t)g(bu\013ering)h (attributes)g(to)g(eac)o(h)g(comm)o(uni)o(cator,)d(that)j(can)f(b)q(e)h (queried)g(and)g(sometimes)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 184 191 184 190 bop 75 -100 a FD(184)532 b Fp(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Ft(mo)q(di\014ed.)i(Problems)c(with)g(this)h(approac)o(h)f(are)h (that)g(the)g(eigh)o(t-parameter)f(mo)q(del)f(allo)o(ws)g(the)i(v)n (alues)f(to)g(b)q(e)75 95 y(coupled)g(\(messages)g(traded)h(against)e (bu\013er)i(space\).)166 145 y(User)j(commen)o(ts)e(at)h(MPI)h(in)f (Europ)q(e)h(w)o(ere)g(extremely)f(negativ)o(e)g(ab)q(out)g(MPI)h(not)f (mandating)e(large)75 195 y(amoun)o(ts)d(of)i(bu\013ering.)166 251 y(Man)o(y)f(users)j(also)d(read)h(this)g(as)g(mandating)e(no)h (bu\013ering;)h(this)g(caused)h(a)f(great)g(deal)g(of)f(confusion.)75 454 y Fs(Application)k(p)o(rovided)e(bu\013ering)75 540 y FD(An)k(application)h(program)e(can)h(optionally)h(pro)o(vide)f (space)g(for)f(MPI's)g(use)i(in)f(bu\013ering)g(message)75 596 y(data)c(via)g(the)g(follo)o(wing)h(call:)75 700 y Fs(MPI)p 160 700 14 2 v 16 w(USER)p 288 700 V 17 w(SPECIFIES)p 519 700 V 17 w(BUFFER\()f(comm)m(,)d(bu\013er,)j(size\))117 777 y Ft(IN)155 b Fs(comm)466 b Ft(is)16 b(a)f(comm)o(unicator)e(with)j (whic)o(h)f(the)i(bu\013er)f(is)g(to)g(b)q(e)g(as-)905 833 y(so)q(ciated)f(\(handle\))117 908 y(IN)155 b Fs(bu\013er)478 b Ft(initial)12 b(bu\013er)j(address)g(\(c)o(hoice\))117 984 y(IN)155 b Fs(size)519 b Ft(bu\013er)15 b(size,)f(in)g(b)o(ytes)g (\(in)o(teger\))75 1108 y FB(int)23 b(MPI)p 245 1108 15 2 v 17 w(User)p 358 1108 V 17 w(specifies)p 591 1108 V 16 w(buffer\(MPI)p 847 1108 V 16 w(Comm)g(comm,)g(void*)g(buffer,)g (int)h(size\))75 1195 y(MPI)p 150 1195 V 17 w(USER)p 263 1195 V 16 w(SPECIFIES)p 495 1195 V 16 w(BUFFER\(COMM,)f(BUFFER,)f (SIZE,)i(IERROR\))170 1251 y()f(BUFFER\(*\))170 1307 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)166 1394 y FD(If)17 b Fs(MPI)p 298 1394 14 2 v 16 w(USER)p 426 1394 V 17 w(SPECIFIES)p 657 1394 V 17 w(BUFFER)g FD(is)h(called)g(and)g(the)f(v)m (alue)h(returned)f(in)h FB(flag)e FD(is)i FB(MPI)p 1801 1394 15 2 v 17 w(OK)p FD(,)75 1450 y(then)j(for)g(subsequen)o(t)g (regular)g(sends)h(using)f(the)g(sp)q(eci\014ed)i(comm)o(unicator,)f (MPI)f(m)o(ust)f(pro)o(vide)75 1507 y(as)e(m)o(uc)o(h)g(safet)o(y)f Fo(as)i(if)27 b FD(outgoing)18 b(message)f(data)h(w)o(ere)g(bu\013ered) g(b)o(y)g(the)g(sending)h(pro)q(cess,)g(in)g(the)75 1563 y(sp)q(eci\014ed)k(bu\013er)e(space,)h(using)f(a)f(circular)i(con)o (tiguous-space)g(allo)q(cation)f(p)q(olicy)l(.)39 b(The)21 b(bu\013er)f(is)75 1620 y(managed)15 b(b)o(y)g(MPI.)166 1676 y(Access)k(to)f(the)h(pro)o(vided)h(bu\013er)f(space)g(is)g(not)f (inherited)j(b)o(y)e(comm)o(unicators)f(deriv)o(ed)i(from)75 1733 y(the)15 b(initially)j(sp)q(eci\014ed)g(one.)166 1865 y Fl(Implemen)o(tati)o(on)12 b(note:)37 b Ft(One)15 b(approac)o(h)f(is)g(for)g(MPI)g(to)g(simply)e(transform)h(all)g(blo)q (c)o(king)g(sends)i(on)75 1915 y(the)f(sp)q(eci\014ed)i(comm)o (unicator)11 b(in)o(to)145 1990 y Fj(\017)23 b Ft(cop)o(y)13 b(data)h(to)g(bu\013er)h(space)145 2064 y Fj(\017)23 b Ft(issue)14 b(non-blo)q(c)o(king)f(send)i(from)d(the)i(bu\013er)h (cop)o(y)145 2139 y Fj(\017)23 b Ft(return)15 b(to)e(application)145 2214 y Fj(\017)23 b Ft(c)o(hec)o(k)14 b(completion)f(on)g(subsequen)o (t)j(MPI)e(call\(s\))166 2289 y(No)i(doubt)g(man)o(y)e(optimizations)g (within)h(MPI)h(are)h(p)q(ossible|the)f(prop)q(osal)g(just)g(requires)i (that)e(MPI)75 2338 y(pro)o(vide)e(at)f(least)i(this)e(m)o(uc)o(h)g (safet)o(y)m(.)166 2477 y FD(Note)e(that)g(collectiv)o(e)j(comm)o (unications)e(and)g(other)f(nonlo)q(cal)i(op)q(erations)f(will)i(share) d(suggested)75 2534 y(or)k(user-supplied)j(comm)o(unication)d (bu\013ers)h(with)f(p)q(oin)o(t)h(to)e(p)q(oin)o(t)i(comm)o(unication.) 1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 185 192 185 191 bop 75 -100 a Fp(7.2.)34 b(ERR)o(OR)17 b(HANDLING)1194 b FD(185)166 45 y Fl(Discussion:)29 b Ft(When)10 b(a)f(comm)o(unicator) d(asso)q(ciated)k(with)f(a)g(user)h(bu\013er)h(is)e(freed,)h(then)g Fv(MPI)p 1667 45 13 2 v 15 w(COMM)p 1812 45 V 14 w(FREE)75 102 y Ft(should)17 b(not)h(return)g(un)o(til)f(all)f(p)q(ending)h(comm) o(unications)e(ha)o(v)o(e)i(terminated.)28 b(Th)o(us,)18 b(after)g(it)f(returns,)i(the)75 158 y(bu\013er)c(can)f(b)q(e)g(freed.) 75 384 y Fq(7.2)59 b(Erro)n(r)21 b(handling)75 485 y FD(This)15 b(section)h(con)o(tains)f(the)g(suggested)g(POSIX)g(binding) i(for)e(error)f(handling)i(in)g(MPI.)e(MPI)h(imple-)75 542 y(men)o(tations)d(that)h(do)f(not)h(run)g(in)g(a)g(POSIX-complian)o (t)i(en)o(vironmen)o(t)e(ma)o(y)f(need)h(to)g(use)g(alternativ)o(e)75 598 y(mec)o(hanisms.)166 655 y(An)k(MPI)h(implemen)o(tation)g(cannot)f (or)g(ma)o(y)f(c)o(ho)q(ose)i(not)f(to)f(handle)j(some)e(errors)f(that) h(o)q(ccur)75 711 y(during)23 b(MPI)f(calls.)41 b(These)22 b(can)g(include)i(errors)d(that)h(generate)f(exceptions)i(or)f(traps,)g (suc)o(h)g(as)75 768 y(\015oating)e(p)q(oin)o(t)h(errors)e(or)h(access) g(violations.)36 b(The)20 b(set)g(of)g(errors)f(that)h(are)g(handled)h (b)o(y)f(MPI)g(is)75 824 y(implemen)o(tation-dep)q(endent.)i(Eac)o(h)15 b(suc)o(h)g(error)g(generates)g(an)g Fx(MPI)h(exception)p FD(.)166 881 y(A)g(user)h(can)f(asso)q(ciate)h(an)f(error)g(handler)h (with)g(a)f(comm)o(unicator.)23 b(The)16 b(sp)q(eci\014ed)j(error)d (han-)75 937 y(dling)k(routine)f(will)i(b)q(e)e(used)g(for)f(an)o(y)h (MPI)f(exception)i(that)e(o)q(ccurs)h(during)g(a)g(call)h(to)e(MPI)g (for)g(a)75 993 y(comm)o(unication)d(with)f(this)g(comm)o(unicator.)19 b(MPI)14 b(calls)h(that)e(are)g(not)h(related)g(to)f(an)o(y)g(comm)o (unica-)75 1050 y(tor)e(are)h(considered)i(to)d(b)q(e)i(attac)o(hed)f (to)f(the)h(comm)o(unicator)g Fv(MPI)p 1230 1050 V 15 w(COMM)p 1375 1050 V 14 w(W)o(ORLD)p FD(.)g(The)g(attac)o(hmen)o(t)75 1106 y(of)i(error)g(handlers)i(to)e(comm)o(unicators)g(is)i(purely)f (lo)q(cal:)21 b(di\013eren)o(t)15 b(pro)q(cesses)g(ma)o(y)f(attac)o(h)g (di\013eren)o(t)75 1163 y(error)g(handlers)j(to)d(the)i(same)e(comm)o (unicator.)166 1219 y(A)j(newly)i(created)e(comm)o(unicator)h(inherits) g(the)g(error)f(handler)h(that)f(is)h(asso)q(ciated)g(with)g(the)75 1276 y(\\paren)o(t")h(comm)o(unicator.)32 b(In)20 b(particular,)h(the)f (user)g(can)f(sp)q(ecify)i(a)e(\\global")h(error)f(handler)i(for)75 1332 y(all)d(comm)o(unicators)e(b)o(y)h(asso)q(ciating)g(this)g (handler)h(with)f(the)g(comm)o(unicator)f Fv(MPI)p 1570 1332 V 14 w(COMM)p 1714 1332 V 15 w(W)o(ORLD)75 1389 y FD(immediately)h(after)d(initialization.)166 1445 y(Sev)o(eral)i (prede\014ned)g(error)f(handlers)h(are)f(a)o(v)m(ailable)i(in)f(MPI:)75 1539 y Fv(MPI)p 152 1539 V 14 w(ERRORSF)m(A)m(T)m(AL)k FD(The)14 b(handler,)h(when)g(called,)g(causes)f(the)g(program)f(to)h (ab)q(ort)f(on)h(all)h(executing)189 1595 y(pro)q(cesses.)22 b(This)16 b(has)g(the)g(same)g(e\013ect)f(as)h(if)g Fs(MPI)p 1089 1595 14 2 v 16 w(ABORT)h FD(w)o(as)e(called)j(b)o(y)e(the)g(pro)q (cess)g(that)189 1652 y(in)o(v)o(ok)o(ed)f(the)g(handler.)75 1746 y Fv(MPI)p 152 1746 13 2 v 14 w(ERRORSRETURN)21 b FD(The)16 b(handler)g(has)f(no)g(e\013ect.)166 1839 y(Implemen)o(tations)k(ma)o(y)f(pro)o(vide)h(additional)h(prede\014ned) g(error)d(handlers)j(and)e(programmers)75 1896 y(can)d(co)q(de)h(their) g(o)o(wn)f(error)f(handlers.)166 1952 y(The)h(error)e(handler)j Fv(MPI)p 610 1952 V 14 w(ERRORSF)m(A)m(T)m(AL)c FD(is)j(asso)q(ciated)g (b)o(y)f(default)h(with)g Fv(MPI)p 1570 1952 V 14 w(COMM)p 1714 1952 V 15 w(W)o(ORLD)75 2009 y FD(after)f(initialization.)22 b(Th)o(us,)14 b(if)h(the)f(user)h(c)o(ho)q(oses)f(not)g(to)g(con)o (trol)g(error)f(handling,)j(ev)o(ery)e(error)g(that)75 2065 y(MPI)j(handles)h(is)f(treated)g(as)f(fatal.)25 b(Since)18 b(\(almost\))e(all)i(MPI)f(calls)h(return)f(an)f(error)h(co) q(de,)g(a)g(user)75 2122 y(ma)o(y)e(c)o(ho)q(ose)h(to)g(handle)h (errors)e(in)i(its)f(main)g(co)q(de,)h(b)o(y)f(testing)g(the)g(return)g (co)q(de)g(of)g(MPI)g(calls)h(and)75 2178 y(executing)f(a)e(suitable)h (reco)o(v)o(ery)f(co)q(de)h(when)g(the)g(call)g(w)o(as)f(not)g (successful.)21 b(In)15 b(suc)o(h)g(case,)f(the)h(error)75 2235 y(handler)g Fv(MPI)p 315 2235 V 15 w(ERRORSRETURN)d FD(will)k(b)q(e)f(used.)20 b(Usually)c(it)e(is)h(more)f(con)o(v)o (enien)o(t)h(and)f(more)g(e\016cien)o(t)75 2291 y(not)f(to)g(test)h (for)f(errors)g(after)g(eac)o(h)g(MPI)h(call,)h(and)f(ha)o(v)o(e)f(suc) o(h)h(error)f(handled)i(b)o(y)f(a)f(non)h(trivial)h(MPI)75 2348 y(error)f(handler.)166 2487 y Fl(Discussion:)43 b Ft(Do)15 b(w)o(e)i(sp)q(ecify)f(that)h Fv(MPI)p 884 2487 V 14 w(ERRORSF)m(A)m(T)m(AL)c Ft(should)j(generate)h(a)f Fv(SIGABRT)f Ft(signal)g(at)75 2543 y(eac)o(h)i(executing)g(pro)q (cess?)27 b(Is)17 b(that)f(the)h(desired)g(e\013ect?)27 b(\(e.g.,)16 b(so)g(as)h(to)f(allo)o(w)f(a)h(debugger)h(to)f(tak)o(e)g (o)o(v)o(er.\))75 2600 y(Same)d(remark)g(applies)g(to)h Fv(MPI)p 594 2600 V 14 w(ABORT)p Ft(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 186 193 186 192 bop 75 -100 a FD(186)532 b Fp(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)166 45 y FD(An)h(MPI)g(error)f(handler)i(is)g(an)e(opaque)h(ob)s(ject,)g(whic) o(h)g(is)h(accessed)f(b)o(y)g(a)g(handle.)26 b(MPI)17 b(calls)75 102 y(are)d(pro)o(vided)h(to)f(create)h(new)f(error)g (handlers,)h(to)f(asso)q(ciate)h(error)e(handlers)j(with)f(comm)o (unicators,)75 158 y(and)g(to)g(test)g(whic)o(h)h(error)e(handler)i(is) g(asso)q(ciated)g(with)f(a)g(comm)o(unicator.)75 262 y Fs(MPI)p 160 262 14 2 v 16 w(CREA)l(TE)p 345 262 V 17 w(ERRHANDLER\()i(function,)f(errhandler)f(\))117 339 y Ft(IN)155 b Fs(function)434 b Ft(user)15 b(de\014ned)g(error)g (handling)e(pro)q(cedure)117 414 y(OUT)108 b Fs(errhandler)397 b Ft(MPI)14 b(error)h(handler)f(\(handle\))75 539 y FB(int)23 b(MPI)p 245 539 15 2 v 17 w(Create)p 406 539 V 17 w(errhandler\(MPI)p 759 539 V 15 w(Handlerfunction)f(*function,)393 595 y(MPI)p 468 595 V 17 w(Errhandler)g(*errhandler\))75 681 y(MPI)p 150 681 V 17 w(CREATE)p 311 681 V 16 w(ERRHANDLER\(FUNCTION,)f (HANDLER,)i(IERROR\))170 738 y(EXTERNAL)g(FUNCTION)170 794 y(INTEGER)g(ERRHANDLER,)g(IERROR)166 881 y FD(Register)12 b(the)g(user)g(routine)g Fs(handler)p 802 881 14 2 v 18 w(function)h FD(for)e(use)h(as)f(an)h(MPI)g(exception)h(handler.)19 b(Returns)75 937 y(in)d Fs(errhandler)f FD(a)g(handle)i(to)d(the)i (registered)f(exception)h(handler.)166 1076 y Fl(Implemen)o(tati)o(on)c (note:)36 b Ft(The)15 b(handle)f(returned)h(ma)o(y)d(con)o(tain)i(the)g (address)h(of)f(the)g(error)h(handling)75 1133 y(routine.)j(Suc)o(h)13 b(call)f(is)h(sup)q(er\015uous)h(in)f(C,)f(whic)o(h)h(has)g(a)f (referencing)i(op)q(erator,)g(but)f(is)f(necessary)j(in)e(F)m(ortran.) 166 1328 y FD(The)j(user)h(routine)f(should)i(b)q(e)f(a)e(C)h(function) h(of)f(t)o(yp)q(e)h Fv(MPI)p 1224 1328 13 2 v 14 w(Handler)p 1371 1328 V 17 w(function)p FD(,)g(whic)o(h)g(is)g(de\014ned)75 1385 y(as)75 1491 y FB(typedef)23 b(void)g (\(MPI_Handler_function\)\(MPI_C)o(omm,)e(int)i(,)h(...\);)75 1597 y FD(The)14 b(\014rst)f(argumen)o(t)f(is)i(the)g(comm)o(unicator)f (in)h(use.)19 b(The)14 b(second)g(is)g(the)f(error)g(co)q(de)h(to)f(b)q (e)h(returned)75 1654 y(b)o(y)k(the)h(MPI)g(routine.)30 b(The)19 b(remaining)h(argumen)o(ts)d(are)h(\\)p FB(varargs)p FD(")f(argumen)o(ts)h(whose)h(n)o(um)o(b)q(er)75 1710 y(and)g(meaning)h(is)g(implemen)o(tation-dep)q(enden)o(t.)34 b(An)20 b(implemen)o(tation)g(should)g(clearly)h(do)q(cumen)o(t)75 1767 y(these)15 b(argumen)o(ts.)189 1873 y Fo(R)n(ationale.)72 b FD(The)21 b(v)m(ariable)h(argumen)o(t)e(list)h(is)g(pro)o(vided)h(b)q (ecause)f(it)g(pro)o(vides)g(an)g(ANSI-)189 1929 y(standard)13 b(ho)q(ok)h(for)g(pro)o(viding)h(additional)g(information)f(to)g(the)g (error)f(handler;)i(without)g(this)189 1986 y(ho)q(ok,)f(ANSI)i(C)f (prohibits)i(additional)f(argumen)o(ts.)j(\()p Fo(End)d(of)g(r)n (ationale.)p FD(\))75 2139 y Fs(MPI)p 160 2139 14 2 v 16 w(SET)p 259 2139 V 17 w(ERRHANDLER\()g(comm)m(,)c(errhandler)j(\)) 117 2217 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(to)j(set)h(the)f(error)h(handler)f(for)g(\(handle\))117 2292 y(IN)155 b Fs(errhandler)397 b Ft(new)15 b(MPI)f(error)g(handler)h (for)e(comm)o(unicator)e(\(handle\))75 2416 y FB(int)23 b(MPI)p 245 2416 15 2 v 17 w(Set)p 334 2416 V 17 w(errhandler\(MPI)p 687 2416 V 15 w(Comm)h(comm,)f(MPI)p 1037 2416 V 17 w(Errhandler)f (errhandler\))75 2503 y(MPI)p 150 2503 V 17 w(SET)p 239 2503 V 17 w(ERRHANDLER\(COMM,)f(ERRHANDLER,)i(IERROR\))170 2559 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)166 2646 y FD(Asso)q(ciates)16 b(the)g(new)g(error)f(handler)h Fs(erro)o(rhandler)f FD(with)h(comm)o(unicator)g Fs(comm)9 b FD(at)15 b(the)h(calling)75 2702 y(pro)q(cess.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 187 194 187 193 bop 75 -100 a Fp(7.3.)34 b(ERR)o(OR)17 b(CODES)1286 b FD(187)75 45 y Fs(MPI)p 160 45 14 2 v 16 w(GET)p 264 45 V 17 w(ERRHANDLER\()16 b(comm)m(,)c(errhandler)j(\))117 122 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(to)j(get)g(the)h(error)g(handler)f(from)e(\(handle\))117 197 y(OUT)108 b Fs(errhandler)397 b Ft(MPI)16 b(error)g(handler)g (curren)o(tly)g(asso)q(ciated)g(with)f(comm)o(u-)905 254 y(nicator)f(\(handle\))75 378 y FB(int)23 b(MPI)p 245 378 15 2 v 17 w(Get)p 334 378 V 17 w(errhandler\(MPI)p 687 378 V 15 w(Comm)h(comm,)f(MPI)p 1037 378 V 17 w(Errhandler)f (*errhandler\))75 465 y(MPI)p 150 465 V 17 w(GET)p 239 465 V 17 w(ERRHANDLER\(COMM,)f(ERRHANDLER,)i(IERROR\))170 521 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)166 608 y FD(Returns)16 b(in)g Fs(errhandler)g FD(\(a)f(handle)i(to\))e(the)g (error)g(handler)i(that)e(is)h(curren)o(tly)g(asso)q(ciated)g(with)75 664 y(comm)o(unicator)f Fs(comm)m FD(.)166 721 y(Example:)22 b(A)16 b(library)h(function)g(ma)o(y)f(register)g(at)f(its)i(en)o(try)f (p)q(oin)o(t)g(the)g(curren)o(t)g(error)g(handler)75 777 y(for)h(a)h(comm)o(unicator,)f(set)h(its)g(o)o(wn)f(priv)m(ate)i (error)e(handler)i(for)e(this)h(comm)o(unicator,)g(and)g(restore)75 834 y(b)q(efore)d(exiting)i(the)e(previous)h(error)e(handler.)166 973 y Fl(Discussion:)30 b Ft(One)10 b(ma)o(y)e(ha)o(v)o(e)h(the)i (function)e Fv(MPI)p 993 973 13 2 v 14 w(SET)p 1083 973 V 16 w(ERRHANDLER)g Ft(b)q(oth)h(set)h(a)e(new)h(error)h(handler)75 1029 y(and)17 b(return)i(the)g(previous)f(error)g(handler)g(\(the)g (function)g(w)o(ould)f(ha)o(v)o(e)g(t)o(w)o(o)g(errhandler)i(argumen)o (ts\),)f(th)o(us)75 1086 y(sa)o(ving)d(one)i(call)e(when)i(b)q(oth)p 574 1086 V 31 w(GET)f(and)p 779 1086 V 31 w(SET)g(are)g(needed.)27 b(But,)16 b(in)g(an)o(y)g(case,)h(one)f(needs)i(b)q(oth)e(the)h(old)75 1142 y(and)d(new)g(error-handlers.)75 1328 y Fs(MPI)p 160 1328 14 2 v 16 w(ERROR)p 323 1328 V 18 w(STRING\()h(erro)o(rco)q (de,)f(len,)i(string)f(\))117 1406 y Ft(IN)155 b Fs(erro)o(rco)q(de)410 b Ft(Error)15 b(co)q(de)g(returned)g(b)o(y)f(an)f(MPI)i(routine)117 1481 y(IN)155 b Fs(len)533 b Ft(Length)14 b(of)g Fv(string)117 1556 y Ft(OUT)108 b Fs(string)481 b Ft(T)m(ext)14 b(that)g(corresp)q (onds)i(to)e(the)g Fv(erro)o(rco)q(de)75 1680 y FB(int)23 b(MPI)p 245 1680 15 2 v 17 w(Error)p 382 1680 V 17 w(string\(int)f (errorcode,)h(int)g(len,)g(char)h(*string\))75 1767 y(MPI)p 150 1767 V 17 w(ERROR)p 287 1767 V 16 w(STRING\(ERRORCODE,)e(STRING,)h (IERROR\))170 1823 y(INTEGER)g(ERRORCODE,)g(IERROR)170 1880 y(CHARACTER\(*\))g(STRING)166 1966 y FD(Returns)16 b(the)f(error)f(string)i(asso)q(ciated)f(with)h(an)f(error)f(co)q(de.) 189 2072 y Fo(R)n(ationale.)38 b FD(The)13 b(form)f(of)g(this)h(prop)q (osal)g(w)o(as)f(c)o(hosen)h(to)f(mak)o(e)g(the)h(F)l(ortran)f(and)h(C) f(bindings)189 2129 y(similar.)30 b(A)18 b(v)o(ersion)g(that)g(returns) g(a)g(p)q(oin)o(ter)g(to)g(a)g(string)g(has)g(t)o(w)o(o)f (di\016culties.)31 b(First,)18 b(the)189 2185 y(return)10 b(string)h(m)o(ust)g(b)q(e)g(statically)h(allo)q(cated)g(and)f (di\013eren)o(t)g(for)f(eac)o(h)h(error)f(message)g(\(allo)o(wing)189 2242 y(the)j(p)q(oin)o(ters)g(returned)g(b)o(y)g(successiv)o(e)h(calls) g(to)e Fv(MPI)p 1117 2242 13 2 v 14 w(ERROR)p 1268 2242 V 14 w(STRING)h FD(to)f(p)q(oin)o(t)h(to)f(the)h(correct)189 2298 y(message.)30 b(Second,)20 b(in)g(F)l(ortran,)e(a)g(function)i (declared)g(as)f(returning)g Fv(CHARA)o(CTER*\(*\))e FD(can)189 2355 y(not)d(b)q(e)i(referenced)g(in,)g(for)f(example,)g(a)g Fv(PRINT)g FD(statemen)o(t.)k(\()p Fo(End)c(of)i(r)n(ationale.)p FD(\))75 2498 y Fq(7.3)59 b(Erro)n(r)21 b(co)r(des)75 2599 y FD(MPI)12 b(sp)q(eci\014es)i(some)d(of)h(the)f(error)h(return)f (v)m(alues)j(that)d(an)g(MPI)h(implemen)o(tation)h(ma)o(y)f(return.)18 b(This)75 2656 y(is)e(not)f(an)g(exhaustiv)o(e)g(list,)h(but)f(MPI)h (implemen)o(tations)g(m)o(ust)f(use)g(these)h(where)f(appropriate.)181 b Fm(CHANGE)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 188 195 188 194 bop 75 -100 a FD(188)538 b Fp(CHAPTER)15 b(7.)29 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)344 43 y FD(MPI)p 436 43 14 2 v 16 w(ERR)p 549 43 V 17 w(BUFFER)128 b(In)o(v)m(alid)17 b(bu\013er)f(p)q(oin)o(ter)344 100 y(MPI)p 436 100 V 16 w(ERR)p 549 100 V 17 w(COUNT)149 b(In)o(v)m(alid)17 b(coun)o(t)e(argumen)o(t)344 156 y(MPI)p 436 156 V 16 w(ERR)p 549 156 V 17 w(TYPE)189 b(In)o(v)m(alid)17 b(datat)o(yp)q(e)e(argumen)o(t)344 212 y(MPI)p 436 212 V 16 w(ERR)p 549 212 V 17 w(T)l(A)o(G)220 b(In)o(v)m(alid)17 b(tag)e(argumen)o(t)344 269 y(MPI)p 436 269 V 16 w(ERR)p 549 269 V 17 w(COMM)166 b(In)o(v)m(alid)17 b(comm)o(unicator)344 325 y(MPI)p 436 325 V 16 w(ERR)p 549 325 V 17 w(RANK)182 b(In)o(v)m(alid)17 b(rank)344 382 y(MPI)p 436 382 V 16 w(ERR)p 549 382 V 17 w(REQUEST)96 b(In)o(v)m(alid)17 b(request)f(\(handle\))344 438 y(MPI)p 436 438 V 16 w(ERR)p 549 438 V 17 w(R)o(OOT)183 b(In)o(v)m(alid)17 b(ro)q(ot)344 495 y(MPI)p 436 495 V 16 w(ERR)p 549 495 V 17 w(GR)o(OUP)150 b(In)o(v)m(alid)17 b(group)344 551 y(MPI)p 436 551 V 16 w(ERR)p 549 551 V 17 w(OP)252 b(In)o(v)m(alid)17 b(op)q(eration)344 608 y(MPI)p 436 608 V 16 w(ERR)p 549 608 V 17 w(TOPOLOGY)51 b(In)o(v)m(alid)17 b(top)q(ology)344 664 y(MPI)p 436 664 V 16 w(ERR)p 549 664 V 17 w(DIMS)200 b(In)o(v)m(alid)17 b(dimension)g(argumen)o(t)344 721 y(MPI)p 436 721 V 16 w(ERR)p 549 721 V 17 w(AR)o(G)216 b(In)o(v)m(alid)17 b(argumen)o(t)e(of)g(some)f(other)h(kind)344 777 y(MPI)p 436 777 V 16 w(ERR)p 549 777 V 17 w(UNKNO)o(WN)66 b(Unkno)o(wn)15 b(error)344 833 y(MPI)p 436 833 V 16 w(ERR)p 549 833 V 17 w(TR)o(UNCA)l(TE)58 b(Message)15 b(truncated)g(on)g(receiv)o(e)344 890 y(MPI)p 436 890 V 16 w(ERR)p 549 890 V 17 w(INTERN)137 b(In)o(ternal)16 b(MPI)f(error)344 946 y(MPI)p 436 946 V 16 w(ERR)p 549 946 V 17 w(LASTCODE)64 b(Last)15 b(standard)g(error)f (co)q(de)75 1077 y Fq(7.4)59 b(Timers)75 1179 y FD(MPI)19 b(de\014nes)h(t)o(w)o(o)e(timers.)32 b(These)20 b(timers)f(are)g(sp)q (eci\014ed)i(ev)o(en)f(though)f(they)g(are)g(not)g(\\message-)75 1235 y(passing")g(b)q(ecause)g(timing)h(parallel)g(programs)d(is)i(imp) q(ortan)o(t)g(in)g(\\p)q(erformance)f(debugging")i(and)75 1292 y(b)q(ecause)d(existing)g(timers)f(\(b)q(oth)g(in)h(POSIX)g (1003.1-1988)d(and)i(1003.4D)e(14.1)h(and)h(in)h(F)l(ortran)e(90\))75 1348 y(are)g(either)h(incon)o(v)o(enien)o(t)g(or)f(do)g(not)g(pro)o (vide)h(adequate)f(access)g(to)g(high-resolution)i(timers.)75 1452 y Fs(MPI)p 160 1452 V 16 w(WTIME\(\))75 1556 y FB(int)23 b(double)g(MPI)p 412 1556 15 2 v 17 w(Wtime\(\))75 1642 y(DOUBLE)g(PRECISION)g(MPI)p 556 1642 V 16 w(WTIME\(\))166 1729 y FD(returns)16 b(a)g(\015oating-p)q(oin)o(t)h(n)o(um)o(b)q(er)g (of)e(seconds,)i(represen)o(ting)g(elapsed)g(w)o(all-clo)q(c)o(k)h (time)f(since)75 1785 y(some)e(time)g(in)h(the)g(past.)75 1889 y Fs(MPI)p 160 1889 14 2 v 16 w(PTIME\(\))75 1992 y FB(int)23 b(double)g(MPI)p 412 1992 15 2 v 17 w(Ptime\(\))75 2079 y(DOUBLE)g(PRECISION)g(MPI)p 556 2079 V 16 w(PTIME\(\))166 2165 y FD(returns)e(a)g(\015oating-p)q(oin)o(t)h(n)o(um)o(b)q(er)g(of)e (seconds,)j(represen)o(ting)f(elapsed)h(pro)q(cess)e(time)h(since)75 2222 y(some)15 b(time)g(in)h(the)g(past.)166 2278 y(The)21 b(\\time)g(in)h(the)f(past")g(is)g(guaran)o(teed)g(not)f(to)h(c)o (hange)g(during)h(the)f(life)h(of)f(the)g(pro)q(cess.)75 2335 y(The)d(user)g(is)g(resp)q(onsible)h(for)e(con)o(v)o(erting)h (large)f(n)o(um)o(b)q(ers)h(of)f(seconds)h(to)f(other)g(units)i(if)f (they)f(are)75 2391 y(preferred.)166 2448 y(These)g(functions)g(are)f (p)q(ortable)h(\(return)f(seconds,)h(not)f("tic)o(ks"\),)g(allo)o(w)h (high-resolution,)h(and)75 2504 y(carry)d(no)g(unnecessary)h(baggage.)j (One)d(w)o(ould)f(use)h(them)f(lik)o(e)h(this:)75 2610 y FB({)147 2667 y(double)23 b(starttime,)f(endtime;)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 189 196 189 195 bop 75 -100 a Fp(7.5.)34 b(ST)l(AR)l(TUP)1411 b FD(189)147 45 y FB(starttime)22 b(=)i(double)f(MPI_Wtime\(\);)170 102 y(....)48 b(stuff)23 b(to)g(be)h(timed)47 b(...)147 158 y(endtime)70 b(=)24 b(double)f(MPI_Wtime\(\);)147 214 y(printf\("That)f(took)h(\045f)h(seconds\\n",endtime-start)o (time\);)75 271 y(})166 351 y FD(The)16 b(times)f(returned)h(b)o(y)g (these)g(routines)f(are)h(lo)q(cal)g(to)f(the)h(no)q(de)g(that)f (called)i(them.)k(There)15 b(is)75 408 y(no)g(requiremen)o(t)h(that)e (di\013eren)o(t)i(no)q(des)g(return)f(\\the)g(same)g(time.")257 540 y Fl(Discussion:)h Ft(User's)f(at)f(MPI)g(in)g(Europ)q(e)h(w)o(ere) g(strongly)e(in)h(fa)o(v)o(or)f(of)g(ha)o(ving)g(timing)f(func-)166 590 y(tions.)75 812 y Fq(7.5)59 b(Sta)n(rtup)75 914 y FD(One)13 b(goal)g(of)f(MPI)h(is)g(to)f(ac)o(hiev)o(e)h Fo(sour)n(c)n(e)h(c)n(o)n(de)g(p)n(ortability)p FD(.)19 b(By)13 b(this)g(w)o(e)f(mean)h(that)f(a)g(program)g(writ-)75 970 y(ten)j(using)i(MPI)e(and)g(complying)i(with)f(the)f(relev)m(an)o (t)h(language)g(standards)f(is)g(p)q(ortable)h(as)f(written,)75 1027 y(and)k(m)o(ust)e(not)h(require)i(an)o(y)e(source)g(co)q(de)h(c)o (hanges)f(when)h(mo)o(v)o(ed)f(from)g(one)g(system)g(to)g(another.)75 1083 y(This)h(explicitly)h(do)q(es)f Fo(not)j FD(sa)o(y)17 b(an)o(ything)i(ab)q(out)f(ho)o(w)f(an)h(MPI)g(program)f(is)i(started)e (or)g(launc)o(hed)75 1140 y(from)h(the)i(command)f(line,)i(nor)e(what)f (the)i(user)f(m)o(ust)f(do)i(to)e(set)h(up)h(the)f(en)o(vironmen)o(t)g (in)h(whic)o(h)75 1196 y(an)h(MPI)g(program)f(will)j(run.)38 b(Ho)o(w)o(ev)o(er,)21 b(an)h(implemen)o(tation)g(ma)o(y)e(require)i (some)f(setup)h(to)e(b)q(e)75 1253 y(p)q(erformed)c(b)q(efore)h(other)f (MPI)g(routines)g(ma)o(y)g(b)q(e)h(called.)24 b(T)l(o)16 b(pro)o(vide)g(for)g(this,)g(MPI)h(includes)h(an)75 1309 y(initialization)g(routine)e Fs(MPI)p 582 1309 14 2 v 16 w(INIT)p FD(.)75 1413 y Fs(MPI)p 160 1413 V 16 w(INIT\(\))75 1517 y FB(int)23 b(MPI)p 245 1517 15 2 v 17 w(Init\(int)g(*argc,)g (char)g(**argv\))75 1603 y(MPI)p 150 1603 V 17 w(INIT\(IERROR\))170 1659 y(INTEGER)g(IERROR)166 1746 y FD(This)17 b(routine)g(m)o(ust)f(b)q (e)h(called)h(b)q(efore)f(an)o(y)f(other)g(MPI)g(routine.)24 b(It)17 b(m)o(ust)f(b)q(e)h(called)h(at)e(most)75 1802 y(once;)f(subsequen)o(t)h(calls)g(are)f(erroneous)g(\(see)g Fs(MPI)p 982 1802 14 2 v 16 w(INITIALIZED)p FD(\).)166 1859 y(All)j(MPI)f(programs)e(m)o(ust)i(con)o(tain)g(a)f(call)i(to)e Fs(MPI)p 1092 1859 V 16 w(init)p FD(;)i(this)f(routine)h(m)o(ust)e(b)q (e)h(called)i(b)q(efore)75 1915 y(an)o(y)e(other)h(MPI)f(routine)h (\(apart)f(from)g FB(MPI)p 877 1915 15 2 v 16 w(INITIALIZED)p FD(\))f(is)i(called.)29 b(The)18 b(v)o(ersion)g(for)f(ANSI)h(C)75 1972 y(accepts)d(the)h Fv(a)o(rgc)f FD(and)g Fv(a)o(rgv)g FD(that)g(are)g(pro)o(vided)h(b)o(y)f(the)g(argumen)o(ts)f(to)h FB(main)p FD(:)75 2047 y FB(MPI_init\()23 b(argc,)g(argv)g(\);)75 2122 y FD(The)15 b(F)l(ortran)f(v)o(ersion)i(tak)o(es)e(no)h(argumen)o (ts.)166 2255 y Fl(Discussion:)24 b Ft(Note)18 b(that)g(POSIX)h (includes)f(a)g Fk(r)n(e)n(quir)n(ement)j Ft(that)d(access)i(to)e(the)h (command)c(line)i(b)q(e)75 2305 y(pro)o(vided)d(to)g(F)m(ortran)f (programs)g(\(see)i(8.9)e(in)g(Std)i(1003.9{1992\).)166 2354 y(MPI)c(implemen)o(tations)d(that)j(read)g(command-line)d(argumen) o(ts)i(should)h(insure)h(the)f(the)h(MPI)f(command-)75 2404 y(line)i(argumen)o(ts)h(are)g(clearly)g(iden)o(tifed,)f(for)g (example,)f(b)o(y)i(pre\014xing)g(them)f(with)h Fb(-mpi)p Ft(.)166 2461 y(Note)f(the)h(use)f(of)g(the)g Fk(addr)n(ess)g Ft(of)f Fb(argc)g Ft(in)h(the)g(C)g(binding)f(of)g Fv(MPI)p 1247 2461 13 2 v 15 w(INIT)p Ft(.)g(This)h(is)f(to)h(allo)o(w)e Fv(MPI)p 1732 2461 V 15 w(INIT)h Ft(to)75 2517 y(remo)o(v)o(e)h(those)i (argumen)o(ts)e(that)h(it)f(uses)i(from)d(the)j(argumen)o(t)e(list.)75 2704 y Fs(MPI)p 160 2704 14 2 v 16 w(FINALIZE\(\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 190 197 190 196 bop 75 -100 a FD(190)532 b Fp(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y FB(int)23 b(MPI)p 245 45 15 2 v 17 w(Finalize\(\))75 132 y(MPI)p 150 132 V 17 w(FINALIZE\(IERROR\))170 188 y(INTEGER)g(IERROR)166 274 y FD(This)17 b(routines)g(cleans)g(all)h (MPI)e(state.)23 b(Once)18 b(this)f(routine)g(is)g(called,)h(no)e(MPI)h (routine)g(\(ev)o(en)75 331 y Fs(MPI)p 160 331 14 2 v 16 w(INIT)p FD(\))12 b(ma)o(y)h(b)q(e)h(called.)21 b(The)14 b(user)g(m)o(ust)f(ensure)h(that)f(all)h(p)q(ending)i(comm)o (unications)e(in)o(v)o(olving)75 387 y(a)h(pro)q(cess)g(complete)h(b)q (efore)g(the)f(pro)q(cess)g(calls)i Fs(MPI)p 1023 387 V 16 w(FINALIZE)p FD(.)75 491 y Fs(MPI)p 160 491 V 16 w(INITIALIZED\()c(\015ag)i(\))117 568 y Ft(OUT)108 b Fs(\015ag)518 b Ft(F)p FD(lag)15 b(is)f(true)h(if)f Fs(MPI)p 1270 568 V 16 w(INIT)g FD(has)g(b)q(een)h(called)h(and)f(false)905 625 y(otherwise.)75 749 y FB(int)23 b(MPI)p 245 749 15 2 v 17 w(Initialized\(int)f(*flag\))75 836 y(MPI)p 150 836 V 17 w(INITIALIZED\(FLAG,)f(IERROR\))170 892 y(LOGICAL)i(FLAG)170 949 y(INTEGER)g(IERROR)166 1035 y FD(This)c(routine)f(ma)o(y)g(b)q(e)h (used)g(to)e(determine)i(whether)g Fs(MPI)p 1242 1035 14 2 v 16 w(INIT)e FD(has)h(b)q(een)h(called.)31 b(It)18 b(is)h(the)75 1091 y Fo(only)c FD(routine)g(that)g(ma)o(y)f(b)q(e)i (called)h(b)q(efore)e Fs(MPI)p 938 1091 V 16 w(INIT)g FD(is)g(called.)75 1195 y Fs(MPI)p 160 1195 V 16 w(ABORT\()h(comm)m(,)c (erro)o(rco)q(de)i(\))117 1272 y Ft(IN)155 b Fs(comm)466 b Ft(comm)o(unicator)11 b(of)j(tasks)g(to)g(ab)q(ort)117 1348 y(IN)155 b Fs(erro)o(rco)q(de)410 b Ft(error)15 b(co)q(de)g(to)e(return)j(to)d(in)o(v)o(oking)f(en)o(vironmen)o(t)75 1472 y FB(int)23 b(MPI)p 245 1472 15 2 v 17 w(Abort\(MPI)p 478 1472 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 1558 y(MPI)p 150 1558 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1615 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)166 1701 y FD(This)d(routine)f(mak)o(es)g(a)g(\\b)q(est)g(attempt")f(to)h(ab)q (ort)g(all)h(tasks)e(in)i(the)g(group)e(of)h Fs(comm)m FD(.)26 b(This)75 1758 y(function)21 b(do)q(es)g(not)f(require)h(that)f (the)g(in)o(v)o(oking)h(en)o(vironmen)o(t)f(tak)o(e)g(an)o(y)g(action)h (with)f(the)h(error)75 1814 y(co)q(de.)g(Ho)o(w)o(ev)o(er,)14 b(a)h(Unix)i(or)e(POSIX)h(en)o(vironmen)o(t)g(should)g(handle)h(this)f (as)f(a)g FB(return)23 b(errorcode)75 1871 y FD(from)14 b(the)i(main)f(program)f(or)h(an)g FB(abort\(errorcode\))p FD(.)166 1927 y(MPI)f(implemen)o(tations)h(are)f(required)h(to)e (de\014ne)i(the)f(b)q(eha)o(vior)h(of)e Fs(MPI)p 1443 1927 14 2 v 16 w(ABORT)i FD(at)f(least)g(for)f(a)75 1984 y FB(comm)i FD(of)f Fv(MPI)p 314 1984 13 2 v 15 w(COMM)p 459 1984 V 14 w(W)o(ORLD)p FD(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 191 198 191 197 bop 75 356 a Fr(Chapter)34 b(8)75 564 y Fy(Pro\014ling)42 b(In)m(terface)75 805 y Fq(8.1)59 b(Requirements)75 906 y FD(T)l(o)15 b(meet)g(the)g(MPI)g(pro\014ling)i(in)o(terface,)e(an)g (implemen)o(tation)i(of)d(the)i(MPI)f(functions)h Fo(must)131 1000 y FD(1.)22 b(pro)o(vide)14 b(a)f(mec)o(hanism)h(through)f(whic)o (h)h(all)g(of)f(the)g(MPI)h(de\014ned)h(functions)f(ma)o(y)e(b)q(e)i (accessed)189 1056 y(with)e(a)g(name)f(shift.)20 b(Th)o(us)11 b(all)i(of)f(the)g(MPI)g(functions)h(\(whic)o(h)f(normally)g(start)f (with)h(the)g(pre\014x)189 1113 y(\\)p FB(MPI)p 287 1113 15 2 v 16 w FD("\))j(should)h(also)f(b)q(e)h(accessable)g(with)g(the)f (pre\014x)h(\\)p FB(PMPI)p 1300 1113 V 16 w FD(".)131 1207 y(2.)22 b(ensure)c(that)f(those)h(MPI)g(functions)h(whic)o(h)f (are)g(not)f(replaced)i(ma)o(y)f(still)h(b)q(e)g(link)o(ed)g(in)o(to)f (an)189 1263 y(executable)e(image)f(without)h(causing)f(name)h (clashes.)131 1357 y(3.)22 b(do)q(cumen)o(t)c(the)f(implemen)o(tation)i (of)e(di\013eren)o(t)h(language)g(bindings)h(of)e(the)h(MPI)g(in)o (terface)f(if)189 1413 y(they)g(are)h(la)o(y)o(ered)g(on)f(top)g(of)h (eac)o(h)f(other,)h(so)f(that)g(the)h(pro\014ler)g(dev)o(elop)q(er)h (kno)o(ws)e(whether)189 1470 y(she)d(m)o(ust)f(implemen)o(t)i(the)e (pro\014le)i(in)o(terface)f(for)f(eac)o(h)h(binding,)h(or)e(can)h (economise)h(b)o(y)e(imple-)189 1526 y(men)o(ting)i(it)h(only)f(for)g (the)g(lo)o(w)o(est)g(lev)o(el)h(routines.)901 b Fm(CHANGE)131 1620 y FD(4.)22 b(where)15 b(the)g(implemen)o(tation)i(of)d(di\013eren) o(t)i(language)f(bindings)i(is)f(is)g(done)f(through)g(a)g(la)o(y)o (ered)189 1677 y(approac)o(h)h(\(e.g.)24 b(the)17 b(F)l(ortran)f (binding)j(is)e(a)f(set)h(of)g(\\wrapp)q(er")f(functions)i(whic)o(h)f (call)h(the)f(C)189 1733 y(implemen)o(tation\),)g(ensure)g(that)f (these)g(wrapp)q(er)h(functions)g(are)f(separable)h(from)f(the)g(rest)g (of)189 1790 y(the)f(library)l(.)189 1865 y(This)k(is)h(necessary)f(to) f(allo)o(w)h(a)g(separate)f(pro\014ling)j(library)e(to)g(b)q(e)g (correctly)h(implemen)o(ted,)189 1921 y(since)g(\(at)f(least)g(with)h (Unix)g(link)o(er)h(seman)o(tics\))e(the)h(pro\014ling)g(library)g(m)o (ust)f(con)o(tain)h(these)189 1978 y(wrapp)q(er)14 b(functions)h(if)g (it)f(is)h(to)f(p)q(erform)g(as)f(exp)q(ected.)21 b(This)15 b(requiremen)o(t)g(allo)o(ws)f(the)h(p)q(erson)189 2034 y(who)10 b(builds)i(the)f(pro\014ling)h(library)f(to)f(extract)g(these) h(functions)g(from)f(the)h(original)h(MPI)e(library)189 2091 y(and)k(add)g(them)g(in)o(to)g(the)f(pro\014ling)j(library)e (without)g(bringing)h(along)f(an)o(y)g(other)f(unnecessary)189 2147 y(co)q(de.)75 2290 y Fq(8.2)59 b(Discussion)75 2392 y FD(The)13 b(ob)s(jectiv)o(e)g(of)g(the)g(MPI)g(pro\014ling)h(in)o (terface)g(is)f(to)f(ensure)i(that)e(it)i(is)f(relativ)o(ely)h(easy)f (for)g(authors)75 2448 y(of)18 b(pro\014ling)i(\(and)f(other)f (similar\))h(to)q(ols)g(to)f(in)o(terface)h(their)g(co)q(des)g(to)f (MPI)h(implemen)o(tations)h(on)75 2505 y(di\013eren)o(t)15 b(mac)o(hines.)166 2561 y(Since)20 b(MPI)e(is)h(a)f(mac)o(hine)h(indep) q(enden)o(t)h(standard)e(with)h(man)o(y)f(di\013eren)o(t)g(implemen)o (tations,)75 2617 y(it)e(is)h(unreasonable)g(to)e(exp)q(ect)i(that)f (the)g(authors)f(of)h(pro\014ling)h(to)q(ols)f(for)g(MPI)g(will)i(ha)o (v)o(e)d(access)i(to)75 2674 y(the)c(source)h(co)q(de)f(whic)o(h)h (implemen)o(ts)h(MPI)e(on)g(an)o(y)g(particular)h(mac)o(hine.)20 b(It)13 b(is)h(therefore)f(necessary)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 192 199 192 198 bop 75 -100 a FD(192)878 b Fp(CHAPTER)15 b(8.)30 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)75 45 y FD(to)21 b(pro)o(vide)i(a)f(mec)o(hanism)g(b)o(y)g(whic)o(h)h(the)f(implemen)o (tors)h(of)f(suc)o(h)g(to)q(ols)g(can)g(collect)i(whatev)o(er)75 102 y(p)q(erformance)15 b(information)h(they)f(wish)h Fo(without)g FD(access)f(to)g(the)g(underlying)i(implemen)o(tation.)166 158 y(W)l(e)d(b)q(eliev)o(e)i(that)e(ha)o(ving)g(suc)o(h)g(an)h(in)o (terface)f(is)h(imp)q(ortan)o(t)e(if)i(MPI)f(is)g(to)g(b)q(e)h (attractiv)o(e)e(to)g(end)75 214 y(users,)i(since)i(the)e(a)o(v)m (ailabilit)o(y)j(of)d(man)o(y)g(di\013eren)o(t)h(to)q(ols)f(will)i(b)q (e)g(a)e(signi\014can)o(t)h(factor)f(in)h(attracting)75 271 y(users)f(to)g(the)g(MPI)g(standard.)166 327 y(The)e(pro\014ling)h (in)o(terface)e(is)h(just)g(that,)f(an)g(in)o(terface.)20 b(It)12 b(sa)o(ys)g Fo(nothing)g FD(ab)q(out)g(the)h(w)o(a)o(y)f(in)h (whic)o(h)75 384 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no) g(attempt)f(to)g(la)o(y)h(do)o(wn)g(what)g(information)g(is)h (collected)g(through)75 440 y(the)g(in)o(terface,)g(or)g(ho)o(w)g(the)g (collected)i(information)e(is)h(sa)o(v)o(ed,)e(\014ltered,)i(or)f (displa)o(y)o(ed.)166 497 y(While)g(the)g(initial)h(imp)q(etus)f(for)f (the)g(dev)o(elopmen)o(t)h(of)e(this)i(in)o(terface)f(arose)g(from)f (the)h(desire)i(to)75 553 y(p)q(ermit)g(the)f(implemen)o(tation)i(of)d (pro\014ling)j(to)q(ols,)d(it)i(is)f(clear)h(that)f(an)g(in)o(terface)g (lik)o(e)h(that)f(sp)q(eci\014ed)75 610 y(ma)o(y)i(also)h(pro)o(v)o(e)f (useful)i(for)e(other)h(purp)q(oses,)g(suc)o(h)g(as)g(\\in)o(ternet)o (w)o(orking")f(m)o(ultiple)i(MPI)f(imple-)75 666 y(men)o(tations.)j (Since)c(all)f(that)f(is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there) h(is)g(no)f(ob)s(jection)h(to)f(its)h(b)q(eing)g(used)75 723 y(wherev)o(er)f(it)h(is)f(useful.)166 779 y(As)j(the)g(issues)h(b)q (eing)g(addressed)f(here)h(are)f(in)o(timately)h(tied)f(up)h(with)f (the)g(w)o(a)o(y)f(in)i(whic)o(h)g(ex-)75 835 y(ecutable)j(images)e (are)g(built,)j(whic)o(h)e(ma)o(y)f(di\013er)h(greatly)f(on)g (di\013eren)o(t)h(mac)o(hines,)h(the)e(examples)75 892 y(giv)o(en)c(b)q(elo)o(w)g(should)h(b)q(e)f(treated)g(solely)g(as)f (one)h(w)o(a)o(y)f(of)g(implemen)o(ting)j(the)d(ob)s(jectiv)o(e)h(of)f (the)h(MPI)75 948 y(pro\014ling)h(in)o(terface.)k(The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o(tation)h(are)e (those)g(detailed)75 1005 y(in)i(the)f(Requiremen)o(ts)i(section)e(ab)q (o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this)i(c)o(hapter)f (is)g(only)h(presen)o(t)f(as)75 1061 y(justi\014cation)g(and)f (discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.)166 1118 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f(in)h (whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed)75 1174 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 1231 y(equally)d(v)m(alid\).)75 1374 y Fq(8.3)59 b(Logic)20 b(of)g(the)f(design)75 1475 y FD(Pro)o(vided)d(that)f(an)g(MPI)g (implemen)o(tation)i(meets)e(the)h(requiremen)o(ts)g(ab)q(o)o(v)o(e,)e (it)i(is)g(p)q(ossible)h(for)e(the)75 1532 y(implemen)o(tor)20 b(of)e(the)h(pro\014ling)h(system)f(to)f(in)o(tercept)i(all)f(of)g(the) g(MPI)g(calls)h(whic)o(h)f(are)g(made)g(b)o(y)75 1588 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h(collect)g(whatev)o (er)f(information)g(she)h(requires)g(b)q(efore)f(calling)75 1645 y(the)c(underlying)i(MPI)d(implemen)o(tation)i(\(through)f(its)g (name)f(shifted)i(en)o(try)e(p)q(oin)o(ts\))h(to)f(ac)o(hiev)o(e)i(the) 75 1701 y(desired)g(e\013ects.)75 1844 y Fq(8.4)59 b(Examples)75 1947 y Fn(8.4.1)49 b(Pro\014ler)16 b(implem)o(ent)o(ation)75 2033 y FD(Supp)q(ose)25 b(that)e(the)h(pro\014ler)h(wishes)g(to)e (accum)o(ulate)i(the)f(total)f(amoun)o(t)g(of)h(data)f(sen)o(t)h(b)o(y) g(the)75 2090 y(MPI)p 167 2090 14 2 v 16 w(SEND)18 b(function,)g(along) f(with)h(the)f(total)g(elapsed)h(time)f(sp)q(en)o(t)h(in)g(the)f (function.)27 b(This)18 b(could)75 2146 y(trivially)f(b)q(e)f(ac)o (hiev)o(ed)g(th)o(us)-822 b Fm(CHANGE)75 2252 y FB(static)23 b(int)g(totalBytes;)75 2308 y(static)g(int)g(totalTime;)75 2365 y(extern)g(int)g(now\(void\);)71 b(/*)23 b(Some)h(suitable)e (timer)i(function)e(!)i(*/)75 2478 y(int)f(MPI_SEND\(void)g(*)g (buffer,)g(const)g(int)h(count,)f(MPI_Datatype)f(datatype,)385 2534 y(int)i(dest,)f(int)g(tag,)h(MPI_comm)e(comm\))75 2591 y({)147 2647 y(int)h(tstart)g(=)h(now\(\);)166 b(/*)24 b(Pass)f(on)h(all)f(the)g(arguments)g(*/)147 2704 y(int)g(result)g(=)h (PMPI_Send\(buffer,count,da)o(tatype,)o(dest,ta)o(g,comm\);)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 193 200 193 199 bop 75 -100 a Fp(8.4.)34 b(EXAMPLES)1370 b FD(193)147 102 y FB(totalBytes)22 b(+=)i(count)f(*)h(MPI_Sizeof\(datatype\);)d(/*) i(Accumulate)g(byte)g(count)g(*/)147 158 y(totalTime)46 b(+=)24 b(now\(\))f(-)h(tstart;)357 b(/*)23 b(and)h(time)333 b(*/)147 271 y(return)23 b(result;)75 327 y(})75 447 y Fn(8.4.2)49 b(MPI)17 b(lib)o(ra)o(ry)f(implem)o(ent)o(ation)75 533 y FD(On)i(a)f(Unix)h(system,)f(in)h(whic)o(h)g(the)g(MPI)f(library) h(is)g(implemen)o(ted)h(in)f(C,)f(then)g(there)h(are)e(v)m(arious)75 590 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i(t)o(w)o(o)e(of)g(the) h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.)35 b(Whic)o(h)20 b(is)h(b)q(etter)75 646 y(dep)q(ends)c(on)e(whether)g(the)h(link)o(er)g (and)f(compiler)i(supp)q(ort)e(w)o(eak)g(sym)o(b)q(ols.)75 764 y Fs(Systems)f(with)i(w)o(eak)f(symb)q(ols)75 850 y FD(If)k(the)f(compiler)i(and)f(link)o(er)h(supp)q(ort)f(w)o(eak)f (external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g (system)75 907 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h (library)g(is)f(required)h(through)f(the)g(use)g(of)g FB(#pragma)23 b(weak)13 b FD(th)o(us)75 999 y FB(#pragma)23 b(weak)g(MPI_Example)g(=)g(PMPI_Example)75 1112 y(int)g (PMPI_Example\(/*)f(appropriate)h(args)g(*/\))75 1169 y({)170 1225 y(/*)h(Useful)f(content)g(*/)75 1282 y(})166 1374 y FD(The)d(e\013ect)g(of)g(this)g FB(#pragma)f FD(is)i(to)e (de\014ne)i(the)f(external)h(sym)o(b)q(ol)f FB(MPI)p 1472 1374 15 2 v 17 w(EXAMPLE)f FD(as)h(a)g(w)o(eak)75 1431 y(de\014nition,)d(this)f(means)g(that)f(the)g(link)o(er)i(will)g (not)e(complain)i(if)f(there)g(is)g(another)f(de\014nition)j(of)d(the) 75 1487 y(sym)o(b)q(ol)g(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1544 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1662 y Fs(Systems)f(without)j(w)o(eak)e(symb)q(ols) 75 1748 y FD(In)k(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h(then)g (one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h(the)f(C)h (macro)75 1804 y(pre-pro)q(cessor)c(th)o(us)75 1897 y FB(#ifdef)23 b(PROFILELIB)75 1953 y(#)95 b(ifdef)23 b(__STDC__)75 2010 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 2066 y(#)95 b(else)75 2123 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 2179 y(#)95 b(endif)75 2236 y(#else)75 2292 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 2349 y(#endif)166 2441 y FD(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2534 y FB(int)23 b(FUNCTION\(MPI_Example\)\(/*)e (appropriate)h(args)i(*/\))75 2591 y({)170 2647 y(/*)g(Useful)f (content)g(*/)75 2704 y(})-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 194 201 194 200 bop 75 -100 a FD(194)873 b Fp(CHAPTER)15 b(8.)35 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)166 45 y FD(The)20 b(same)g(source)g(\014le)h(can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q (duce)i(b)q(oth)f(v)o(ersions)g(of)g(the)g(library)l(,)75 102 y(dep)q(ending)d(on)f(the)f(state)f(of)h(the)g FB(PROFILELIB)f FD(macro)h(sym)o(b)q(ol.)166 158 y(It)e(is)g(required)h(that)e(the)h (standard)f(MPI)h(library)g(b)q(e)h(built)g(in)f(suc)o(h)g(a)g(w)o(a)o (y)e(that)h(the)h(inclusion)i(of)75 214 y(MPI)g(functions)g(can)g(b)q (e)h(ac)o(hiev)o(ed)f(one)g(at)g(a)f(time.)20 b(This)c(is)f(a)f (somewhat)g(unpleasan)o(t)i(requiremen)o(t,)75 271 y(since)22 b(it)f(ma)o(y)f(mean)h(that)f(eac)o(h)g(external)i(function)f(has)g(to) f(b)q(e)h(compiled)i(from)d(a)g(separate)g(\014le.)75 327 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g(necessary)g(so)f(that)g(the)g (author)g(of)g(the)h(pro\014ling)g(library)g(need)h(only)f(de\014ne)g (those)75 384 y(MPI)f(functions)g(whic)o(h)g(she)g(wishes)h(to)e(in)o (tercept,)h(references)g(to)f(an)o(y)g(others)g(b)q(eing)i(ful\014lled) h(b)o(y)e(the)75 440 y(normal)g(MPI)g(library)l(.)21 b(Therefore)15 b(the)h(link)g(step)f(can)h(lo)q(ok)f(something)h(lik)o (e)g(this)75 547 y FB(\045)24 b(cc)f(...)h(-lmyprof)f(-lpmpi)g(-lmpi) 166 653 y FD(Here)e FB(libmyprof.a)e FD(con)o(tains)i(the)g(pro\014ler) g(functions)h(whic)o(h)f(in)o(tercept)g(some)g(of)f(the)h(MPI)75 709 y(functions.)j FB(libpmpi.a)15 b FD(con)o(tains)i(the)f(\\name)h (shifted")g(MPI)f(functions,)h(and)g FB(libmpi.a)e FD(con)o(tains)75 766 y(the)g(normal)g(de\014nitions)i(of)e(the)g(MPI)h(functions.)75 887 y Fn(8.4.3)49 b(Complications)75 973 y Fs(Multiple)16 b(counting)-245 1011 y Fm(CHANGE)75 1059 y FD(Since)24 b(parts)d(of)h(the)g(MPI)g(library)h(ma)o(y)e(themselv)o(es)i(b)q(e)g (implemen)o(ted)h(using)f(more)e(basic)i(MPI)75 1116 y(functions)c(\(e.g.)27 b(a)18 b(p)q(ortable)g(implemen)o(tation)i(of)d (the)i(collectiv)o(e)g(op)q(erations)f(implemen)o(ted)i(using)75 1172 y(p)q(oin)o(t)d(to)f(p)q(oin)o(t)h(comm)o(uncations\),)f(there)h (is)g(p)q(oten)o(tial)g(for)f(pro\014ling)h(functions)h(to)d(b)q(e)j (called)g(from)75 1228 y(within)k(an)e(MPI)h(function)g(whic)o(h)g(w)o (as)f(called)i(from)e(a)g(pro\014ling)i(function.)37 b(This)21 b(could)g(lead)h(to)75 1285 y(\\double)15 b(coun)o(ting")e (of)h(the)f(time)h(sp)q(en)o(t)g(in)h(the)e(inner)i(routine.)20 b(Since)15 b(this)f(e\013ect)f(could)i(actually)f(b)q(e)75 1341 y(useful)f(under)g(some)e(circumstances)i(\(e.g.)18 b(it)12 b(migh)o(t)g(allo)o(w)g(one)g(to)f(answ)o(er)h(the)f(question)i (\\Ho)o(w)e(m)o(uc)o(h)75 1398 y(time)19 b(is)g(sp)q(en)o(t)f(in)i(the) e(p)q(oin)o(t)h(to)f(p)q(oin)o(t)h(routines)f(when)h(they're)f(called)i (from)e(collectiv)o(e)i(functions)75 1454 y(?"\),)13 b(w)o(e)h(ha)o(v)o(e)f(decided)j(not)d(to)g(enforce)h(an)o(y)f (restrictions)h(on)g(the)g(author)f(of)g(the)h(MPI)g(library)g(whic)o (h)75 1511 y(w)o(ould)i(o)o(v)o(ercome)g(this.)23 b(Therefore)16 b(the)g(author)f(of)h(the)g(pro\014ling)h(library)g(should)g(b)q(e)g(a) o(w)o(are)e(of)h(this)75 1567 y(problem,)i(and)f(guard)g(against)f(it)h (themself.)26 b(In)18 b(a)f(single)h(threaded)f(w)o(orld)g(this)h(is)f (easily)h(ac)o(hiev)o(ed)75 1624 y(through)g(use)g(of)f(a)h(static)g(v) m(ariable)h(in)g(the)f(pro\014ling)h(co)q(de)f(whic)o(h)h(remem)o(b)q (ers)f(if)g(y)o(ou)g(are)g(already)75 1680 y(inside)i(a)d(pro\014ling)i (routine.)29 b(It)18 b(b)q(ecomes)g(more)f(complex)i(in)g(a)e(m)o (ulti-threaded)j(en)o(vironmen)o(t)e(\(as)75 1737 y(do)q(es)e(the)f (meaning)h(of)e(the)i(times)f(recorded)h(!\))75 1857 y Fs(Link)o(er)f(o)q(ddities)-245 1886 y Fm(CHANGE)75 1942 y FD(The)g(Unix)g(link)o(er)g(traditionally)h(op)q(erates)e(in)i (one)e(pass)g(:)20 b(the)14 b(e\013ect)g(of)g(this)h(is)g(that)f (functions)h(from)75 1999 y(libraries)f(are)e(only)h(included)j(in)d (the)g(image)f(if)h(they)g(are)f(needed)i(at)e(the)h(time)g(the)f (library)h(is)h(scanned.)75 2055 y(When)j(com)o(bined)g(with)g(w)o(eak) f(sym)o(b)q(ols,)g(or)g(m)o(ultiple)i(de\014nitions)g(of)e(the)h(same)f (function,)h(this)g(can)75 2112 y(cause)e(o)q(dd)h(\(and)f(unexp)q (ected\))i(e\013ects.)166 2168 y(Consider,)24 b(for)d(instance,)j(an)d (implemen)o(tation)j(of)d(MPI)h(in)g(whic)o(h)h(the)f(F)l(ortran)f (binding)i(is)75 2225 y(ac)o(hiev)o(ed)c(b)o(y)e(using)i(wrapp)q(er)f (functions)g(on)g(top)g(of)f(the)h(C)f(implemen)o(tation.)29 b(The)18 b(author)g(of)f(the)75 2281 y(pro\014le)e(library)h(then)e (assumes)h(that)f(it)g(is)h(reasonable)g(only)g(to)f(pro)o(vide)h (pro\014le)g(functions)h(for)e(the)g(C)75 2338 y(binding,)20 b(since)e(F)l(ortran)e(will)j(ev)o(en)o(tually)g(call)f(these,)g(and)g (the)f(cost)g(of)g(the)h(wrapp)q(ers)f(is)h(assumed)75 2394 y(to)f(b)q(e)g(small.)27 b(Ho)o(w)o(ev)o(er,)16 b(if)i(the)f(wrapp)q(er)g(functions)h(are)f(not)g(in)h(the)f (pro\014ling)i(library)l(,)f(then)g(none)75 2451 y(of)c(the)g (pro\014led)i(en)o(try)e(p)q(oin)o(ts)g(will)i(b)q(e)f(unde\014ned)h (when)f(the)g(pro\014ling)g(library)g(is)g(called.)21 b(Therefore)75 2507 y(none)c(of)g(the)g(pro\014ling)h(co)q(de)g(will)g (b)q(e)g(included)h(in)f(the)f(image.)25 b(When)18 b(the)f(standard)f (MPI)h(library)75 2563 y(is)f(scanned,)f(the)g(F)l(ortran)f(wrapp)q (ers)h(will)i(b)q(e)e(resolv)o(ed,)h(and)f(will)h(also)f(pull)i(in)f (the)f(base)g(v)o(ersions)g(of)75 2620 y(the)g(MPI)g(functions.)20 b(The)15 b(o)o(v)o(erall)g(e\013ect)g(is)g(that)f(the)h(co)q(de)g(will) i(link)f(successfully)l(,)h(but)d(will)j(not)d(b)q(e)75 2676 y(pro\014led.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 195 202 195 201 bop 75 -100 a Fp(8.5.)34 b(MUL)l(TIPLE)16 b(LEVELS)g(OF)f (INTER)o(CEPTION)739 b FD(195)166 45 y(T)l(o)17 b(o)o(v)o(ercome)f (this)h(w)o(e)g(m)o(ust)f(ensure)h(that)g(the)g(F)l(ortran)e(wrapp)q (er)i(functions)h(are)f(included)i(in)75 102 y(the)d(pro\014ling)i(v)o (ersion)e(of)g(the)g(library)l(.)23 b(W)l(e)17 b(ensure)f(that)g(this)g (is)h(p)q(ossible)h(b)o(y)e(requiring)h(that)f(these)75 158 y(b)q(e)f(separable)h(from)d(the)i(rest)f(of)g(the)h(base)g(MPI)f (library)l(.)21 b(This)15 b(allo)o(ws)g(them)g(to)e(b)q(e)j FB(ar)p FD(ed)e(out)h(of)f(the)75 214 y(base)h(library)h(and)g(in)o(to) f(the)g(pro\014ling)i(one.)75 358 y Fq(8.5)59 b(Multiple)20 b(levels)f(of)h(interception)75 459 y FD(The)15 b(sc)o(heme)h(giv)o(en) g(here)f(do)q(es)h(not)f(directly)h(supp)q(ort)f(the)h(nesting)g(of)e (pro\014ling)j(functions,)e(since)i(it)75 516 y(pro)o(vides)f(only)f(a) g(single)h(alternativ)o(e)g(name)f(for)f(eac)o(h)h(MPI)g(function.)21 b(Consideration)16 b(w)o(as)e(giv)o(en)i(to)75 572 y(an)f(implemen)o (tation)i(whic)o(h)f(w)o(ould)g(allo)o(w)g(m)o(ultiple)h(lev)o(els)g (of)e(call)h(in)o(terception,)h(ho)o(w)o(ev)o(er)d(w)o(e)h(w)o(ere)75 629 y(unable)j(to)e(construct)g(an)g(implemen)o(tation)i(of)e(this)h (whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o(wing)i(disadv)m(an-) 75 685 y(tages)143 779 y FA(\017)23 b FD(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 873 y FA(\017)23 b FD(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o(en) g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 966 y(Since)i(one)e(of)g(the)h(ob)s(jectiv)o(es)f(of)g(MPI)h(is)f(to)g (p)q(ermit)h(e\016cien)o(t,)g(lo)o(w)f(latency)h(implemen)o(tations,)h (and)75 1023 y(it)j(is)f(not)g(the)h(business)g(of)f(a)g(standard)g(to) g(require)h(a)f(particular)h(implemen)o(tation)g(language,)g(w)o(e)75 1079 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h(outlined)h(ab)q(o)o(v) o(e.)166 1136 y(Note,)f(ho)o(w)o(ev)o(er,)f(that)h(it)h(is)f(p)q (ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o(v)o(e)g(to)g(implemen) o(t)i(a)e(m)o(ulti-lev)o(el)75 1192 y(system,)k(since)g(the)g(function) g(called)i(b)o(y)d(the)h(user)f(ma)o(y)g(call)i(man)o(y)e(di\013eren)o (t)g(pro\014ling)i(functions)75 1249 y(b)q(efore)15 b(calling)i(the)f (underlying)h(MPI)e(function.)166 1305 y(Unfortunately)22 b(suc)o(h)g(an)g(implemen)o(tation)h(ma)o(y)e(require)h(more)g(co-op)q (eration)g(b)q(et)o(w)o(een)g(the)75 1362 y(di\013eren)o(t)g (pro\014ling)i(libraries)g(than)e(is)h(required)g(for)e(the)i(single)g (lev)o(el)h(implemen)o(tation)f(detailed)75 1418 y(ab)q(o)o(v)o(e.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 196 203 196 202 bop 75 356 a Fr(Chapter)34 b(9)75 564 y Fy(Initial)43 b(Implemen)m(tation)f(Subset)75 805 y Fq(9.1)59 b(Intro)r(duction)75 906 y FD(This)17 b(c)o(hapter)f(de\014nes)h(the)g(minimal)h(subset)e (of)g(MPI)g(for)g(initial)i(implemen)o(tation.)25 b(This)17 b(subset)f(is)75 963 y(b)q(eing)c(de\014ned)h(so)d(that)g(consisten)o (t)i(implemen)o(tations)g(can)f(app)q(ear)g(more)g(rapidly)l(.)20 b(It)11 b(w)o(as)f(recognized)75 1019 y(early)17 b(in)h(the)f(pro)q (cess)g(that)f(MPI)h(needed)h(to)e(app)q(ear)h(as)f(quic)o(kly)i(as)f (p)q(ossible)h(and)f(practical.)26 b(The)75 1076 y(creation)16 b(of)f(a)g(subset)h(will)h(hop)q(efully)g(allo)o(w)f(users)g(earlier)g (access)g(to)e(the)i(standard)f(and)h(still)h(allo)o(w)75 1132 y(for)e(the)g(writing)h(of)e(p)q(ortable)i(message)f(passing)g(co) q(des.)166 1188 y(This)20 b(subset)g(should)h(not)e(in)i(an)o(y)e(w)o (a)o(y)g(b)q(e)h(construed)g(as)f(a)h(limitation)h(on)e(MPI)h (implemen-)75 1245 y(tations.)35 b(It)20 b(is)h(strictly)g(the)g(minim) o(um)g(necessary)g(to)e(ha)o(v)o(e)h(an)h(initial)h(MPI)e(subset)h (conforming)75 1301 y(implemen)o(tation.)28 b(It)18 b(is)g(hop)q(ed)g (that)f(an)g(o\016cially)i(sanctioned)g(subset)e(will)i(encourage)f (and)g(allo)o(w)75 1358 y(implemen)o(tors)j(to)e(in)o(tro)q(duce)i(MPI) f(in)g(a)g(more)g(timely)g(fashion.)35 b(It)20 b(should)h(b)q(e)f (noted,)h(ho)o(w)o(ev)o(er,)75 1414 y(that)d(implemen)o(tation)h(of)f (the)g(subset)h(do)q(es)g(not)e(mak)o(e)h(an)g(implemen)o(tation)i (conform)e(with)g(MPI.)75 1471 y(The)h(subset)g(is)g(a)f(p)q(oten)o (tial)h(\014rst)g(step)f(in)i(the)f(pro)q(cess.)30 b(All)20 b(implemen)o(tations)g(shall)g(conform)e(to)75 1527 y(the)f(en)o(tire)g (standard;)f(implemen)o(tations)i(are)e(encouraged)h(to)f(do)g(the)h (full)g(standard)f(as)h(rapidly)g(as)75 1584 y(p)q(ossible.)166 1640 y(The)f(subset)g(presen)o(ted)g(is)h(consisten)o(t)f(with)g(the)g (complete)h(MPI)f(standard.)21 b(This)16 b(w)o(as)g(an)f(im-)75 1697 y(p)q(ortan)o(t)e(goal)h(so)g(the)h(additional)g(MPI)f(features)g (could)i(b)q(e)e(added)h(without)g(c)o(hanging)f(an)o(y)g(previous)75 1753 y(functionalit)o(y)g(from)e(the)h(user's)g(p)q(ersp)q(ectiv)o(e.) 20 b(Th)o(us,)13 b(users)g(can)g(b)q(e)g(assured)g(that)f(programs)g (written)75 1810 y(no)o(w)f(for)h(the)g(subset)g(will)h(run)f(without)g (mo)q(di\014cation)h(under)g(the)f(full)h(MPI)f(standard.)18 b(F)l(urthermore,)75 1866 y(using)i(additional)g(features)f(of)f(the)h (full)i(MPI)e(standard)f(in)i(the)f(future)g(will)i(not)d(require)i(c)o (hanges)75 1922 y(to)e(co)q(de)i(already)f(written.)30 b(This)20 b(requires)f(that)g(the)f(MPI)h(functions)h(ha)o(v)o(e)e (their)i(full)g(calling)g(ar-)75 1979 y(gumen)o(ts)e(in)h(the)f (subset.)28 b(T)l(o)18 b(ac)o(hiev)o(e)h(this)f(goal,)g(the)g(subset)h (will)g(generally)h(limit)f(access)f(to)g(full)75 2035 y(MPI)d(functionalit)o(y)h(through)f(complete)g(elimination)i(of)e (functions)g(or)g(limitations)h(on)f(the)g(creation)75 2092 y(mec)o(hanisms)h(for)e(certain)i(MPI)f(input)h(argumen)o(ts.)75 2235 y Fq(9.2)59 b(Criteria)21 b(and)e(Rationale)75 2336 y FD(Ha)o(ving)g(the)g(subset)g(b)q(e)h(consisten)o(t)f(with)g(the)g (en)o(tire)g(MPI)g(standard)g(w)o(as)f(considered)i(critical)h(to)75 2393 y(the)16 b(e\013ort.)22 b(In)16 b(addition)i(to)d(this,)h(there)h (w)o(ere)e(man)o(y)h(p)q(ossible)i(criteria)f(to)e(use)h(in)h (determining)h(the)75 2449 y(elemen)o(ts)d(of)e(MPI)h(to)f(include)j (in)f(the)f(subset.)20 b(The)14 b(main)g(criteria)h(used)f(w)o(ere)g (that)f(the)h(MPI)g(subset)75 2506 y(should:)131 2600 y(1.)22 b(Con)o(tain)12 b(routines)h(that)e(are)h(as)g(close)h(as)f(p)q (ossible)i(to)e(curren)o(t)g(standard)g(practice)h(to)f(minimize)189 2656 y(the)j(e\013ort)f(to)h(p)q(ort)f(co)q(des.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 197 204 197 203 bop 75 -100 a Fp(9.3.)34 b(SUBSET)16 b(FUNCTIONALITY)1036 b FD(197)131 45 y(2.)22 b(Con)o(tain)15 b(new)g(and)g(imp)q(ortan)o(t)g (MPI)g(features.)131 139 y(3.)22 b(Allo)o(w)17 b(dev)o(elop)q(ers)h(to) e(b)q(e)i(able)f(to)g(ha)o(v)o(e)f(the)h(subset)g(sho)o(w)f(up)i(as)e (rapidly)i(as)f(p)q(ossible)h(while)189 195 y(still)e(meeting)g(the)f (other)g(criteria.)166 289 y(There)i(w)o(ere)g(man)o(y)f(rationales)h (for)g(these)g(criteria.)26 b(It)17 b(w)o(as)f(recognized)i(that)e(it)h (is)h(imp)q(ossible)75 346 y(to)d(come)g(up)h(with)f(an)g(ideal)i (subset,)e(and)g(compromises)h(w)o(ere)f(necessary)l(.)20 b(It)c(w)o(as)e(felt)i(that)e(curren)o(t)75 402 y(users)22 b(should)g(b)q(e)h(comfortable)e(in)h(migrating)g(to)f(MPI.)g(Th)o(us,) i(the)f(subset)f(should)i(con)o(tain)f(the)75 459 y(message)16 b(passing)h(elemen)o(ts)g(that)f(represen)o(t)g(common)g(practice)h(to) q(da)o(y)l(.)23 b(F)l(urthermore,)16 b(it)h(w)o(as)f(felt)75 515 y(that)22 b(the)h(initial)h(\(and)f(later\))f(v)o(ersion)h(of)f (MPI)h(should)g(main)o(tain)g(e\016ciency;)28 b(otherwise,)c(users)75 571 y(could)18 b(su\013er)f(from)g(disenc)o(han)o(tmen)o(t)h(with)g (MPI)f(on)g(their)h(\014rst)f(usage)g(and)h(migh)o(t)f(nev)o(er)g(giv)o (e)h(the)75 628 y(standard)12 b(a)g(second)g(try)l(.)19 b(Ha)o(ving)12 b(the)h(common)e(practice)i(routines)g(in)g(the)f (subset)h(should)g(encourage)75 684 y(dev)o(elop)q(ers)i(to)e(optimize) i(these)f(routines)g(in)g(the)g(initial)i(v)o(ersion)e(of)f(MPI.)g (Also,)h(although)g(some)g(felt)75 741 y(that)g(the)g(p)q(ortabilit)o (y)h(o\013ered)f(b)o(y)g(the)h(MPI)f(subset)h(w)o(as)e(su\016cien)o(t)i (to)f(en)o(tice)h(users,)f(man)o(y)g(felt)h(that)75 797 y(the)j(subset)g(should)g(con)o(tain)g(some)g(new)g(features)f(o)o(v)o (er)g(and)h(ab)q(o)o(v)o(e)f(curren)o(t)h(common)f(practice)h(as)75 854 y(an)i(added)h(incen)o(tiv)o(e.)36 b(The)20 b(additional)h(items)g (selected)g(w)o(ere)f(mean)o(t)f(to)g(represen)o(t)i(some)e(of)h(the)75 910 y(signi\014can)o(t)e(new)f(features)f(of)h(MPI.)f(Balanced)i (against)f(these)g(\014rst)f(t)o(w)o(o)g(goals)h(is)g(the)g(need)h(for) e(the)75 967 y(subset)e(to)f(sho)o(w)g(up)i(in)f(a)g(timely)h(fashion.) k(Th)o(us,)14 b(eac)o(h)g(feature)g(c)o(hosen)g(for)f(inclusion)j(in)f (the)f(subset)75 1023 y(w)o(as)21 b(deemed)h(of)f(su\016cien)o(t)h(imp) q(ortance)g(to)f(out)o(w)o(eigh)h(its)f(added)h(complexit)o(y)h(in)f (implemen)o(ting)75 1080 y(the)c(subset.)27 b(Although)18 b(some)f(functions)i(seem)f(easy)f(to)g(implemen)o(t,)i(there)f(are)f (often)g(o)o(v)o(erlo)q(ok)o(ed)75 1136 y(costs)j(in)h(testing,)g(do)q (cumen)o(tation,)h(and)f(dev)o(elopmen)o(t)g(that)f(w)o(ere)g (considered)h(b)q(efore)g(a)f(feature)75 1192 y(w)o(as)15 b(added)i(to)f(the)g(MPI)h(subset.)23 b(Inclusion)c(of)d(to)q(o)f(man)o (y)h(routines)h(migh)o(t)f(lead)h(to)f(a)g(mo)q(derately)75 1249 y(e\016cien)o(t)21 b(implemen)o(tation)h(of)e(them)h(all)g (instead)h(of)e(a)g(v)o(ery)g(e\016cien)o(t)i(and)f(p)q(ossibly)h(more) e(useful)75 1305 y(implemen)o(tation)c(of)f(a)g(smaller)h(subset.)75 1449 y Fq(9.3)59 b(Subset)19 b(F)n(unctionalit)n(y)75 1550 y FD(The)c(follo)o(wing)g(sections)g(con)o(tain)g(a)f(general)i (summary)e(and)g(sp)q(eci\014c)j(routines)e(of)f(what)g(is)h(included) 75 1607 y(in/excluded)f(from)e(the)g(subset)g(for)f(eac)o(h)h(c)o (hapter.)19 b(The)12 b(general)g(requiremen)o(ts)h(of)e(eac)o(h)h(c)o (hapter)g(also)75 1663 y(hold)h(for)f(the)h(subset.)19 b(As)12 b(examples,)h(the)g(message)f(en)o(v)o(elop)q(e)h(in)h (subsection)f(3.2.2,)e(the)i(seman)o(tics)f(in)75 1719 y(section)j(3.4,)f(the)h(t)o(yp)q(e)g(matc)o(hing)g(in)g(section)h (3.5,)d(and)j(the)e(comm)o(unication)i(ob)s(jects)e(in)i(subsection)75 1776 y(3.8.1)e(are)g(all)j(part)d(of)h(the)g(subset.)75 1898 y Fn(9.3.1)49 b(P)o(oint)16 b(to)h(P)o(oint)f(F)o(unctionalit)o(y) 75 1983 y FD(The)j(p)q(oin)o(t)h(to)e(p)q(oin)o(t)i(routines)f(are)g (not)g(only)h(cen)o(tral)f(to)f(MPI)h(but)h(represen)o(t)f(common)f (practice)75 2040 y(to)q(da)o(y)l(.)k(As)16 b(suc)o(h,)g(the)h(ma)s (jorit)o(y)d(of)i(the)g(functionalit)o(y)h(in)g(this)g(c)o(hapter)f(is) g(included)j(in)e(the)f(subset.)75 2096 y(Deriv)o(ed)f(datat)o(yp)q(es) e(represen)o(t)h(a)g(signi\014can)o(t)h(new)f(feature)g(in)h(MPI)f(and) h(is)f(included)j(in)e(the)f(subset.)75 2153 y(Ho)o(w)o(ev)o(er,)20 b(only)h(basic)g(datat)o(yp)q(es)f(of)g(section)h(3.2.1)e(will)j(b)q(e) f(allo)o(w)o(ed)g(as)f(input)h(to)f(the)g(datat)o(yp)q(e)75 2209 y(construction)15 b(routines)h(describ)q(ed)h(in)f(section)g (3.13.1.)166 2266 y(T)l(o)e(reduce)h(the)g(di\016cult)o(y)g(of)f (implemen)o(tation,)i(sev)o(eral)e(features)g(w)o(ere)g(left)h(out)f (of)g(the)g(subset.)75 2322 y(The)19 b(abilit)o(y)h(to)e(test)h(for)f (m)o(ultiple)j(completions)e(of)g(messages)f(is)i(remo)o(v)o(ed.)30 b(Also,)20 b(the)f(abilit)o(y)h(to)75 2379 y(cancel)13 b(a)e(message)g(and)g(use)h(p)q(ersisten)o(t)g(comm)o(unication)g(ob)s (jects)f(is)h(remo)o(v)o(ed.)18 b(Finally)l(,)c(the)e(routines)75 2435 y(asso)q(ciated)j(with)h(ready)f(and)g(sync)o(hronous)h(comm)o (unication)g(mo)q(des)f(are)g(excluded.)166 2492 y(The)f(table)h(b)q (elo)o(w)g(con)o(tains)f(a)g(listing)i(b)o(y)e(section)h(of)f(whic)o(h) h(routines)f(from)g(the)g(p)q(oin)o(t)h(to)e(p)q(oin)o(t)75 2548 y(comm)o(unication)j(c)o(hapter)f(are)g(included)j(in/excluded)f (from)e(the)g(subset.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 198 205 198 204 bop 75 -100 a FD(198)593 b Fp(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)120 45 y FD(Section)62 b(Title)400 b(Included)414 b(Excluded)p 75 68 1886 2 v 120 120 a(3.2)146 b(Basic)16 b(Send)g(Op)q(eration)63 b Fs(MPI)p 905 120 14 2 v 16 w(SEND)367 b FD(||)120 189 y(3.2.1)110 b(Message)15 b(Data)217 b(All)17 b(t)o(yp)q(es)402 b(||)120 258 y(3.3)146 b(Basic)16 b(Recv)g(Op)q(eration)61 b Fs(MPI)p 905 258 V 16 w(RECV)369 b FD(||)120 327 y(3.3.1)110 b(Return)16 b(Status)213 b Fs(MPI)p 905 327 V 16 w(GET)p 1009 327 V 17 w(SOURCE)205 b FD(||)820 383 y Fs(MPI)p 905 383 V 16 w(GET)p 1009 383 V 17 w(T)l(A)o(G)820 439 y(MPI)p 905 439 V 16 w(GET)p 1009 439 V 17 w(COUNT)120 508 y FD(3.7)146 b(Comm.)19 b(Mo)q(des)202 b(Standard)400 b Fs(MPI)p 1487 508 V 16 w(RSEND)1402 565 y(MPI)p 1487 565 V 16 w(SSEND)120 634 y FD(3.8.2)110 b(Comm.)19 b(Initiation)147 b Fs(MPI)p 905 634 V 16 w(ISEND)354 b(MPI)p 1487 634 V 16 w(IRSEND)820 690 y(MPI)p 905 690 V 16 w(IRECV)i(MPI)p 1487 690 V 16 w(ISSEND)120 759 y FD(3.8.3)110 b(Comm.)19 b(Completion)102 b Fs(MPI)p 905 759 V 16 w(W)l(AIT)371 b FD(||)820 816 y Fs(MPI)p 905 816 V 16 w(TEST)120 884 y FD(3.8.5)110 b(Multiple)17 b(Completion)83 b(||)492 b Fs(MPI)p 1487 884 V 16 w(W)l(AIT)l(ANY)1402 941 y(MPI)p 1487 941 V 16 w(TEST)l(ANY)1402 997 y(MPI)p 1487 997 V 16 w(W)l(AIT)l(ALL)1402 1054 y(MPI)p 1487 1054 V 16 w(TEST)l(ALL)120 1123 y FD(3.9)146 b(Prob)q(e)15 b(&)h(Cancel)178 b Fs(MPI)p 905 1123 V 16 w(PROBE)336 b(MPI)p 1487 1123 V 16 w(CANCEL)820 1179 y(MPI)p 905 1179 V 16 w(IPROBE)323 b(MPI)p 1487 1179 V 16 w(TEST)p 1617 1179 V 17 w(CANCELLED)120 1248 y FD(3.10)123 b(P)o(ersisten)o(t)15 b(Comm.)139 b(||)492 b Fs(MPI)p 1487 1248 V 16 w(CREA)l(TE)p 1672 1248 V 18 w(SEND)355 1305 y FD(Ob)s(jects)894 b Fs(MPI)p 1487 1305 V 16 w(CREA)l(TE)p 1672 1305 V 18 w(RSEND)1402 1361 y(MPI)p 1487 1361 V 16 w(CREA)l(TE)p 1672 1361 V 18 w(SSEND)1402 1417 y(MPI)p 1487 1417 V 16 w(CREA)l(TE)p 1672 1417 V 18 w(RECV)1402 1474 y(MPI)p 1487 1474 V 16 w(ST)l(ART)1402 1530 y(MPI)p 1487 1530 V 16 w(ST)l(ART)l(ALL)1402 1587 y(MPI)p 1487 1587 V 16 w(REQUEST)p 1706 1587 V 18 w(FREE)120 1656 y FD(3.11)123 b(Send-receiv)o(e)252 b Fs(MPI)p 905 1656 V 16 w(SENDRECV)g(MPI)p 1487 1656 V 16 w(SENDRECV)p 1735 1656 V 18 w(REPLA)o(CE)120 1725 y FD(3.12)123 b(Null)17 b(pro)q(cess)252 b Fv(MPI)p 897 1725 13 2 v 15 w(PROCNULL)278 b FD(||)120 1794 y(3.13.1)87 b(Datet)o(yp)q(e)15 b(const.)180 b Fs(MPI)p 905 1794 14 2 v 16 w(TYPE)p 1038 1794 V 17 w(CONTIGUOUS)62 b FD(See)16 b(text)f(ab)q(o)o(v)o(e)820 1850 y Fs(MPI)p 905 1850 V 16 w(TYPE)p 1038 1850 V 17 w(VECTOR)820 1906 y(MPI)p 905 1906 V 16 w(TYPE)p 1038 1906 V 17 w(HVECTOR)820 1963 y(MPI)p 905 1963 V 16 w(TYPE)p 1038 1963 V 17 w(INDEXED)820 2019 y(MPI)p 905 2019 V 16 w(TYPE)p 1038 2019 V 17 w(HINDEXED)820 2076 y(MPI)p 905 2076 V 16 w(TYPE)p 1038 2076 V 17 w(STRUCT)120 2145 y FD(3.13.2)87 b(Address)16 b(and)f(exten)o(t)109 b Fs(MPI)p 905 2145 V 16 w(ADDRESS)355 2201 y FD(functions)285 b Fs(MPI)p 905 2201 V 16 w(TYPE)p 1038 2201 V 17 w(EXTENT)820 2258 y(MPI)p 905 2258 V 16 w(TYPE)p 1038 2258 V 17 w(SIZE)820 2314 y(MPI)p 905 2314 V 16 w(TYPE)p 1038 2314 V 17 w(COUNT)120 2383 y FD(3.13.3)87 b(Lo)o(w)o(er-b)q(ound)16 b(and)147 b Fs(MPI)p 905 2383 V 16 w(LB)355 2439 y FD(upp)q(er-b)q(ound)212 b Fs(MPI)p 905 2439 V 16 w(UB)120 2508 y FD(3.13.4)87 b(Commit)15 b(and)g(free)158 b Fs(MPI)p 905 2508 V 16 w(TYPE)p 1038 2508 V 17 w(COMMIT)164 b FD(||)820 2565 y Fs(MPI)p 905 2565 V 16 w(TYPE)p 1038 2565 V 17 w(FREE)120 2634 y FD(3.13.5)87 b(Use)15 b(of)g(General...)163 b Fs(MPI)p 905 2634 V 16 w(GET)p 1009 2634 V 17 w(ELEMENTS)145 b FD(||)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 199 206 199 205 bop 75 -100 a Fp(9.3.)34 b(SUBSET)16 b(FUNCTIONALITY)1036 b FD(199)75 45 y Fn(9.3.2)49 b(Collective)17 b(Comm)o(unication)d(F)o (unctionalit)o(y)75 131 y FD(As)k(with)f(p)q(oin)o(t)h(to)f(p)q(oin)o (t)h(functionalit)o(y)l(,)i(collectiv)o(e)f(comm)o(unication)f(is)g (common)g(practice)g(to)q(da)o(y)l(,)75 187 y(and)d(the)f(functionalit) o(y)i(is)e(included)j(in)f(the)e(subset.)20 b(F)l(urthermore,)13 b(the)i(abilit)o(y)g(to)f(de\014ne)i(arbitrary)75 244 y(groups)e(in)i(the)f(MPI)g(subset,)f(whic)o(h)i(are)e(then)h(asso)q (ciated)g(with)g(a)g(comm)o(unicator,)f(allo)o(ws)h(the)g(user)75 300 y(to)k(p)q(erform)g(collectiv)o(e)j(comm)o(unication)e(op)q (erations)g(o)o(v)o(er)f(an)g(arbitrary)g(set)h(of)f(pro)q(cesses.)33 b(This)75 357 y(increases)15 b(the)f(functionalit)o(y)h(of)f(the)g (collectiv)o(e)i(comm)o(unication)e(routines)h(o)o(v)o(er)e(what)g(is)i (commonly)75 413 y(a)o(v)m(ailable)i(to)q(da)o(y)l(.)166 470 y(T)l(o)i(ease)h(initial)i(implemen)o(tation,)f(the)f(user)g (de\014ned)h(reduce)f(functions)h(are)e(excluded)j(from)75 526 y(the)d(subset.)33 b(Note)19 b(that)f(all)j(of)d(the)i(system)f (pro)o(vided)h(reduce)g(functions)g(listed)h(in)f(section)g(4.5.1)75 583 y(are)i(include)i(in)f(the)f(subset.)40 b(Also,)24 b(the)e(scan)g(functions)h(are)f(excluded)i(from)d(the)h(subset.)41 b(Fi-)75 639 y(nally)l(,)25 b(the)e(generalized)h(\\v")e(v)o(ersions)h (of)f(routines)h(are)f(excluded)j(from)c(the)i(subset)g(as)f(w)o(ell)h (as)75 695 y Fs(MPI)p 160 695 14 2 v 16 w(REDUCE)p 352 695 V 17 w(SCA)l(TTER)p FD(.)166 752 y(The)f(table)g(b)q(elo)o(w)g(con) o(tains)f(a)g(listing)i(b)o(y)f(section)g(of)f(whic)o(h)h(routines)g (from)f(the)g(collectiv)o(e)75 808 y(comm)o(unication)16 b(c)o(hapter)f(are)g(included)j(in/excluded)f(from)e(the)g(subset.)120 925 y(Section)62 b(Title)459 b(Included)246 b(Excluded)p 75 947 1869 2 v 120 1000 a(4.3)146 b(Barrier)15 b(Sync)o(h.)267 b Fs(MPI)p 964 1000 14 2 v 15 w(BARRIER)130 b FD(||)120 1068 y(4.4.1)110 b(Broadcast)354 b Fs(MPI)p 964 1068 V 15 w(BCAST)172 b FD(||)120 1137 y(4.4.2)110 b(Gather)414 b Fs(MPI)p 964 1137 V 15 w(GA)l(THER)142 b(MPI)p 1378 1137 V 16 w(GA)l(THERV)120 1206 y FD(4.4.4)110 b(Scatter)412 b Fs(MPI)p 964 1206 V 15 w(SCA)l(TTER)119 b(MPI)p 1378 1206 V 16 w(SCA)l(TTERV)120 1275 y FD(4.4.6)110 b(Gather-to-all)296 b Fs(MPI)p 964 1275 V 15 w(ALLGA)l(THER)62 b(MPI)p 1378 1275 V 16 w(ALLGA)l(THERV)120 1344 y FD(4.4.7)110 b(All-to-All)18 b(Scatter/Gather)44 b Fs(MPI)p 964 1344 V 15 w(ALL)l(TO)o(ALL)98 b(MPI)p 1378 1344 V 16 w(ALL)l(TO)o(ALL)-5 b(V)120 1413 y FD(4.5.1)110 b(Reduce)411 b Fs(MPI)p 964 1413 V 15 w(REDUCE)120 1482 y FD(4.5.2)110 b(User-Reduce)720 b Fs(MPI)p 1378 1482 V 16 w(USER)p 1506 1482 V 17 w(REDUCE)1293 1538 y(MPI)p 1378 1538 V 16 w(USER)p 1506 1538 V 17 w(REDUCEA)120 1607 y FD(4.5.3)110 b(All-Reduce)338 b Fs(MPI)p 964 1607 V 15 w(ALLREDUCE)61 b(MPI)p 1378 1607 V 16 w(USER)p 1506 1607 V 17 w(ALLREDUCE)1293 1664 y(MPI)p 1378 1664 V 16 w(USER)p 1506 1664 V 17 w(ALLREDUCEA)120 1733 y FD(4.5.4)110 b(Reduce-Scatter)668 b Fs(MPI)p 1378 1733 V 16 w(REDUCE)p 1570 1733 V 17 w(SCA)l(TTER)1293 1789 y(MPI)p 1378 1789 V 16 w(USER)p 1506 1789 V 17 w(REDUCE)p 1699 1789 V 18 w(SCA)l(TTER)1293 1846 y(MPI)p 1378 1846 V 16 w(USER)p 1506 1846 V 17 w(REDUCE)p 1699 1846 V 18 w(SCA)l(TTERA)120 1914 y FD(4.5.5)110 b(Scan)461 b(||)324 b Fs(MPI)p 1378 1914 V 16 w(SCAN)1293 1971 y(MPI)p 1378 1971 V 16 w(USER)p 1506 1971 V 17 w(SCAN)1293 2027 y(MPI)p 1378 2027 V 16 w(USER)p 1506 2027 V 17 w(SCANA)75 2149 y Fn(9.3.3)49 b(Groups,)15 b(Contexts,)g(Comm)o(unicato)o(rs,)d(and)17 b(Cacheing)75 2235 y FD(Groups,)22 b(con)o(texts,)f(and)g(comm)o (unicators)f(are)h(used)g(throughout)g(the)g(MPI)f(standard.)37 b(Though)75 2291 y(these)19 b(concepts)g(are)g(used)h(in)f(some)g(pac)o (k)m(ages)g(to)q(da)o(y)l(,)g(their)g(inclusion)j(in)d(MPI)g(represen)o (ts)g(a)g(sig-)75 2348 y(ni\014can)o(t)e(new)g(feature)g(o)o(v)o(er)f (what)g(is)h(generally)h(a)o(v)m(ailable)g(in)g(common)e(practice)i(to) q(da)o(y)l(.)24 b(F)l(or)16 b(these)75 2404 y(reasons,)i(this)h (functionalit)o(y)g(is)g(included)i(in)e(the)f(subset.)29 b(Ho)o(w)o(ev)o(er,)17 b(implemen)o(tation)j(of)e(groups,)75 2461 y(con)o(texts,)d(and)i(comm)o(unicators)f(is)g(also)g(a)g (signi\014can)o(t)i(e\013ort.)j(As)16 b(a)g(compromise,)h(the)f(MPI)g (in)o(tra-)75 2517 y(comm)o(unicator)21 b(functionalit)o(y)i(is)f (included,)k(but)21 b(the)h(in)o(ter-comm)o(unicator)g(functionalit)o (y)h(is)f(ex-)75 2574 y(cluded.)30 b(The)18 b(cac)o(heing)h(facilit)o (y)g(w)o(as)e(included)k(to)c(supp)q(ort)h(the)h(top)q(ology)e (functionalit)o(y)i(and)g(ex-)75 2630 y(tensions)d(to)e(MPI.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 200 207 200 206 bop 75 -100 a FD(200)593 b Fp(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)166 45 y FD(The)f(table)g(b)q(elo)o(w)h(con)o(tains)f(a)f(listing)i(b)o(y)f (section)h(of)e(whic)o(h)i(routines)f(from)f(the)h(Groups,)f(Con-)75 102 y(texts,)g(Comm)o(unicators,)g(and)i(Cac)o(heing)f(c)o(hapter)g (are)g(included)j(in/excluded)g(from)c(the)i(subset.)75 218 y(Section)g(Title)302 b(Included)522 b(Excluded)p 75 240 1924 2 v 75 293 a(5.2.4)64 b(Prede\014ned)17 b(Comm.)26 b Fs(MPI)p 716 293 14 2 v 16 w(COMM)p 874 293 V 16 w(W)o(ORLD)272 b FD(||)75 362 y(5.3.1)64 b(Group)15 b(Accessors)66 b Fs(MPI)p 716 362 V 16 w(GROUP)p 884 362 V 18 w(SIZE)329 b FD(||)631 418 y Fs(MPI)p 716 418 V 16 w(GROUP)p 884 418 V 18 w(RANK)631 475 y(MPI)p 716 475 V 16 w(GROUP)p 884 475 V 18 w(TRANSLA)l(TE)p 1158 475 V 17 w(RANKS)75 543 y FD(5.3.2)64 b(Group)15 b(Const.)125 b Fs(MPI)p 716 543 V 16 w(GROUP)p 884 543 V 18 w(UNION)281 b FD(||)631 600 y Fs(MPI)p 716 600 V 16 w(GROUP)p 884 600 V 18 w(INTERSECTION)631 656 y(MPI)p 716 656 V 16 w(GROUP)p 884 656 V 18 w(DIFFERENCE)631 713 y(MPI)p 716 713 V 16 w(GROUP)p 884 713 V 18 w(INCL)631 769 y(MPI)p 716 769 V 16 w(GROUP)p 884 769 V 18 w(EX)o(CL)631 826 y(MPI)p 716 826 V 16 w(GROUP)p 884 826 V 18 w(RANGE)p 1050 826 V 17 w(INCL)631 882 y(MPI)p 716 882 V 16 w(GROUP)p 884 882 V 18 w(RANGE)p 1050 882 V 17 w(EX)o(CL)75 951 y FD(5.3.3)64 b(Group)15 b(Destruct.)70 b Fs(MPI)p 716 951 V 16 w(GROUP)p 884 951 V 18 w(FREE)313 b FD(||)75 1020 y(5.4.1)64 b(Comm.)19 b(Access.)90 b Fs(MPI)p 716 1020 V 16 w(COMM)p 874 1020 V 16 w(SIZE)341 b FD(||)631 1076 y Fs(MPI)p 716 1076 V 16 w(COMM)p 874 1076 V 16 w(RANK)631 1133 y(MPI)p 716 1133 V 16 w(COMM)p 874 1133 V 16 w(GROUP)75 1202 y FD(5.4.2)64 b(Comm.)19 b(Const.)103 b Fs(MPI)p 716 1202 V 16 w(COMM)p 874 1202 V 16 w(DUP)341 b FD(||)631 1258 y Fs(MPI)p 716 1258 V 16 w(COMM)p 874 1258 V 16 w(MAKE)631 1315 y(MPI)p 716 1315 V 16 w(COMM)p 874 1315 V 16 w(SPLIT)75 1384 y FD(5.4.3)64 b(Comm.)19 b(Destruct.)48 b Fs(MPI)p 716 1384 V 16 w(COMM)p 874 1384 V 16 w(FREE)325 b FD(||)75 1453 y(5.5.1)64 b(Comm.)19 b(Access.)90 b(||)600 b Fs(MPI)p 1406 1453 V 16 w(COMM)p 1564 1453 V 16 w(TEST)p 1694 1453 V 17 w(INTER)75 1521 y FD(5.5.2)64 b(In)o(tercomm.)20 b(Const.)f(||)600 b Fs(MPI)p 1406 1521 V 16 w(INTERCOMM)p 1696 1521 V 16 w(ST)l(ART)264 1578 y FD(&)16 b(Destr.)884 b Fs(MPI)p 1406 1578 V 16 w(INTERCOMM)p 1696 1578 V 16 w(FINISH)1321 1634 y(MPI)p 1406 1634 V 16 w(INTER)p 1554 1634 V 16 w(COMM)p 1712 1634 V 17 w(MERGE)75 1703 y FD(5.5.3)64 b(Name)15 b(Service)128 b(||)600 b Fs(MPI)p 1406 1703 V 16 w(INTERCOMM)p 1696 1703 V 16 w(NAME)p 1841 1703 V 17 w(ST)l(ART)75 1772 y FD(5.6.1)64 b(F)l(unctionalit)o(y)135 b Fs(MPI)p 716 1772 V 16 w(A)l(TTR)p 849 1772 V 17 w(GET)p 954 1772 V 17 w(KEY)264 b FD(||)631 1829 y Fs(MPI)p 716 1829 V 16 w(A)l(TTR)p 849 1829 V 17 w(FREE)p 975 1829 V 17 w(KEY)631 1885 y(MPI)p 716 1885 V 16 w(A)l(TTR)p 849 1885 V 17 w(PUT)p 957 1885 V 16 w(V)l(ALUE)631 1942 y(MPI)p 716 1942 V 16 w(A)l(TTR)p 849 1942 V 17 w(GET)p 954 1942 V 17 w(V)l(ALUE)631 1998 y(MPI)p 716 1998 V 16 w(A)l(TTR)p 849 1998 V 17 w(DELETE)p 1036 1998 V 16 w(V)l(ALUE)75 2161 y Fn(9.3.4)49 b(T)l(op)q(ology)19 b(F)o(unctionalit)o(y)75 2247 y FD(The)d(capabilit)o(y)h(presen)o(ted)f (in)g(the)g(pro)q(cess)f(top)q(ology)h(c)o(hapter)f(represen)o(ts)g(a)h (new)f(functionalit)o(y)i(in)75 2304 y(MPI)i(and)g(is)h(not)f (generally)h(common)f(practice)h(to)q(da)o(y)l(.)31 b(T)l(o)19 b(mak)o(e)f(this)i(a)o(v)m(ailable)h(in)f(the)f(subset,)75 2360 y(the)g(functionalit)o(y)i(asso)q(ciated)e(with)g(Cartesian)g(top) q(ologies)h(is)g(included,)i(but)d(the)h(general)f(graph)75 2416 y(top)q(ologies)d(are)f(excluded.)166 2473 y(The)22 b(table)h(b)q(elo)o(w)g(con)o(tains)f(a)g(listing)i(b)o(y)e(section)h (of)f(whic)o(h)h(routines)f(from)g(the)g(top)q(ology)75 2529 y(c)o(hapter)15 b(are)g(included)j(in/excluded)f(from)e(the)g (subset.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 201 208 201 207 bop 75 -100 a Fp(9.3.)34 b(SUBSET)16 b(FUNCTIONALITY)1036 b FD(201)120 45 y(Section)62 b(Title)403 b(Included)285 b(Excluded)p 75 68 1868 2 v 120 120 a(6.5)146 b(T)l(op)q(ology)15 b(Const.)170 b Fs(MPI)p 908 120 14 2 v 16 w(MAKE)p 1053 120 V 16 w(CART)91 b(MPI)p 1361 120 V 16 w(MAKE)p 1506 120 V 16 w(GRAPH)823 176 y(MPI)p 908 176 V 16 w(MAKE)p 1053 176 V 16 w(DIMS)120 245 y FD(6.5.1)110 b(T)l(op)q(ol.)20 b(inquiry)d(func.)98 b Fs(MPI)p 908 245 V 16 w(TOPO)p 1050 245 V 18 w(ST)l(A)l(TUS)823 302 y(MPI)p 908 302 V 16 w(GET)p 1012 302 V 17 w(CARTDIM)45 b(MPI)p 1361 302 V 16 w(GET)p 1465 302 V 17 w(GRAPHDIMS)823 358 y(MPI)p 908 358 V 16 w(GET)p 1012 358 V 17 w(CART)131 b(MPI)p 1361 358 V 16 w(GET)p 1465 358 V 17 w(GRAPH)823 415 y(MPI)p 908 415 V 16 w(CART)p 1043 415 V 17 w(RANK)96 b(MPI)p 1361 415 V 16 w(GRAPH)p 1527 415 V 18 w(NEIGHBOR)p 1771 415 V 18 w(COUNT)823 471 y(MPI)p 908 471 V 16 w(CART)p 1043 471 V 17 w(COORD)62 b(MPI)p 1361 471 V 16 w(GRAPH)p 1527 471 V 18 w(NEIGHBORS)120 540 y FD(6.5.2)110 b(Cart.)19 b(Shift)d(Co)q(ord.)128 b Fs(MPI)p 908 540 V 16 w(CART)p 1043 540 V 17 w(SHIFT)120 609 y FD(6.5.3)110 b(P)o(art.)19 b(of)14 b(Cart.)19 b(Struct.)65 b Fs(MPI)p 908 609 V 16 w(CART)p 1043 609 V 17 w(SUB)120 678 y FD(6.5.4)110 b(Lo)o(w{lev)o(el)16 b(top)q(ol.)k(func.)57 b Fs(MPI)p 908 678 V 16 w(MAP)p 1023 678 V 17 w(CART)120 b(MPI)p 1361 678 V 16 w(MAP)p 1476 678 V 17 w(GRAPH)75 796 y Fn(9.3.5)49 b(Language)18 b(Binding)75 882 y FD(The)e(subset)f(shall)i (con)o(tain)e(the)g(F)l(ortran)g(77)f(and)i(C)f(language)h(bindings.)22 b(All)16 b(routines)g(included)i(in)75 938 y(the)d(subset)h(will)h (conform)d(to)h(the)g(rules)h(stated)f(in)h(c)o(hapter)f(2)g(and)g (annex)g(A.)75 1056 y Fn(9.3.6)49 b(MPI)17 b(Environmental)e (Management)75 1142 y FD(The)k(abilit)o(y)i(for)d(the)i(user)f(to)g (mak)o(e)f(inquiries)k(of)c(the)i(MPI)f(system)g(is)g(an)g(imp)q(ortan) o(t)g(feature)g(in)75 1198 y(p)q(ortable)f(programming.)26 b(F)l(or)16 b(this)i(reason,)f(most)g(of)g(the)g(functions)h(are)f (included)j(in)f(the)e(subset.)75 1255 y(Ho)o(w)o(ev)o(er,)e(the)i (abilit)o(y)g(for)f(the)g(user)h(to)f(sp)q(ecify)h(an)g(error)e (handler)j(is)f(not)f(included)j(in)e(the)f(subset.)75 1311 y(This)g(is)f(consisten)o(t)h(with)f(not)g(including)j(user)e (called)g(functions)g(in)g(the)g(subset.)166 1368 y(The)e(table)h(b)q (elo)o(w)g(con)o(tains)f(a)g(listing)i(b)o(y)e(section)h(of)f(whic)o(h) h(routines)f(from)g(the)g(MPI)h(En)o(viron-)75 1424 y(men)o(tal)g (Managemen)o(t)f(c)o(hapter)h(are)g(included)j(in/excluded)g(from)c (the)i(subset.)75 1541 y(Section)g(Title)222 b(Included)513 b(Excluded)p 75 1563 1785 2 v 75 1615 a(7.1.1)64 b(En)o(vir.)20 b(Inquires)d Fs(MPI)p 636 1615 14 2 v 16 w(GET)p 740 1615 V 16 w(V)l(ALID)p 883 1615 V 17 w(T)l(A)o(G)p 986 1615 V 17 w(RANGE)84 b FD(||)551 1672 y Fs(MPI)p 636 1672 V 16 w(GET)p 740 1672 V 16 w(PROCESSOR)p 1015 1672 V 19 w(NAME)551 1728 y(MPI)p 636 1728 V 16 w(GET)p 740 1728 V 16 w(HOST)75 1797 y FD(7.1.2)64 b(Bu\013ering)135 b Fs(MPI)p 636 1797 V 16 w(GET)p 740 1797 V 16 w(BUFFER)p 925 1797 V 18 w(P)l(ARAMS)113 b(MPI)p 1317 1797 V 16 w(USER)p 1445 1797 V 17 w(SPECIFIES)p 1676 1797 V 16 w(BUFFER)551 1854 y(MPI)p 636 1854 V 16 w(SUGGEST)p 851 1854 V 17 w(BUFFER)p 1037 1854 V 18 w(P)l(ARAMS)75 1923 y FD(7.2)100 b(Error)14 b(handling)25 b Fs(MPI)p 636 1923 V 16 w(GET)p 740 1923 V 16 w(ERRORMODE)551 1979 y(MPI)p 636 1979 V 16 w(SET)p 735 1979 V 16 w(ERRORMODE)204 b(MPI)p 1317 1979 V 16 w(SET)p 1416 1979 V 16 w(ERRHANDLER)551 2036 y(MPI)p 636 2036 V 16 w(ERROR)p 799 2036 V 18 w(STRING)75 2104 y FD(7.5)100 b(Startup)165 b Fs(MPI)p 636 2104 V 16 w(INIT)494 b FD(||)551 2161 y Fs(MPI)p 636 2161 V 16 w(FINALIZE)551 2217 y(MPI)p 636 2217 V 16 w(INITIALIZED)551 2274 y(MPI)p 636 2274 V 16 w(ABORT)75 2392 y Fn(9.3.7)49 b(Pro\014ling)18 b(F)o(unctionalit)o(y)75 2478 y FD(Pro\014ling)c(is)g (a)e(capabilit)o(y)j(that)d(will)j(b)q(e)f(of)e(great)g(utilit)o(y)j (in)f(MPI)f(for)f(determining)j(the)e(p)q(erformance)75 2534 y(of)g(eac)o(h)g(implemen)o(tation.)20 b(F)l(urthermore,)12 b(although)i(implemen)o(tation)g(of)e(the)h(pro\014ling)i(capabilities) 75 2591 y(doubles)k(the)f(n)o(um)o(b)q(er)g(of)f(routines)h(supplied,)j (these)d(routines)g(can)g(easily)h(b)q(e)f(generated)g(from)f(the)75 2647 y(routines)11 b(required)h(elsewhere)g(in)g(the)f(subset.)18 b(Giv)o(en)11 b(these)g(considerations,)i(the)d(subset)h(will)i(con)o (tain)75 2704 y(the)i(shado)o(w)g(routines)h(\\PMPI)p 635 2704 V 16 w(")e(for)h(eac)o(h)g(routine)h(included)i(in)e(the)f (subset.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 202 209 202 208 bop 75 -100 a FD(202)593 b Fp(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)75 45 y Fq(9.4)59 b(Subset)19 b(T)-5 b(esting)75 147 y FD(The)12 b(test)g(suite)h(is)g(exp)q(ected)g(to)f(include)i(a)e(subset)g(that)g (will)i(v)o(erify)e(the)g(functionalit)o(y)i(of)e(this)g(subset.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 203 210 203 209 bop 75 377 a Fy(Biblio)q(graph)m(y)98 600 y FD([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 656 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 713 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 806 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 863 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 919 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1013 y([3])j(A.)g(Beguelin,)j(J.)d(Dongarra,)f(A.)h(Geist,)h(R.)f(Manc) o(hek,)h(S.)f(Otto,)g(and)h(J.)e(W)l(alp)q(ole.)39 b(PVM:)168 1070 y(Exp)q(eriences,)22 b(curren)o(t)c(status)g(and)h(future)g (direction.)32 b(In)19 b Fo(Sup)n(er)n(c)n(omputing'93)h(Pr)n(o)n(c)n (e)n(e)n(dings)p FD(,)168 1126 y(pages)15 b(765{6,)f(1993.)98 1220 y([4])21 b(A.)14 b(Beguelin,)j(J.)d(Dongarra,)e(A.)i(Geist,)h(R.)f (Manc)o(hek,)g(and)g(V.)g(Sunderam.)19 b(Visualization)e(and)168 1276 y(debugging)g(in)f(a)f(heterogeneous)g(en)o(vironmen)o(t.)20 b Fo(IEEE)15 b(Computer)p FD(,)h(26\(6\):88{95,)11 b(June)17 b(1993.)98 1370 y([5])k(Luc)14 b(Bomans)f(and)h(Rolf)f(Hemp)q(el.)18 b(The)c(Argonne/GMD)e(macros)g(in)i(F)o(OR)l(TRAN)g(for)e(p)q(ortable) 168 1427 y(parallel)20 b(programming)e(and)h(their)f(implemen)o(tation) i(on)e(the)h(In)o(tel)g(iPSC/2.)29 b Fo(Par)n(al)r(lel)19 b(Com-)168 1483 y(puting)p FD(,)c(15:119{132,)d(1990.)98 1577 y([6])21 b(R.)14 b(Butler)h(and)f(E.)f(Lusk.)18 b(User's)13 b(guide)i(to)e(the)h(P4)g(programming)f(system.)k(T)l(ec)o (hnical)f(Rep)q(ort)168 1633 y(TM-ANL{92/17,)e(Argonne)h(National)h (Lab)q(oratory)l(,)e(1992.)98 1727 y([7])21 b(Robin)d(Calkin,)e(Rolf)h (Hemp)q(el,)g(Hans-Christian)g(Hopp)q(e,)g(and)f(P)o(eter)f(Wypior.)22 b(P)o(ortable)16 b(pro-)168 1784 y(gramming)c(with)h(the)f(parmacs)f (message{passing)h(library)l(.)k Fo(Par)n(al)r(lel)c(Computing,)i(sp)n (e)n(cial)f(issue)168 1840 y(on)j(message{p)n(assing)f(interfac)n(es)p FD(,)f(to)g(app)q(ear.)98 1934 y([8])21 b(S.)c(Chittor)g(and)h(R.)f(J.) g(En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d (mesh{connected)i(w)o(ormhole{)168 1990 y(routed)f(net)o(w)o(orks)e (for)i(in)o(terpro)q(cessor)f(comm)o(unication)i(in)f(m)o (ulticomputers.)25 b(In)17 b Fo(Pr)n(o)n(c)n(e)n(e)n(dings)168 2047 y(of)g(the)f(1990)i(Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p FD(,)c(pages)j(647{656,)e(1990.)98 2141 y([9])21 b(S.)f(Chittor)g(and)g (R.)g(J.)f(En)o(b)q(o)q(dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g (of)f(mapping)i(on)e(the)h(comm)o(unica-)168 2197 y(tion)g(p)q (erformance)g(of)f(large)g(m)o(ulticomputers.)34 b(In)20 b Fo(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f(1991)h(International)168 2253 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n (essing,)f(vol.)i(II)f(\(Softwar)n(e\))p FD(,)f(pages)h(I)q(I{1)h({)f (I)q(I{4,)g(1991.)75 2347 y([10])21 b(J.)e(Dongarra,)f(A.)g(Geist,)h (R.)g(Manc)o(hek,)g(and)g(V.)f(Sunderam.)31 b(In)o(tegrated)18 b(PVM)g(framew)o(ork)168 2404 y(supp)q(orts)c(heterogeneous)g(net)o(w)o (ork)f(computing.)18 b Fo(Computers)e(in)e(Physics)p FD(,)f(7\(2\):166{75,)e(April)168 2460 y(1993.)75 2554 y([11])21 b(J.)c(J.)g(Dongarra,)f(R.)h(Hemp)q(el,)i(A.)e(J.)g(G.)f(Hey) l(,)i(and)f(D.)f(W.)h(W)l(alk)o(er.)25 b(A)17 b(prop)q(osal)g(for)g(a)g (user-)168 2610 y(lev)o(el,)c(message)d(passing)h(in)o(terface)g(in)h (a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(hnical)g(Rep) q(ort)168 2667 y(TM-12231,)g(Oak)j(Ridge)g(National)g(Lab)q(oratory)l (,)e(F)l(ebruary)h(1993.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 204 211 204 210 bop 75 -100 a FD(204)1368 b Fp(BIBLIOGRAPHY)75 45 y FD([12])21 b(Edin)o(burgh)g(P)o(arallel)f(Computing)g(Cen)o(tre,)g (Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fo(CHIMP)19 b(Conc)n(epts)p FD(,)168 102 y(June)e(1991.)75 195 y([13])k(Edin)o (burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f(Univ)o(ersit)o(y)h(of)g (Edin)o(burgh.)22 b Fo(CHIMP)15 b(V)m(ersion)h(1.0)168 252 y(Interfac)n(e)p FD(,)e(Ma)o(y)g(1992.)75 346 y([14])21 b(D.)d(F)l(eitelson.)28 b(Comm)o(unicators:)d(Ob)s(ject-based)18 b(m)o(ultipart)o(y)g(in)o(teractions)g(for)g(parallel)h(pro-)168 402 y(gramming.)j(T)l(ec)o(hnical)c(Rep)q(ort)f(91-12,)e(Dept.)g (Computer)h(Science,)i(The)e(Hebrew)g(Univ)o(ersit)o(y)168 459 y(of)f(Jerusalem,)h(No)o(v)o(em)o(b)q(er)f(1991.)75 552 y([15])21 b(D.)16 b(F)l(ry)o(e,)g(R.)h(Bry)o(an)o(t,)f(H.)g(Ho,)g (R.)h(La)o(wrence,)g(and)f(M.)g(Snir.)25 b(An)17 b(external)g(user)g (in)o(terface)g(for)168 609 y(scalable)g(parallel)g(systems.)i(T)l(ec)o (hnical)e(rep)q(ort,)d(IBM,)i(Ma)o(y)e(1992.)75 703 y([16])21 b(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o(eyton,)f(and)i (P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f(PICL:)g (a)168 759 y(p)q(ortable)f(instrumen)o(ted)f(comm)o(unication)h (library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e(TM-11616,)f(Oak)h(Ridge) 168 816 y(National)k(Lab)q(oratory)l(,)e(Octob)q(er)i(1990.)75 909 y([17])21 b(O.)c(Kr\177)-23 b(amer)16 b(and)h(H.)g(M)q(\177)-24 b(uhlen)o(b)q(ein.)26 b(Mapping)17 b(strategies)g(in)g(message{based)f (m)o(ultipro)q(cessor)168 966 y(systems.)k Fo(Par)n(al)r(lel)15 b(Computing)p FD(,)g(9:213{225,)d(1989.)75 1060 y([18])21 b(nCUBE)16 b(Corp)q(oration.)j Fo(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e (Guide,)i(r2.0)p FD(,)f(Decem)o(b)q(er)f(1990.)75 1153 y([19])21 b(P)o(arasoft)14 b(Corp)q(oration.)22 b Fo(Expr)n(ess)16 b(V)m(ersion)g(1.0:)22 b(A)17 b(Communic)n(ation)g(Envir)n(onment)e (for)j(Par-)168 1210 y(al)r(lel)e(Computers)p FD(,)f(1988.)75 1304 y([20])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g (system.)22 b(In)17 b Fo(Pr)n(o)n(c)n(e)n(e)n(dings)e(of)i(the)g(Thir)n (d)g(Confer)n(enc)n(e)e(on)168 1360 y(Hyp)n(er)n(cub)n(e)g(Concurr)n (ent)f(Computers)i(and)f(Applic)n(ations)p FD(,)e(pages)h(384{390.)d(A) o(CM)j(Press,)f(1988.)75 1454 y([21])21 b(A.)14 b(Skjellum)i(and)e(A.)g (Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f (comm)o(unication)h(library)168 1510 y(atop)g(the)f(reactiv)o(e)h(k)o (ernel.)20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g (editors,)g Fo(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 1567 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g (Confer)n(enc)n(e)p FD(,)e(pages)i(767{776.)d(IEEE)168 1623 y(Press,)d(1990.)75 1717 y([22])21 b(A.)12 b(Skjellum,)i(S.)e (Smith,)h(C.)e(Still,)j(A.)d(Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q (co)q(de)g(message)f(passing)168 1774 y(system.)20 b(T)l(ec)o(hnical)d (rep)q(ort,)d(La)o(wrence)i(Liv)o(ermore)f(National)h(Lab)q(oratory)l (,)e(Septem)o(b)q(er)i(1992.)75 1867 y([23])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g(passing)g(in)i(a)e (distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(h-)168 1924 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National)g(Lab)q (oratory)l(,)e(August)h(1992.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 205 212 205 211 bop 75 356 a Fr(Annex)33 b(A)75 564 y Fy(Language)41 b(Binding)75 805 y Fq(A.1)59 b(Intro)r(duction)75 906 y FD(In)15 b(this)f(section)h(w)o(e)f(summarize)g(the)h(sp)q(eci\014c)h (bindings)g(for)d(b)q(oth)h(F)l(ortran)f(and)h(C.)g(W)l(e)g(presen)o(t) g(\014rst)75 963 y(the)h(C)g(bindings,)i(then)e(the)h(F)l(ortran)e (bindings.)22 b(Listings)16 b(are)f(alphab)q(etical)i(within)f(c)o (hapter.)75 1106 y Fq(A.2)59 b(De\014ned)19 b(Constants)g(fo)n(r)i(C)f (and)f(F)n(o)n(rtran)75 1207 y FD(These)e(are)g(required)h(de\014ned)h (constan)o(ts,)d(to)g(b)q(e)i(de\014ned)h(in)e(the)h(\014les)g FB(mpi.h)e FD(\(for)g(C\))h(and)g FB(mpif.h)75 1264 y FD(\(for)d(F)l(ortran\).)75 1370 y FB(/*)24 b(return)f(codes)g(*/)75 1427 y(MPI_SUCCESS)75 1539 y(/*)h(assorted)e(constants)h(\(both)g(C)h (and)f(Fortran\))g(*/)75 1596 y(MPI_BOTTOM)75 1652 y(MPI_PROCNULL)75 1709 y(MPI_ANY_SOURCE)75 1765 y(MPI_ANY_TAG)75 1822 y(MPI_UNDEFINED)75 1878 y(MPI_UNDEFINED_RANK)75 1935 y(MPI_UB)75 1991 y(MPI_LB)75 2104 y(/*)h(constant)e(array)i(sizes)f(\(Fortran\))f(*/)75 2160 y(MPI_STATUS_SIZE)75 2273 y(/*)i(attributes)e(\(C)i(and)f (Fortran\))g(*/)75 2330 y(MPI_UNDEFINED_ATTR)75 2386 y(MPI_FOUND)75 2443 y(MPI_NOT_FOUND)75 2499 y(MPI_ATTR_COPY_FREELY)75 2556 y(MPI_ATTR_COPY_NEVER)75 2612 y(MPI_ATTRCOPY_CALLBACK)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 206 213 206 212 bop 75 -100 a FD(206)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y FB(/*)24 b(Error-handling)e (specifiers)g(\(C)i(and)f(Fortran\))g(*/)75 102 y(MPI_ERRORSFATAL)75 158 y(MPI_ERRORSRETURN)75 214 y(MPI_ERRORSCALLUSERROUTINE)75 327 y(/*)h(Pre-defined)e(error)h(handlers)g(\(C)g(and)h(Fortran\))f(*/) 75 384 y(MPI_NOHANDLER)75 440 y(MPI_DEFAULTHANDLER)75 553 y(/*)h(Maximum)e(sizes)i(for)f(strings)g(*/)75 610 y(MPI_MAX_INTERCOMM_NAME_SIZ)o(E)75 723 y(/*)h(elementary)e(datatypes)h (\(C\))g(*/)75 779 y(MPI_CHAR)75 835 y(MPI_SHORT)75 892 y(MPI_INT)75 948 y(MPI_LONG)75 1005 y(MPI_FLOAT)75 1061 y(MPI_DOUBLE)75 1118 y(MPI_BYTE)75 1174 y(MPI_UNSIGNED_CHAR)75 1231 y(MPI_UNSIGNED_SHORT)75 1287 y(MPI_UNSIGNED_LONG)75 1344 y(MPI_UNSIGNED_INT)75 1456 y(/*)h(elementary)e(datatypes)h (\(Fortran\))f(*/)75 1513 y(MPI_INTEGER)75 1569 y(MPI_REAL)75 1626 y(MPI_DOUBLE_PRECISION)75 1682 y(MPI_COMPLEX)75 1739 y(MPI_DOUBLE_COMPLEX)75 1795 y(MPI_LOGICAL)75 1852 y(MPI_CHARACTER)75 1908 y(MPI_BYTE)75 2021 y(/*)i(reserved)e (communicators)g(\(C)i(and)f(Fortran\))g(*/)75 2077 y(MPI_COMM_WORLD)75 2190 y(/*)h(collective)e(operations)h(\(C)g(and)h(Fortran\))e(*/)75 2247 y(MPI_MAX)75 2303 y(MPI_MIN)75 2360 y(MPI_SUM)75 2416 y(MPI_PROD)75 2473 y(MPI_AND)75 2529 y(MPI_OR)75 2586 y(MPI_XOR)75 2642 y(MPI_MAXLOC)75 2698 y(MPI_MINLOC)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 207 214 207 213 bop 75 -100 a Fp(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b FD(207)75 102 y FB(/*)24 b(topologies)e(\(C)i(and)f(Fortran\))g(*/)75 158 y(MPI_REORDER)75 214 y(MPI_NOREORDER)75 271 y(MPI_GRAPHTOP)75 327 y(MPI_CARTTOP)75 384 y(MPI_NOTOP)75 440 y(MPI_PER)75 497 y(MPI_NOPER)166 659 y FD(The)15 b(follo)o(wing)h(are)f(de\014ned)i (C)e(t)o(yp)q(e)g(de\014nitions,)i(also)e(included)j(in)e(the)f(\014le) h FB(mpi.h)p FD(.)75 766 y FB(/*)24 b(opaque)f(types)g(\(C\))g(*/)75 822 y(MPI_Persistence)75 879 y(MPI_Mode)75 935 y(MPI_Op)75 992 y(MPI_Comm_name)75 1048 y(MPI_Aint)75 1104 y(MPI_Errmode)75 1217 y(/*)h(assorted)e(opaque)h(structures)g(\(C\))g(*/)75 1274 y(MPI_Status)75 1330 y(MPI_Group)75 1387 y(MPI_Communicator)75 1443 y(MPI_Datatype)75 1500 y(MPI_Comm_request)75 1556 y(MPI_Attribute)75 1613 y(MPI_Intercomm_request)75 1756 y Fq(A.3)59 b(C)20 b(bindings)g(fo)n(r)h(P)n(oint-to-P)n(oint)e (Communication)75 1857 y FD(These)d(are)e(presen)o(ted)i(here)g(in)g (the)f(order)g(of)g(their)g(app)q(earance)h(in)g(the)g(c)o(hapter.)75 1914 y FB(int)23 b(MPI)p 245 1914 15 2 v 17 w(Send\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 1914 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 1970 y(int)h(tag,)f(MPI)p 683 1970 V 17 w(Comm)g(comm\))75 2057 y(int)g(MPI)p 245 2057 V 17 w(Recv\(void*)g(buf,)g(int)g(count,)h (MPI)p 979 2057 V 16 w(Datatype)f(datatype,)g(int)g(source,)393 2113 y(int)h(tag,)f(MPI)p 683 2113 V 17 w(Comm)g(comm,)g(MPI)p 1034 2113 V 17 w(Status)g(*status\))75 2200 y(int)g(MPI)p 245 2200 V 17 w(Get)p 334 2200 V 17 w(source\(MPI)p 591 2200 V 16 w(Status)g(status,)g(int)g(*source\))75 2286 y(int)g(MPI)p 245 2286 V 17 w(Get)p 334 2286 V 17 w(tag\(MPI)p 519 2286 V 16 w(Status)g(status,)g(int)h(*tag\))75 2372 y(int)f(MPI)p 245 2372 V 17 w(Get)p 334 2372 V 17 w(count\(MPI)p 567 2372 V 16 w(Status)g(status,)g(MPI)p 1013 2372 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 2459 y(int)f(MPI)p 245 2459 V 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2459 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2515 y(int)h(tag,)f(MPI)p 683 2515 V 17 w(Comm)g(comm\))75 2602 y(int)g(MPI)p 245 2602 V 17 w(Ssend\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 2602 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2658 y(int)h(tag,)f(MPI)p 683 2658 V 17 w(Comm)g(comm\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 208 215 208 214 bop 75 -100 a FD(208)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y FB(int)23 b(MPI)p 245 45 15 2 v 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 45 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 102 y(int)h(tag,)f(MPI)p 683 102 V 17 w(Comm)g(comm,)g(MPI)p 1034 102 V 17 w(Comm)p 1147 102 V 17 w(request)g(request\))75 188 y(int)g(MPI)p 245 188 V 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g (MPI)p 1026 188 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 244 y(int)g(tag,)f(MPI)p 683 244 V 17 w(Comm)g(comm,)g(MPI)p 1034 244 V 17 w(Comm)p 1147 244 V 17 w(request)g(request\))75 331 y(int)g(MPI)p 245 331 V 17 w(Issend\(void*)f(buf,)i(int)f(count,)g (MPI)p 1026 331 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 387 y(int)g(tag,)f(MPI)p 683 387 V 17 w(Comm)g(comm,)g(MPI)p 1034 387 V 17 w(Comm)p 1147 387 V 17 w(request)g(request\))75 474 y(int)g(MPI)p 245 474 V 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 474 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 530 y(int)h(tag,)f(MPI)p 683 530 V 17 w(Comm)g(comm,)g(MPI)p 1034 530 V 17 w(Comm)p 1147 530 V 17 w(request)g(request\))75 617 y(int)g(MPI)p 245 617 V 17 w(Wait\(MPI)p 454 617 V 16 w(Comm)p 566 617 V 17 w(request)g(*request,)f(MPI)p 1084 617 V 17 w(Status)h(*status\))75 703 y(int)g(MPI)p 245 703 V 17 w(Test\(MPI)p 454 703 V 16 w(Comm)p 566 703 V 17 w(request)g(*request,)f(int)i(flag,)f(MPI)p 1323 703 V 17 w(Status)g(*status\))75 790 y(int)g(MPI)p 245 790 V 17 w(Waitany\(int)g(count,)g(MPI)p 788 790 V 16 w(Comm)p 900 790 V 17 w(request)g(*array)p 1252 790 V 16 w(of)p 1316 790 V 17 w(requests,)g(int)g(*index,)393 846 y(MPI)p 468 846 V 17 w(Status)g(*status\))75 933 y(int)g(MPI)p 245 933 V 17 w(Testany\(int)g(count,)g(MPI)p 788 933 V 16 w(Comm)p 900 933 V 17 w(request)g(*array)p 1252 933 V 16 w(of)p 1316 933 V 17 w(requests,)g(int)g(*index,)393 989 y(int)h(*flag,)f(MPI)p 731 989 V 16 w(Status)g(*status\))75 1075 y(int)g(MPI)p 245 1075 V 17 w(Waitall\(int)g(count,)g(MPI)p 788 1075 V 16 w(Comm)p 900 1075 V 17 w(request)g(*array)p 1252 1075 V 16 w(of)p 1316 1075 V 17 w(requests,)393 1132 y(MPI)p 468 1132 V 17 w(Status)g(*array)p 796 1132 V 16 w(of)p 860 1132 V 17 w(statuses\))75 1218 y(int)g(MPI)p 245 1218 V 17 w(Testall\(int)g(count,)g(MPI)p 788 1218 V 16 w(Comm)p 900 1218 V 17 w(request)g(*array)p 1252 1218 V 16 w(of)p 1316 1218 V 17 w(requests,)g(int)g(*flag,)393 1275 y(MPI)p 468 1275 V 17 w(Status)g(*array)p 796 1275 V 16 w(of)p 860 1275 V 17 w(statuses\))75 1361 y(int)g(MPI)p 245 1361 V 17 w(Iprobe\(int)g(source,)g(int)g(tag,)g(MPI)p 1002 1361 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 1418 y(MPI)p 468 1418 V 17 w(Status)g(*status\))75 1504 y(int)g(MPI)p 245 1504 V 17 w(Probe\(int)g(source,)g(int)g(tag,)h(MPI)p 979 1504 V 16 w(Comm)g(comm,)f(MPI)p 1330 1504 V 17 w(Status)g (*status\))75 1591 y(int)g(MPI)p 245 1591 V 17 w(Cancel\(MPI)p 502 1591 V 16 w(Comm)p 614 1591 V 17 w(request)g(request\))75 1677 y(int)g(MPI)p 245 1677 V 17 w(Test)p 358 1677 V 17 w(cancelled\(MPI)p 687 1677 V 15 w(Status)g(status,)g(int)h(*flag\)) 75 1764 y(int)f(MPI)p 245 1764 V 17 w(Create)p 406 1764 V 17 w(send\(void*)f(buf,)h(int)h(count,)f(MPI)p 1139 1764 V 17 w(Datatype)f(datatype,)h(int)h(dest,)393 1820 y(int)g(tag,)f(MPI)p 683 1820 V 17 w(Comm)g(comm,)g(MPI)p 1034 1820 V 17 w(Comm)p 1147 1820 V 17 w(request)g(request\))75 1907 y(int)g(MPI)p 245 1907 V 17 w(Create)p 406 1907 V 17 w(rsend\(void*)f(buf,)h(int)h(count,)f(MPI)p 1163 1907 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 1963 y(int)h(tag,)f(MPI)p 683 1963 V 17 w(Comm)g(comm,)g(MPI)p 1034 1963 V 17 w(Comm)p 1147 1963 V 17 w(request)g(request\))75 2049 y(int)g(MPI)p 245 2049 V 17 w(Create)p 406 2049 V 17 w(ssend\(void*)f(buf,)h(int)h(count,)f(MPI)p 1163 2049 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 2106 y(int)h(tag,)f(MPI)p 683 2106 V 17 w(Comm)g(comm,)g(MPI)p 1034 2106 V 17 w(Comm)p 1147 2106 V 17 w(request)g(request\))75 2192 y(int)g(MPI)p 245 2192 V 17 w(Create)p 406 2192 V 17 w(recv\(void*)f(buf,)h(int)h(count,)f(MPI)p 1139 2192 V 17 w(Datatype)f(datatype,)h(int)h(source,)393 2249 y(int)g(tag,)f(MPI)p 683 2249 V 17 w(Comm)g(comm,)g(MPI)p 1034 2249 V 17 w(Comm)p 1147 2249 V 17 w(request)g(request\))75 2335 y(int)g(MPI)p 245 2335 V 17 w(Start\(MPI)p 478 2335 V 16 w(Comm)p 590 2335 V 17 w(request)g(request\))75 2422 y(int)g(MPI)p 245 2422 V 17 w(Startall\(int)f(count,)h(MPI)p 811 2422 V 17 w(Comm)p 924 2422 V 17 w(request)g(*array)p 1276 2422 V 16 w(of)p 1340 2422 V 17 w(requests\))75 2508 y(int)g(MPI)p 245 2508 V 17 w(Request)p 430 2508 V 16 w(free\(MPI)p 638 2508 V 17 w(Comm)p 751 2508 V 16 w(request)g(request\))75 2595 y(int)g(MPI)p 245 2595 V 17 w(Sendrecv\(void)f(*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 2595 V 17 w(Datatype)f(sendtype,)393 2651 y(int)i(dest,)f(int)g (sendtag,)g(void)g(*recvbuf,)g(int)g(recvcount,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 209 216 209 215 bop 75 -100 a Fp(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b FD(209)393 45 y FB(MPI)p 468 45 15 2 v 17 w(Datatype)23 b(recvtype,)f(int)i(source,)f (MPI)p 1297 45 V 16 w(Datatype)g(recvtag,)393 102 y(MPI)p 468 102 V 17 w(Comm)g(comm,)g(MPI)p 819 102 V 17 w(Status)g(*status\)) 75 188 y(int)g(MPI)p 245 188 V 17 w(Sendrecv)p 454 188 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 188 V 17 w(Datatype)g(datatype,)393 244 y(int)h(dest,)f(int)g(sendtag,) g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 244 V 16 w(Comm)h(comm,)393 301 y(MPI)p 468 301 V 17 w(Status)f(*status\))75 387 y(int)g(MPI)p 245 387 V 17 w(Type)p 358 387 V 17 w(contiguous\(int)f(count,)h(MPI)p 972 387 V 17 w(Datatype)f(oldtype,) 393 444 y(MPI)p 468 444 V 17 w(Datatype)h(*newtype\))75 530 y(int)g(MPI)p 245 530 V 17 w(Type)p 358 530 V 17 w(vector\(int)f(count,)h(int)h(blocklength,)e(int)i(stride,)393 587 y(MPI)p 468 587 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 587 V 17 w(Datatype)h(*newtype\))75 673 y(int)g(MPI)p 245 673 V 17 w(Type)p 358 673 V 17 w(hvector\(int)f(count,)h(int)h (blocklength,)e(MPI)p 1306 673 V 17 w(Aint)h(stride,)393 730 y(MPI)p 468 730 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 730 V 17 w(Datatype)h(*newtype\))75 816 y(int)g(MPI)p 245 816 V 17 w(Type)p 358 816 V 17 w(indexed\(int)f(count,)h(int)h (*array)p 1068 816 V 16 w(of)p 1132 816 V 17 w(blocklengths,)393 873 y(int)g(*array)p 636 873 V 16 w(of)p 700 873 V 17 w(displacements,)e(MPI)p 1147 873 V 17 w(Datatype)g(oldtype,)393 929 y(MPI)p 468 929 V 17 w(Datatype)h(*newtype\))75 1015 y(int)g(MPI)p 245 1015 V 17 w(Type)p 358 1015 V 17 w(hindexed\(int)f (count,)h(int)h(*array)p 1092 1015 V 16 w(of)p 1156 1015 V 17 w(blocklengths,)393 1072 y(MPI)p 468 1072 V 17 w(Aint)f(*array)p 748 1072 V 17 w(of)p 813 1072 V 17 w(displacements,)e(MPI)p 1259 1072 V 17 w(Datatype)i(oldtype,)393 1128 y(MPI)p 468 1128 V 17 w(Datatype)g(*newtype\))75 1215 y(int)g(MPI)p 245 1215 V 17 w(Type)p 358 1215 V 17 w(struct\(int)f(count,)h(int)h (*array)p 1044 1215 V 16 w(of)p 1108 1215 V 17 w(blocklengths,)393 1271 y(MPI)p 468 1271 V 17 w(Aint)f(*array)p 748 1271 V 17 w(of)p 813 1271 V 17 w(displacements,)e(MPI)p 1259 1271 V 17 w(Datatype)i(*array)p 1635 1271 V 16 w(of)p 1699 1271 V 17 w(types,)393 1328 y(MPI)p 468 1328 V 17 w(Datatype)g(*newtype\))75 1414 y(int)g(MPI)p 245 1414 V 17 w(Address\(void*)f(location,)h(MPI)p 907 1414 V 17 w(Aint)g(address\))75 1501 y(int)g(MPI)p 245 1501 V 17 w(Type)p 358 1501 V 17 w(extent\(MPI)p 615 1501 V 16 w(Datatype)f(datatype,)h(MPI)p 1156 1501 V 17 w(Aint)g(*extent\)) 75 1587 y(int)g(MPI)p 245 1587 V 17 w(Type)p 358 1587 V 17 w(size\(MPI)p 567 1587 V 16 w(Datatype)g(datatype,)f(MPI)p 1108 1587 V 17 w(Aint)h(*size\))75 1674 y(int)g(MPI)p 245 1674 V 17 w(Type)p 358 1674 V 17 w(count\(MPI)p 591 1674 V 16 w(Datatype)g(datatype,)f(int)i(*count\))75 1760 y(int)f(MPI)p 245 1760 V 17 w(Type)p 358 1760 V 17 w(commit\(MPI)p 615 1760 V 16 w(Datatype)f(*datatype\))75 1846 y(int)h(MPI)p 245 1846 V 17 w(Type)p 358 1846 V 17 w(free\(MPI)p 567 1846 V 16 w(Datatype)g(*datatype\))75 1933 y(int)g(MPI)p 245 1933 V 17 w(Get)p 334 1933 V 17 w(elements\(MPI)p 639 1933 V 16 w(Status)g(status,)f(MPI)p 1084 1933 V 17 w(Datatype)h(datatype,)g(int)g(*count\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 210 217 210 216 bop 75 -100 a FD(210)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fq(A.4)59 b(C)20 b(Bindings)g(fo)n (r)h(Collective)e(Communication)75 147 y FB(int)k(MPI)p 245 147 15 2 v 17 w(Barrier\(MPI)p 526 147 V 16 w(Comm)g(comm)h(\))75 233 y(int)47 b(MPI)p 269 233 V 17 w(Bcast\(void*)22 b(buf,)i(int)f (count,)g(MPI)p 1026 233 V 17 w(Datatype)g(datatype,)f(int)i(root,)393 289 y(MPI)p 468 289 V 17 w(Comm)f(comm)h(\))75 376 y(int)f(MPI)p 245 376 V 17 w(Gather\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1217 376 V 17 w(Datatype)h(sendtype,)393 432 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 432 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 489 y(MPI)p 468 489 V 17 w(Comm)f(comm\))75 575 y(int)g(MPI)p 245 575 V 17 w(Gatherv\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 575 V 17 w(Datatype)h(sendtype,)393 632 y(void*)g(recvbuf,)g(int)h (*recvcounts,)e(int)h(*displs,)393 688 y(MPI)p 468 688 V 17 w(Datatype)g(recvtype,)f(int)i(root,)f(MPI)p 1249 688 V 17 w(Comm)g(comm\))75 775 y(int)g(MPI)p 245 775 V 17 w(Scatter\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 775 V 17 w(Datatype)h(sendtype,)393 831 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 831 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 888 y(MPI)p 468 888 V 17 w(Comm)f(comm\))75 974 y(int)g(MPI)p 245 974 V 17 w(Scatterv\(void*)f(sendbuf,)h(int)g(*sendcounts,)g(int)g (*displs,)393 1030 y(MPI)p 468 1030 V 17 w(Datatype)g(sendtype,)f (void*)i(recvbuf,)e(int)i(recvcount,)393 1087 y(MPI)p 468 1087 V 17 w(Datatype)f(recvtype,)f(int)i(root,)f(MPI)p 1249 1087 V 17 w(Comm)g(comm\))75 1173 y(int)g(MPI)p 245 1173 V 17 w(Allgather\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1289 1173 V 17 w(Datatype)f(sendtype,)393 1230 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1230 V 17 w(Datatype)h(recvtype,)393 1286 y(MPI)p 468 1286 V 17 w(Comm)g(comm\))75 1373 y(int)g(MPI)p 245 1373 V 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1373 V 16 w(Datatype)g(sendtype,)393 1429 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1486 y(MPI)p 468 1486 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1486 V 17 w(Comm)h(comm\))75 1572 y(int)g(MPI)p 245 1572 V 17 w(Alltoall\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1572 V 17 w(Datatype)f(sendtype,)393 1629 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1629 V 17 w(Datatype)h(recvtype,)393 1685 y(MPI)p 468 1685 V 17 w(Comm)g(comm\))75 1772 y(int)g(MPI)p 245 1772 V 17 w(Alltoallv\(void*)f(sendbuf,)h(int)g(*sendcounts,)f(int) i(*sdispls,)393 1828 y(MPI)p 468 1828 V 17 w(Datatype)f(sendtype,)f (void*)i(recvbuf,)e(int)i(*recvcounts,)393 1884 y(int)g(*rdispls,)e (MPI)p 802 1884 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1884 V 17 w(Comm)i(comm\))75 1971 y(int)f(MPI)p 245 1971 V 17 w(Reduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)h(count,)393 2027 y(MPI)p 468 2027 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2027 V 17 w(Op)i(op,)f(int)h(root,)f(MPI)p 1505 2027 V 17 w(Comm)g(comm\))75 2114 y(int)g(MPI)p 245 2114 V 17 w(User)p 358 2114 V 17 w(reduce\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)g(count,)393 2170 y(MPI)p 468 2170 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(int)i(root,)f(MPI)p 1607 2170 V 17 w(Comm)g(comm\))75 2257 y(int)g(MPI)p 245 2257 V 17 w(User)p 358 2257 V 17 w(reducea\(void*)f(sendbuf,)h (void*)g(recvbuf,)g(int)g(count,)393 2313 y(MPI)p 468 2313 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(int)i(root,)f (MPI)p 1607 2313 V 17 w(Comm)g(comm\))75 2400 y(int)g(MPI)p 245 2400 V 17 w(Allreduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g (count,)393 2456 y(MPI)p 468 2456 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2456 V 17 w(Op)i(op,)f(MPI)p 1266 2456 V 17 w(Comm)g(comm\))75 2543 y(int)g(MPI)p 245 2543 V 17 w(User)p 358 2543 V 17 w(allreduce\(void*)f(sendbuf,)g(void*)i(recvbuf,)e(int)i(count,)393 2599 y(MPI)p 468 2599 V 17 w(Datatype)f(datatype,)f(void)i(function,)e (MPI)p 1368 2599 V 17 w(Comm)h(comm\))75 2685 y(int)g(MPI)p 245 2685 V 17 w(User)p 358 2685 V 17 w(allreducea\(void*)f(sendbuf,)g (void*)h(recvbuf,)g(int)h(count,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 211 218 211 217 bop 75 -100 a Fp(A.5.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(GR)o (OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)160 b FD(211)393 45 y FB(MPI)p 468 45 15 2 v 17 w(Datatype)23 b(datatype,)f(void)i (function,)e(MPI)p 1368 45 V 17 w(Comm)h(comm\))75 132 y(int)g(MPI)p 245 132 V 17 w(Reduce)p 406 132 V 17 w(scatter\(void*)f (sendbuf,)g(void*)i(recvbuf,)e(int)i(*recvcounts,)393 188 y(MPI)p 468 188 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 188 V 17 w(Op)i(op,)f(MPI)p 1266 188 V 17 w(Comm)g(comm\))75 274 y(int)g(MPI)p 245 274 V 17 w(Scan\(void*)g(sendbuf,)f(void*)i (recvbuf,)e(int)i(count,)393 331 y(MPI)p 468 331 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 331 V 17 w(Op)i(op,)f(MPI)p 1266 331 V 17 w(Comm)g(comm)h(\))75 417 y(int)f(MPI)p 245 417 V 17 w(User)p 358 417 V 17 w(scan\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)h(count,)393 474 y(MPI)p 468 474 V 17 w(Datatype)f(datatype,)f(void)i(function,)e(MPI)p 1368 474 V 17 w(Comm)h(comm\))75 560 y(int)g(MPI)p 245 560 V 17 w(User)p 358 560 V 17 w(scana\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)g(count,)393 617 y(MPI)p 468 617 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(MPI)p 1368 617 V 17 w(Comm)h(comm\))75 790 y Fq(A.5)59 b(C)20 b(Bindings)g(fo)n(r) h(Groups,)e(Contexts,)f(and)h(Communicato)n(rs)75 891 y FB(int)k(MPI)p 245 891 V 17 w(Group)p 382 891 V 17 w(size\(MPI)p 591 891 V 16 w(Group)g(group,)g(int)g(*size\))75 978 y(int)g(MPI)p 245 978 V 17 w(Group)p 382 978 V 17 w(rank\(MPI)p 591 978 V 16 w(Group)g(group,)g(int)g(*rank\))75 1064 y(int)g(MPI)p 245 1064 V 17 w(Group)p 382 1064 V 17 w(translate)p 615 1064 V 16 w(ranks)g(\(MPI)p 870 1064 V 16 w(Group)h(group1,)f(int)g(n,)h(int)f(*ranks1,)393 1121 y(MPI)p 468 1121 V 17 w(Group)g(group2,)g(int)g(*ranks2\))75 1207 y(int)g(MPI)p 245 1207 V 17 w(Group)p 382 1207 V 17 w(union\(MPI)p 615 1207 V 16 w(Group)g(group1,)g(MPI)p 1037 1207 V 17 w(Group)g(group2,)g(MPI)p 1460 1207 V 16 w(Group)h(*newgroup\))75 1294 y(int)f(MPI)p 245 1294 V 17 w(Group)p 382 1294 V 17 w(intersection\(MPI)p 783 1294 V 15 w(Group)g(group1,)g(MPI)p 1204 1294 V 17 w(Group)g(group2,) 393 1350 y(MPI)p 468 1350 V 17 w(Group)g(*newgroup\))75 1437 y(int)g(MPI)p 245 1437 V 17 w(Group)p 382 1437 V 17 w(difference\(MPI)p 735 1437 V 15 w(Group)g(group1,)g(MPI)p 1156 1437 V 17 w(Group)g(group2,)393 1493 y(MPI)p 468 1493 V 17 w(Group)g(*newgroup\))75 1580 y(int)g(MPI)p 245 1580 V 17 w(Group)p 382 1580 V 17 w(incl\(MPI)p 591 1580 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1580 V 17 w(Group)h(*newgroup\))75 1666 y(int)g(MPI)p 245 1666 V 17 w(Group)p 382 1666 V 17 w(excl\(MPI)p 591 1666 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1666 V 17 w(Group)h(*newgroup\))75 1752 y(int)g(MPI)p 245 1752 V 17 w(Group)p 382 1752 V 17 w(range)p 519 1752 V 16 w(incl\(MPI)p 727 1752 V 16 w(Group)g(group,)h(int)f(n,)h(int)f (**ranges,)393 1809 y(MPI)p 468 1809 V 17 w(Group)g(*newgroup\))75 1895 y(int)g(MPI)p 245 1895 V 17 w(Group)p 382 1895 V 17 w(range)p 519 1895 V 16 w(excl\(MPI)p 727 1895 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 1952 y(MPI)p 468 1952 V 17 w(Group)g(*newgroup\))75 2038 y(int)g(MPI)p 245 2038 V 17 w(Group)p 382 2038 V 17 w(free\(MPI)p 591 2038 V 16 w(Group)g(*group\))75 2125 y(int)g(MPI)p 245 2125 V 17 w(Comm)p 358 2125 V 17 w(size\(MPI)p 567 2125 V 16 w(Comm)g(comm,)g(int)h(*size\))75 2211 y(int)f(MPI)p 245 2211 V 17 w(Comm)p 358 2211 V 17 w(rank\(MPI)p 567 2211 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 2298 y(int)f(MPI)p 245 2298 V 17 w(Comm)p 358 2298 V 17 w(group\(MPI)p 591 2298 V 16 w(Comm)g(comm,)g(MPI)p 941 2298 V 17 w(Group)g(*group\))75 2384 y(int)g(MPI)p 245 2384 V 17 w(Comm)p 358 2384 V 17 w(dup\(MPI)p 543 2384 V 16 w(Comm)g(comm,)h(MPI)p 894 2384 V 16 w(Comm)g(*newcomm\))75 2471 y(int)f(MPI)p 245 2471 V 17 w(Comm)p 358 2471 V 17 w(make\(MPI)p 567 2471 V 16 w(Comm)g(comm,)g(MPI)p 917 2471 V 17 w(Group)g(group,)g(MPI)p 1316 2471 V 17 w(Comm)h(*newcomm\))75 2557 y(int)f(MPI)p 245 2557 V 17 w(Comm)p 358 2557 V 17 w(split\(MPI)p 591 2557 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 2557 V 16 w(Comm)g(*newcomm\))75 2643 y(int)f(MPI)p 245 2643 V 17 w(Comm)p 358 2643 V 17 w(free\(MPI)p 567 2643 V 16 w(Comm)g(*comm\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 212 219 212 218 bop 75 -100 a FD(212)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y FB(int)23 b(MPI)p 245 45 15 2 v 17 w(Comm)p 358 45 V 17 w(test)p 471 45 V 16 w(inter\(MPI)p 703 45 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 132 y(int)g(MPI)p 245 132 V 17 w(Intercomm)p 478 132 V 16 w(start\(MPI)p 710 132 V 16 w(Comm)g(local)p 965 132 V 17 w(comm,)g(int)h(local)p 1341 132 V 16 w(leader,)393 188 y(MPI)p 468 188 V 17 w(Comm)f(peer)p 700 188 V 17 w(comm,)g(int)h(remote)p 1100 188 V 16 w(leader,)f(int)g(tag,)393 244 y(MPI)p 468 244 V 17 w(Intercomm)p 701 244 V 16 w(request)g(*inter) p 1052 244 V 16 w(request\))75 331 y(int)g(MPI)p 245 331 V 17 w(Intercomm)p 478 331 V 16 w(finish\(MPI)p 734 331 V 16 w(Intercomm)p 966 331 V 16 w(request)g(*inter)p 1317 331 V 16 w(request,)393 387 y(MPI)p 468 387 V 17 w(Comm)g(*newcomm\))75 474 y(int)g(MPI)p 245 474 V 17 w(Intercomm)p 478 474 V 16 w(merge\(MPI)p 710 474 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 530 y(MPI)p 468 530 V 17 w(Comm)g(*newintracomm\))75 617 y(int)g(MPI)p 245 617 V 17 w(Intercomm)p 478 617 V 16 w(name)p 590 617 V 17 w(start\(MPI)p 823 617 V 16 w(Comm)g(comm,)g(char)h(*name,)393 673 y(MPI)p 468 673 V 17 w(Intercomm)p 701 673 V 16 w(request)f(*inter) p 1052 673 V 16 w(request\))75 760 y(int)g(MPI)p 245 760 V 17 w(Attr)p 358 760 V 17 w(create)p 519 760 V 16 w(key\(int)g(copy)p 822 760 V 17 w(fn\(\),)g(int)g(delete)p 1221 760 V 17 w(fn\(\),)g(int)g(keyval,)393 816 y(void*)g(extra)p 659 816 V 17 w(state\))75 903 y(int)g(MPI)p 245 903 V 17 w(Attr)p 358 903 V 17 w(free)p 471 903 V 16 w(key\(int)g(keyval\))75 989 y(int)g(MPI)p 245 989 V 17 w(Put)p 334 989 V 17 w(attr\(MPI)p 543 989 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 989 V 16 w(val\))75 1075 y(int)g(MPI)p 245 1075 V 17 w(Get)p 334 1075 V 17 w(attr\(MPI)p 543 1075 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 1075 V 16 w(val,)g(int)h(*found\))75 1162 y(int)f(MPI)p 245 1162 V 17 w(Delete)p 406 1162 V 17 w(Attr\(MPI)p 615 1162 V 16 w(Comm)g(comm,)g(int)h(keyval\))75 1335 y Fq(A.6)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro)r(cess)e(T)-5 b(op)r(ologies)75 1437 y FB(int)23 b(MPI)p 245 1437 V 17 w(Make)p 358 1437 V 17 w(cart\(MPI)p 567 1437 V 16 w(comm)g(comm)p 798 1437 V 17 w(old,)g(int)h(ndims,)f(int)g(*dims,)g (int)h(*periods,)393 1493 y(int)g(reorder,)e(MPI)p 778 1493 V 17 w(comm)h(*comm)p 1034 1493 V 17 w(cart\))75 1580 y(int)g(MPI)p 245 1580 V 17 w(Make)p 358 1580 V 17 w(dims\(int)g(nnodes,)g(int)g(ndims,)g(int)g(*dims\))75 1666 y(int)g(MPI)p 245 1666 V 17 w(Make)p 358 1666 V 17 w(graph\(MPI)p 591 1666 V 16 w(comm)g(comm)p 822 1666 V 17 w(old,)g(int)h(nnodes,)e(int)i(*index,)f(int)g(*edges,)393 1722 y(int)h(reorder,)e(MPI)p 778 1722 V 17 w(comm)h(*comm)p 1034 1722 V 17 w(graph\))75 1809 y(int)g(MPI)p 245 1809 V 17 w(Topo)p 358 1809 V 17 w(status\(MPI)p 615 1809 V 16 w(comm)g(comm,)g(int)h(*status\))75 1895 y(int)f(MPI)p 245 1895 V 17 w(Get)p 334 1895 V 17 w(graphdims\(MPI)p 663 1895 V 15 w(comm)h(comm,)f(int)g(*nnodes,)g(int)h(*nedges\))75 1982 y(int)f(MPI)p 245 1982 V 17 w(Get)p 334 1982 V 17 w(graph\(MPI)p 567 1982 V 16 w(comm)g(comm,)g(int)h(maxindex,)f(int)g (maxedges,)g(int)g(*index,)393 2038 y(int)h(*edges\))75 2125 y(int)f(MPI)p 245 2125 V 17 w(Get)p 334 2125 V 17 w(cartdim\(MPI)p 615 2125 V 16 w(comm)g(comm,)g(int)h(*ndims\))75 2211 y(int)f(MPI)p 245 2211 V 17 w(Get)p 334 2211 V 17 w(cart\(MPI)p 543 2211 V 16 w(comm)g(comm,)h(int)f(maxdims,)g(int)g (*dims,)g(int)h(*periods,)393 2268 y(int)g(*coords\))75 2354 y(int)f(MPI)p 245 2354 V 17 w(Cart)p 358 2354 V 17 w(rank\(MPI)p 567 2354 V 16 w(comm)g(comm,)g(int)h(*coords,)f(int)g (*rank\))75 2441 y(int)g(MPI)p 245 2441 V 17 w(Cart)p 358 2441 V 17 w(coords\(MPI)p 615 2441 V 16 w(comm)g(comm,)g(int)h (rank,)f(int)g(maxdims,)g(int)g(*coords\))75 2527 y(int)g(MPI)p 245 2527 V 17 w(Graph)p 382 2527 V 17 w(neighbors)p 615 2527 V 16 w(count\(MPI)p 847 2527 V 16 w(comm)g(comm,)g(int)h(rank,)f (int)g(*nneighbors\))75 2613 y(int)g(MPI)p 245 2613 V 17 w(Graph)p 382 2613 V 17 w(neighbors\(MPI)p 711 2613 V 15 w(comm)g(comm,)h(int)f(rank,)g(int)h(*maxneighbors,)393 2670 y(int)g(*neighbors\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 213 220 213 219 bop 75 -100 a Fp(A.7.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(ENVIR)o (ONMENT)l(AL)g(INQUIR)l(Y)582 b FD(213)75 45 y FB(int)23 b(MPI)p 245 45 15 2 v 17 w(Cart)p 358 45 V 17 w(shift\(MPI)p 591 45 V 16 w(comm)g(comm,)g(int)h(direction,)e(int)i(disp,)f(int)g (*rank)p 1681 45 V 17 w(source,)393 102 y(int)h(*rank)p 612 102 V 16 w(dest\))75 188 y(int)f(MPI)p 245 188 V 17 w(Cart)p 358 188 V 17 w(sub\(MPI)p 543 188 V 16 w(comm)g(comm,)h (int)f(*remain)p 1085 188 V 16 w(dims,)g(MPI)p 1316 188 V 17 w(comm)h(*comm)p 1573 188 V 16 w(new\))75 274 y(int)f(MPI)p 245 274 V 17 w(Map)p 334 274 V 17 w(cart\(MPI)p 543 274 V 16 w(comm)g(comm)p 774 274 V 17 w(old,)g(int)h(ndims,)f(int)g(*dims,) g(int)h(*period,)393 331 y(int)g(*newrank\))75 417 y(int)f(MPI)p 245 417 V 17 w(Map)p 334 417 V 17 w(graph\(MPI)p 567 417 V 16 w(comm)g(comm)p 798 417 V 17 w(old,)g(int)h(nnodes,)f(int)g (*index,)g(int)g(*edges,)393 474 y(int)h(*newrank\))75 647 y Fq(A.7)59 b(C)20 b(bindings)g(fo)n(r)h(Environmental)d(Inquiry)75 749 y FB(int)23 b(MPI)p 245 749 V 17 w(Get)p 334 749 V 17 w(valid)p 471 749 V 16 w(tag)p 559 749 V 17 w(range\(int)g(*low,)g (int)h(*high\))75 835 y(int)f(MPI)p 245 835 V 17 w(Get)p 334 835 V 17 w(processor)p 567 835 V 16 w(name\(int)g(len,)g(char)g (*name\))75 921 y(int)g(MPI)p 245 921 V 17 w(Get)p 334 921 V 17 w(host\(MPI)p 543 921 V 16 w(Comm)g(comm,)h(int)f(*rank\))75 1008 y(int)g(MPI)p 245 1008 V 17 w(Get)p 334 1008 V 17 w(buffer)p 495 1008 V 16 w(params\(int)g(*numsend,)f(int)i(*sizesend,)e (int)i(*numrecv,)393 1064 y(int)g(*sizerecv,)e(int)i(*totalnumsend,)e (int)h(*totalsizesend,)393 1121 y(int)h(*totalnumrecv,)e(int)h (*totalsizerecv\))75 1207 y(int)g(MPI)p 245 1207 V 17 w(Suggest)p 430 1207 V 16 w(buffer)p 590 1207 V 17 w(params\(int)f (numsend,)h(int)h(sizesend,)e(int)i(numrecv,)393 1264 y(int)g(sizerecv,)e(int)i(totalnumsend,)e(int)h(totalsizesen,)393 1320 y(int)h(totalnumrecv,)e(int)h(totalsizerecv,)f(int)i(*flag\))75 1407 y(int)f(MPI)p 245 1407 V 17 w(User)p 358 1407 V 17 w(specifies)p 591 1407 V 16 w(buffer\(MPI)p 847 1407 V 16 w(Comm)g(comm,)g(void*)g(buffer,)g(int)h(size\))75 1493 y(int)f(MPI)p 245 1493 V 17 w(Get)p 334 1493 V 17 w(errormode\(MPI)p 663 1493 V 15 w(Comm)h(comm,)f(MPI)p 1013 1493 V 17 w(Errmode)g(*mode\))75 1580 y(int)g(MPI)p 245 1580 V 17 w(Set)p 334 1580 V 17 w(errormode\(MPI)p 663 1580 V 15 w(Comm)h(comm,)f(MPI)p 1013 1580 V 17 w(Errmode)g(mode\)) 75 1666 y(int)g(MPI)p 245 1666 V 17 w(Set)p 334 1666 V 17 w(errhandler\(MPI)p 687 1666 V 15 w(Comm)h(comm,)f(void)g (handler\))75 1752 y(int)g(MPI)p 245 1752 V 17 w(Error)p 382 1752 V 17 w(string\(int)f(errorcode,)h(int)g(len,)g(char)h (*string\))75 1839 y(int)f(MPI)p 245 1839 V 17 w(Init\(int)g(*argc,)g (char)g(**argv\))75 1925 y(int)g(MPI)p 245 1925 V 17 w(Finalize\(\))75 2012 y(int)g(MPI)p 245 2012 V 17 w(Initialized\(int)f (*flag\))75 2098 y(int)h(MPI)p 245 2098 V 17 w(Abort\(MPI)p 478 2098 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 2272 y Fq(A.8)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(P)n(oint-to-P)n (oint)f(Communication)75 2373 y FB(MPI)p 150 2373 V 17 w(SEND\(BUF,)i(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR\))170 2429 y()g(BUF\(*\))170 2486 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(IERROR)75 2572 y(MPI)p 150 2572 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS,)g(IERROR\))170 2629 y()g(BUF\(*\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 214 221 214 220 bop 75 -100 a FD(214)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y FB(INTEGER)23 b(COUNT,)g(DATATYPE,) g(SOURCE,)g(TAG,)g(COMM,)g(STATUS\(MPI)p 1463 45 15 2 v 16 w(STATUS)p 1623 45 V 17 w(SIZE\),)170 102 y(IERROR)75 188 y(MPI)p 150 188 V 17 w(GET)p 239 188 V 17 w(SOURCE\(STATUS,)e (SOURCE,)i(IERROR\))170 244 y(INTEGER)g(STATUS\(MPI)p 604 244 V 16 w(STATUS)p 764 244 V 17 w(SIZE\),)g(SOURCE,)f(IERROR)75 331 y(MPI)p 150 331 V 17 w(GET)p 239 331 V 17 w(TAG\(STATUS,)g(TAG,)h (IERROR\))170 387 y(INTEGER)g(STATUS\(MPI)p 604 387 V 16 w(STATUS)p 764 387 V 17 w(SIZE\),)g(TAG,)g(IERROR)75 474 y(MPI)p 150 474 V 17 w(GET)p 239 474 V 17 w(COUNT\(STATUS,)f (DATATYPE,)g(COUNT,)h(IERROR\))170 530 y(INTEGER)g(STATUS\(MPI)p 604 530 V 16 w(STATUS)p 764 530 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)75 617 y(MPI)p 150 617 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,) g(DEST,)g(TAG,)g(COMM,)h(IERROR\))170 673 y()f(BUF\(*\))170 730 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 816 y(MPI)p 150 816 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 873 y()f(BUF\(*\))170 929 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 1015 y(MPI)p 150 1015 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(REQUEST,)e(IERROR\))170 1072 y()h(BUF\(*\))170 1128 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1215 y(MPI)p 150 1215 V 17 w(IRSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1271 y()g(BUF\(*\))170 1328 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1414 y(MPI)p 150 1414 V 17 w(ISSEND\(BUF,)g(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 1471 y()g(BUF\(*\))170 1527 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1614 y(MPI)p 150 1614 V 17 w(IRECV\(BUF,)g(COUNT,)h (DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1670 y()g(BUF\(*\))170 1726 y(INTEGER)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1813 y(MPI)p 150 1813 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 1869 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1869 V 16 w(STATUS)p 979 1869 V 16 w(SIZE\),)g(IERROR)75 1956 y(MPI)p 150 1956 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 2012 y(LOGICAL)g(FLAG)170 2069 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 2069 V 16 w(STATUS)p 979 2069 V 16 w(SIZE\),)g(IERROR)75 2155 y(MPI)p 150 2155 V 17 w(WAITANY\(COUNT,)f(ARRAY)p 645 2155 V 16 w(OF)p 709 2155 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g (IERROR\))170 2212 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2212 V 17 w(OF)p 716 2212 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 2212 V 16 w(STATUS)p 1610 2212 V 16 w(SIZE\),)170 2268 y(IERROR)75 2355 y(MPI)p 150 2355 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 2355 V 16 w(OF)p 709 2355 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))170 2411 y(LOGICAL)g(FLAG)170 2468 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2468 V 17 w(OF)p 716 2468 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 2468 V 16 w(STATUS)p 1610 2468 V 16 w(SIZE\),)170 2524 y(IERROR)75 2610 y(MPI)p 150 2610 V 17 w(WAITALL\(COUNT,)f(ARRAY)p 645 2610 V 16 w(OF)p 709 2610 V 17 w(REQUESTS,)h(ARRAY)p 1085 2610 V 16 w(OF)p 1149 2610 V 17 w(STATUSES,)g(IERROR\))170 2667 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2667 V 17 w(OF)p 716 2667 V 17 w(REQUESTS\(*\),)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 215 222 215 221 bop 75 -100 a Fp(A.8.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(POINT-TO-POINT)h(COMMUNICA)l(TION)185 b FD(215)170 45 y FB(ARRAY)p 293 45 15 2 v 17 w(OF)p 358 45 V 17 w(STATUSES\(MPI)p 663 45 V 15 w(STATUS)p 822 45 V 17 w(SIZE,*\),)23 b(IERROR)75 132 y(MPI)p 150 132 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 132 V 16 w(OF)p 709 132 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 132 V 16 w(OF)p 1292 132 V 17 w(STATUSES,)g(IERROR\))170 188 y(LOGICAL)g(FLAG)170 244 y(INTEGER)g(COUNT,)g(ARRAY)p 651 244 V 17 w(OF)p 716 244 V 17 w(REQUESTS\(*\),)170 301 y(ARRAY)p 293 301 V 17 w(OF)p 358 301 V 17 w(STATUSES\(MPI)p 663 301 V 15 w(STATUS)p 822 301 V 17 w(SIZE,*\),)g(IERROR)75 387 y(MPI)p 150 387 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 444 y(LOGICAL)g(FLAG)170 500 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 500 V 16 w(STATUS)p 1218 500 V 16 w(SIZE\),)g(IERROR)75 587 y(MPI)p 150 587 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g (IERROR\))170 643 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 643 V 16 w(STATUS)p 1218 643 V 16 w(SIZE\),)g(IERROR)75 730 y(MPI)p 150 730 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 786 y(INTEGER)h(REQUEST,)g(IERROR)75 873 y(MPI)p 150 873 V 17 w(TEST)p 263 873 V 16 w(CANCELLED\(STATUS,)f(FLAG,)h(IERROR\)) 170 929 y(LOGICAL)g(FLAG)170 985 y(INTEGER)g(STATUS\(MPI)p 604 985 V 16 w(STATUS)p 764 985 V 17 w(SIZE\),)g(IERROR)75 1072 y(MPI)p 150 1072 V 17 w(CREATE)p 311 1072 V 16 w(SEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g(REQUEST,)g(IERROR\))170 1128 y()g(BUF\(*\))170 1185 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1271 y(MPI)p 150 1271 V 17 w(CREATE)p 311 1271 V 16 w(RSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1328 y()g(BUF\(*\))170 1384 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1471 y(MPI)p 150 1471 V 17 w(CREATE)p 311 1471 V 16 w(SSEND\(BUF,)h(COUNT,)g (DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1527 y()g(BUF\(*\))170 1584 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1670 y(MPI)p 150 1670 V 17 w(CREATE)p 311 1670 V 16 w(RECV\(BUF,)h(COUNT,)g (DATATYPE,)f(SOURCE,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1726 y()g(BUF\(*\))170 1783 y(INTEGER)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1869 y(MPI)p 150 1869 V 17 w(START\(REQUEST,)f(IERROR\))170 1926 y(INTEGER)h (REQUEST,)g(IERROR)75 2012 y(MPI)p 150 2012 V 17 w(STARTALL\(COUNT,)f (ARRAY)p 669 2012 V 16 w(OF)p 733 2012 V 17 w(REQUESTS,)h(IERROR\))170 2069 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2069 V 17 w(OF)p 716 2069 V 17 w(REQUESTS\(*\),)f(IERROR)75 2155 y(MPI)p 150 2155 V 17 w(REQUEST)p 335 2155 V 16 w(FREE\(REQUEST,)g(IERROR\))170 2212 y(INTEGER)h(REQUEST,)g(IERROR)75 2298 y(MPI)p 150 2298 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(DEST,)i (SENDTAG,)e(RECVBUF,)393 2355 y(RECVCOUNT,)h(RECVTYPE,)f(SOURCE,)h (RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 2411 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 2468 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h(RECVTYPE,)170 2524 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 2524 V 16 w(STATUS)p 1122 2524 V 16 w(SIZE\),)h(IERROR)75 2610 y(MPI)p 150 2610 V 17 w(SENDRECV)p 359 2610 V 16 w(REPLACE\(BUF,)e(COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g (RECVTAG,)393 2667 y(COMM,)g(STATUS,)g(IERROR\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 216 223 216 222 bop 75 -100 a FD(216)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y FB()23 b(BUF\(*\))170 102 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g (RECVTAG,)g(COMM,)170 158 y(STATUS\(MPI)p 413 158 15 2 v 16 w(STATUS)p 573 158 V 17 w(SIZE\),)g(IERROR)75 244 y(MPI)p 150 244 V 17 w(TYPE)p 263 244 V 16 w(CONTIGUOUS\(COUNT,)f (OLDTYPE,)h(NEWTYPE,)f(IERROR\))170 301 y(INTEGER)h(COUNT,)g(OLDTYPE,)g (NEWTYPE,)g(IERROR)75 387 y(MPI)p 150 387 V 17 w(TYPE)p 263 387 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)g(OLDTYPE,)f (NEWTYPE,)h(IERROR\))170 444 y(INTEGER)g(COUNT,)g(BLOCKLENGTH,)g (STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 530 y(MPI)p 150 530 V 17 w(TYPE)p 263 530 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)f(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 587 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 673 y(MPI)p 150 673 V 17 w(TYPE)p 263 673 V 16 w(INDEXED\(COUNT,)f (ARRAY)p 757 673 V 17 w(OF)p 822 673 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 673 V 16 w(OF)p 1357 673 V 17 w(DISPLACEMENTS,)393 730 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 786 y(INTEGER)g(COUNT,)g (ARRAY)p 651 786 V 17 w(OF)p 716 786 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY) p 1258 786 V 17 w(OF)p 1323 786 V 17 w(DISPLACEMENTS\(*\),)170 843 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 929 y(MPI)p 150 929 V 17 w(TYPE)p 263 929 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 929 V 17 w(OF)p 846 929 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 929 V 16 w(OF)p 1381 929 V 17 w(DISPLACEMENTS,)393 985 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1042 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1042 V 17 w(OF)p 716 1042 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1042 V 17 w(OF)p 1323 1042 V 17 w(DISPLACEMENTS\(*\),)170 1098 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1185 y(MPI)p 150 1185 V 17 w(TYPE)p 263 1185 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 1185 V 17 w(OF)p 798 1185 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 1185 V 16 w(OF)p 1333 1185 V 17 w(DISPLACEMENTS,)393 1241 y(ARRAY)p 516 1241 V 17 w(OF)p 581 1241 V 17 w(TYPES,)h(NEWTYPE,)f (IERROR\))170 1298 y(INTEGER)h(COUNT,)g(ARRAY)p 651 1298 V 17 w(OF)p 716 1298 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 1298 V 17 w(OF)p 1323 1298 V 17 w(DISPLACEMENTS\(*\),)170 1354 y(ARRAY)p 293 1354 V 17 w(OF)p 358 1354 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)75 1441 y(MPI)p 150 1441 V 17 w(ADDRESS\(LOCATION,)e (ADDRESS,)i(IERROR\))170 1497 y()g(LOCATION\(*\))170 1554 y(INTEGER)g(ADDRESS,)g(IERROR)75 1640 y(MPI)p 150 1640 V 17 w(TYPE)p 263 1640 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h (IERROR\))170 1696 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)75 1783 y(MPI)p 150 1783 V 17 w(TYPE)p 263 1783 V 16 w(SIZE\(DATATYPE,)f (SIZE,)i(IERROR\))170 1839 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)75 1926 y(MPI)p 150 1926 V 17 w(TYPE)p 263 1926 V 16 w(COUNT\(DATATYPE,)f (COUNT,)h(IERROR\))170 1982 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)75 2069 y(MPI)p 150 2069 V 17 w(TYPE)p 263 2069 V 16 w(COMMIT\(DATATYPE,)f (IERROR\))170 2125 y(INTEGER)h(DATATYPE,)g(IERROR)75 2212 y(MPI)p 150 2212 V 17 w(TYPE)p 263 2212 V 16 w(FREE\(DATATYPE,)f (IERROR\))170 2268 y(INTEGER)h(DATATYPE,)g(IERROR)75 2355 y(MPI)p 150 2355 V 17 w(GET)p 239 2355 V 17 w(ELEMENTS\(STATUS,)e (DATATYPE,)i(COUNT,)g(IERROR\))170 2411 y(INTEGER)g(STATUS\(MPI)p 604 2411 V 16 w(STATUS)p 764 2411 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)75 2584 y Fq(A.9)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i (Collective)e(Communication)75 2686 y FB(MPI)p 150 2686 V 17 w(BARRIER\(COMM,)j(IERROR\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 217 224 217 223 bop 75 -100 a Fp(A.9.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(COLLECTIVE)g(COMMUNICA)l(TION)280 b FD(217)170 45 y FB(INTEGER)23 b(COMM,)h(IERROR)75 132 y(MPI)p 150 132 15 2 v 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,)g(COMM,)g (IERROR\))170 188 y()g(BUFFER\(*\))170 244 y(INTEGER)g(COUNT,)g (DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 331 y(MPI)p 150 331 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 387 y(ROOT,)g(COMM,)h(IERROR\))170 444 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 500 y(INTEGER)g (SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g(COMM,)g (IERROR)75 587 y(MPI)p 150 587 V 17 w(GATHERV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)g(RECVBUF,)f(RECVCOUNTS,)h(DISPLS,)393 643 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 700 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 756 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g(ROOT,)170 813 y(COMM,)h(IERROR)75 899 y(MPI)p 150 899 V 17 w(SCATTER\(SENDBUF,)d (SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f(RECVCOUNT,)h(RECVTYPE,)393 955 y(ROOT,)g(COMM,)h(IERROR\))170 1012 y()f(SENDBUF\(*\),)g (RECVBUF\(*\))170 1068 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,) h(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR)75 1155 y(MPI)p 150 1155 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i(DISPLS,)g(SENDTYPE,) f(RECVBUF,)h(RECVCOUNT,)393 1211 y(RECVTYPE,)g(ROOT,)g(COMM,)g (IERROR\))170 1268 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1324 y(INTEGER)g(SENDCOUNTS\(*\),)f(DISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 1381 y(COMM,)h(IERROR)75 1467 y(MPI)p 150 1467 V 17 w(ALLGATHER\(SENDBUF,)d(SENDCOUNT,)i (SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 1524 y(COMM,)g(IERROR\))170 1580 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1636 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (COMM,)g(IERROR)75 1723 y(MPI)p 150 1723 V 17 w(ALLGATHERV\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNTS,)f(DISPLS,)393 1779 y(RECVTYPE,)h(COMM,)g(IERROR\))170 1836 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1892 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f (RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g(COMM,)170 1949 y(IERROR)75 2035 y(MPI)p 150 2035 V 17 w(ALLTOALL\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 2092 y(COMM,)g(IERROR\))170 2148 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2205 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,) h(RECVTYPE,)g(COMM,)g(IERROR)75 2291 y(MPI)p 150 2291 V 17 w(ALLTOALLV\(SENDBUF,)e(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h (RECVBUF,)g(RECVCOUNTS,)393 2347 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g (IERROR\))170 2404 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2460 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 2517 y(RECVTYPE,)h(COMM,)g (IERROR)75 2603 y(MPI)p 150 2603 V 17 w(REDUCE\(SENDBUF,)f(RECVBUF,)g (COUNT,)h(DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR\))170 2660 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 218 225 218 224 bop 75 -100 a FD(218)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y FB(INTEGER)23 b(COUNT,)g(DATATYPE,) g(OP,)g(ROOT,)h(COMM,)f(IERROR)75 132 y(MPI)p 150 132 15 2 v 17 w(USER)p 263 132 V 16 w(REDUCE\(SENDBUF,)f(RECVBUF,)h(COUNT,) g(DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM,)393 188 y(IERROR\))170 244 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 301 y(EXTERNAL)g (FUNCTION)170 357 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g (IERROR)75 444 y(MPI)p 150 444 V 17 w(USER)p 263 444 V 16 w(REDUCEA\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)f (ROOT,)h(COMM,)393 500 y(IERROR\))170 557 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 613 y(EXTERNAL)g(FUNCTION)170 670 y(INTEGER)g(COUNT,) g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 756 y(MPI)p 150 756 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g(DATATYPE,)g(OP,)g (COMM,)g(IERROR\))170 813 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 869 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h(IERROR)75 955 y(MPI)p 150 955 V 17 w(USER)p 263 955 V 16 w(ALLREDUCE\(SENDBUF,)e (RECVBUF,)h(COUNT,)g(DATATYPE,)f(FUNCTION,)h(COMM,)393 1012 y(IERROR\))170 1068 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1125 y(EXTERNAL)g(FUNCTION)170 1181 y(INTEGER)g(COUNT,)g(DATATYPE,)g (COMM,)g(IERROR)75 1268 y(MPI)p 150 1268 V 17 w(USER)p 263 1268 V 16 w(ALLREDUCEA\(SENDBUF,)f(RECVBUF,)g(COUNT,)h(DATATYPE,)g (FUNCTION,)g(COMM,)393 1324 y(IERROR\))170 1381 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 1437 y(EXTERNAL)g(FUNCTION)170 1494 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)75 1580 y(MPI)p 150 1580 V 17 w(REDUCE)p 311 1580 V 16 w (SCATTER\(SENDBUF,)f(RECVBUF,)h(RECVCOUNTS,)f(DATATYPE,)h(OP,)g(COMM,) 393 1636 y(IERROR\))170 1693 y()g(SENDBUF\(*\),)g(RECVBUF\(*\)) 170 1749 y(INTEGER)g(RECVCOUNTS,)g(DATATYPE,)f(OP,)i(COMM,)f(IERROR)75 1836 y(MPI)p 150 1836 V 17 w(SCAN\(SENDBUF,)f(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 1892 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1949 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)75 2035 y(MPI)p 150 2035 V 17 w(USER)p 263 2035 V 16 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)g (COMM,)g(IERROR\))170 2092 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2148 y(EXTERNAL)g(FUNCTION)170 2205 y(INTEGER)g(COUNT,)g(DATATYPE,)g (COMM,)g(IERROR)75 2291 y(MPI)p 150 2291 V 17 w(USER)p 263 2291 V 16 w(SCANA\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)g (FUNCTION,)f(COMM,)i(IERROR\))170 2347 y()f(SENDBUF\(*\),)g (RECVBUF\(*\))170 2404 y(EXTERNAL)g(FUNCTION)170 2460 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 219 226 219 225 bop 75 -100 a Fp(A.10.)27 b(F)o(OR)l(TRAN)11 b(BINDINGS)g(F)o(OR)g(GR)o(OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)p FD(219)75 45 y Fq(A.10)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i (Groups,)f(Contexts,)d(and)j(Communicato)n(rs)75 147 y FB(MPI)p 150 147 15 2 v 17 w(GROUP)p 287 147 V 16 w(SIZE\(GROUP,)j (SIZE,)g(IERROR\))170 203 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 289 y(MPI)p 150 289 V 17 w(GROUP)p 287 289 V 16 w(RANK\(GROUP,)f(RANK,) g(IERROR\))170 346 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)75 432 y(MPI)p 150 432 V 17 w(GROUP)p 287 432 V 16 w(TRANSLATE)p 519 432 V 16 w(RANKS\(GROUP1,)e(N,)i(RANKS1,)f(GROUP2,)g(RANKS2,)g (IERROR\))170 489 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1\(*\),)e(GROUP2,)h (RANKS2\(*\),)g(IERROR)75 575 y(MPI)p 150 575 V 17 w(GROUP)p 287 575 V 16 w(UNION\(GROUP1,)f(GROUP2,)h(NEWGROUP,)g(IERROR\))170 632 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 718 y(MPI)p 150 718 V 17 w(GROUP)p 287 718 V 16 w (INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 775 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 861 y(MPI)p 150 861 V 17 w(GROUP)p 287 861 V 16 w(DIFFERENCE\(GROUP1,)f (GROUP2,)h(NEWGROUP,)f(IERROR\))170 918 y(INTEGER)h(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 1004 y(MPI)p 150 1004 V 17 w(GROUP)p 287 1004 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g(NEWGROUP,)g(IERROR\))170 1060 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f(NEWGROUP,)f(IERROR)75 1147 y(MPI)p 150 1147 V 17 w(GROUP)p 287 1147 V 16 w(EXCL\(GROUP,)h(N,) g(RANKS,)g(NEWGROUP,)g(IERROR\))170 1203 y(INTEGER)g(GROUP,)g(N,)h (RANKS\(*\),)f(NEWGROUP,)f(IERROR)75 1290 y(MPI)p 150 1290 V 17 w(GROUP)p 287 1290 V 16 w(RANGE)p 423 1290 V 17 w(INCL\(GROUP,)g(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1346 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 1433 y(MPI)p 150 1433 V 17 w(GROUP)p 287 1433 V 16 w(RANGE)p 423 1433 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1489 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 1576 y(MPI)p 150 1576 V 17 w(GROUP)p 287 1576 V 16 w(FREE\(GROUP,)g (IERROR\))170 1632 y(INTEGER)g(GROUP,)g(IERROR)75 1719 y(MPI)p 150 1719 V 17 w(COMM)p 263 1719 V 16 w(SIZE\(COMM,)g(SIZE,)g (IERROR\))170 1775 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 1862 y(MPI)p 150 1862 V 17 w(COMM)p 263 1862 V 16 w(RANK\(COMM,)g (RANK,)g(IERROR\))170 1918 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 2004 y(MPI)p 150 2004 V 17 w(COMM)p 263 2004 V 16 w(GROUP\(COMM,)g (GROUP,)g(IERROR\))170 2061 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)75 2147 y(MPI)p 150 2147 V 17 w(COMM)p 263 2147 V 16 w(DUP\(COMM,)g (NEWCOMM,)g(IERROR\))170 2204 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)75 2290 y(MPI)p 150 2290 V 17 w(COMM)p 263 2290 V 16 w(MAKE\(COMM,)h (GROUP,)g(NEWCOMM,)g(IERROR\))170 2347 y(INTEGER)g(COMM,)h(GROUP,)f (NEWCOMM,)f(IERROR)75 2433 y(MPI)p 150 2433 V 17 w(COMM)p 263 2433 V 16 w(SPLIT\(COMM,)h(COLOR,)g(KEY,)g(NEWCOMM,)g(IERROR\))170 2490 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g(NEWCOMM,)g(IERROR)75 2576 y(MPI)p 150 2576 V 17 w(COMM)p 263 2576 V 16 w(FREE\(COMM,)g (IERROR\))170 2633 y(INTEGER)g(COMM,)h(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 220 227 220 226 bop 75 -100 a FD(220)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y FB(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(TEST)p 375 45 V 17 w(INTER\(COMM,)22 b(FLAG,)i(IERROR\))170 102 y(INTEGER)f(COMM,)h(IERROR)170 158 y(LOGICAL)f(FLAG)75 244 y(MPI)p 150 244 V 17 w(INTERCOMM)p 383 244 V 16 w(START\(LOCAL)p 663 244 V 15 w(COMM,)h(LOCAL)p 942 244 V 16 w(LEADER,)f(PEER)p 1245 244 V 17 w(COMM,)g(REMOTE)p 1549 244 V 16 w(LEADER,)g(TAG,)393 301 y(INTER)p 516 301 V 17 w(REQUEST,)f(IERROR\))170 357 y(INTEGER)h(LOCAL)p 484 357 V 17 w(COMM,)g(LOCAL)p 764 357 V 17 w(LEADER,)f(PEER)p 1067 357 V 17 w(COMM,)h(REMOTE)p 1371 357 V 17 w(LEADER,)g(TAG,)170 414 y(INTER)p 293 414 V 17 w(REQUEST,)g(IERROR)75 500 y(MPI)p 150 500 V 17 w(INTERCOMM)p 383 500 V 16 w(FINISH\(INTER)p 687 500 V 15 w(REQUEST,)g(IERROR\))170 557 y(INTEGER)g(INTER)p 484 557 V 17 w(REQUEST,)g(NEWCOMM,)f(IERROR)75 643 y(MPI)p 150 643 V 17 w(INTERCOMM)p 383 643 V 16 w(MERGE\(INTERCOMM,)f(HIGH,)j (NEWINTERCOMM,)e(IERROR\))170 700 y(INTEGER)h(INTERCOMM,)g (NEWINTRACOMM,)f(IERROR)170 756 y(LOGICAL)h(HIGH)75 843 y(MPI)p 150 843 V 17 w(INTERCOMM)p 383 843 V 16 w(NAME)p 495 843 V 16 w(START\(COMM,)g(NAME,)g(NEW)p 1013 843 V 17 w(COMM,)g(INTER)p 1293 843 V 16 w(REQUEST,)g(IERROR\))170 899 y(CHARACTER)g(*\(*\))g(NAME)170 955 y(INTEGER)g(COMM,)h(NEW)p 580 955 V 16 w(COMM,)f(INTER)p 859 955 V 17 w(REQUEST,)g(IERROR)75 1042 y(MPI)p 150 1042 V 17 w(ATTR)p 263 1042 V 16 w(CREATE)p 423 1042 V 17 w(KEY\(COPY)p 632 1042 V 16 w(FN,)g(DELETE)p 887 1042 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1310 1042 V 17 w(STATE,)g(IERROR\))170 1098 y(EXTERNAL)g(COPY)p 484 1098 V 17 w(FN,)g(DELETE)p 740 1098 V 17 w(FN)170 1155 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 1155 V 17 w(STATE,)g(IERROR)75 1241 y(MPI)p 150 1241 V 17 w(ATTR)p 263 1241 V 16 w(FREE)p 375 1241 V 17 w(KEY\(KEYVAL,)f(IERROR\))170 1298 y(INTEGER)h(KEYVAL,)g (IERROR)75 1384 y(MPI)p 150 1384 V 17 w(PUT)p 239 1384 V 17 w(ATTR\(COMM,)f(KEYVAL,)h(ATTRIBUTE)p 925 1384 V 16 w(VAL,)g(IERROR\))170 1441 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 1441 V 16 w(VAL,)i(IERROR)75 1527 y(MPI)p 150 1527 V 17 w(GET)p 239 1527 V 17 w(ATTR\(COMM,)e(KEYVAL,)h(ATTRIBUTE)p 925 1527 V 16 w(VAL,)g(FOUND,)g(IERROR\))170 1584 y(INTEGER)g(COMM,)h (KEYVAL,)e(ATTRIBUTE)p 914 1584 V 16 w(VAL,)i(IERROR)170 1640 y(LOGICAL)f(FOUND)75 1726 y(MPI)p 150 1726 V 17 w(DELETE)p 311 1726 V 16 w(ATTR\(COMM,)g(KEYVAL,)g(IERROR\))170 1783 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)75 1956 y Fq(A.11)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro)r(cess)e(T)-5 b(op)r(ologies)75 2058 y FB(MPI)p 150 2058 V 17 w(MAKE)p 263 2058 V 16 w(CART\(COMM)p 495 2058 V 16 w(OLD,)24 b(NDIMS,)f(DIMS,)g(PERIODS,)g(REORDER,)f(COMM)p 1466 2058 V 17 w(CART,)h(IERROR\))170 2114 y(INTEGER)g(COMM)p 460 2114 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(PERIODS\(*\),)f(REORDER,)h (COMM)p 1575 2114 V 17 w(CART,)170 2171 y(IERROR)75 2257 y(MPI)p 150 2257 V 17 w(MAKE)p 263 2257 V 16 w(DIMS\(NNODES,)g(NDIMS,)g (DIMS,)g(IERROR\))170 2313 y(INTEGER)g(NNODES,)g(NDIMS,)g(DIMS\(*\),)g (IERROR)75 2400 y(MPI)p 150 2400 V 17 w(MAKE)p 263 2400 V 16 w(GRAPH\(COMM)p 519 2400 V 16 w(OLD,)h(NNODES,)e(INDEX,)h(EDGES,)h (REORDER,)e(COMM)p 1490 2400 V 17 w(GRAPH,)h(IERROR\))170 2456 y(INTEGER)g(COMM)p 460 2456 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g (EDGES\(*\),)f(REORDER,)h(COMM)p 1575 2456 V 17 w(GRAPH,)170 2513 y(IERROR)75 2599 y(MPI)p 150 2599 V 17 w(TOPO)p 263 2599 V 16 w(STATUS\(COMM,)g(STATUS,)g(IERROR\))170 2656 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 221 228 221 227 bop 75 -100 a Fp(A.12.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(ENVIR)o(ONMENT)l(AL)g(INQUIR)l(Y)364 b FD(221)75 45 y FB(MPI)p 150 45 15 2 v 17 w(GET)p 239 45 V 17 w(GRAPHDIMS\(COMM,)21 b(NNODES,)i(NEDGES,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(NNODES,)e(NEDGES,)h(IERROR)75 188 y(MPI)p 150 188 V 17 w(GET)p 239 188 V 17 w(GRAPH\(COMM,)f (MAXINDEX,)h(MAXEDGES,)f(INDEX,)h(EDGES,)g(IERROR\))170 244 y(INTEGER)g(COMM,)h(MAXINDEX,)e(MAXEDGES,)h(INDEX\(*\),)f (EDGES\(*\),)h(IERROR)75 331 y(MPI)p 150 331 V 17 w(GET)p 239 331 V 17 w(CARTDIM\(COMM,)f(NDIMS,)h(IERROR\))170 387 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)75 474 y(MPI)p 150 474 V 17 w(GET)p 239 474 V 17 w(CART\(COMM,)f(MAXDIMS,)h(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 530 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(PERIODS\(*\),)f(COORDS\(*\),)h(IERROR)75 617 y(MPI)p 150 617 V 17 w(CART)p 263 617 V 16 w(RANK\(COMM,)g(COORDS,) g(RANK,)g(IERROR\))170 673 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h (IERROR)75 760 y(MPI)p 150 760 V 17 w(CART)p 263 760 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g(COORDS,)f(IERROR\))170 816 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f(COORDS\(*\),)h(IERROR)75 903 y(MPI)p 150 903 V 17 w(GRAPH)p 287 903 V 16 w(NEIGHBORS)p 519 903 V 16 w(COUNT\(COMM,)g(RANK,)g(NNEIGHBORS,)f(IERROR\))170 959 y(INTEGER)h(COMM,)h(RANK,)f(NNEIGHBORS,)f(IERROR)75 1045 y(MPI)p 150 1045 V 17 w(GRAPH)p 287 1045 V 16 w(NEIGHBORS\(COMM,)g (RANK,)h(MAXNEIGHBORS,)f(NEIGHBORS,)h(IERROR\))170 1102 y(INTEGER)g(COMM,)h(RANK,)f(MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR)75 1188 y(MPI)p 150 1188 V 17 w(CART)p 263 1188 V 16 w(SHIFT\(COMM,)h (DIRECTION,)f(DISP,)h(RANK)p 1067 1188 V 17 w(SOURCE,)g(RANK)p 1371 1188 V 17 w(DEST,)g(IERROR\))170 1245 y(INTEGER)g(COMM,)h (DIRECTION,)e(DISP,)h(RANK)p 1009 1245 V 17 w(SOURCE,)g(RANK)p 1313 1245 V 16 w(DEST,)h(IERROR)75 1331 y(MPI)p 150 1331 V 17 w(CART)p 263 1331 V 16 w(SUB\(COMM,)f(REMAIN)p 662 1331 V 16 w(DIMS,)h(COMM)p 918 1331 V 16 w(NEW,)g(IERROR\))170 1388 y(INTEGER)f(COMM,)h(REMAIN)p 652 1388 V 16 w(DIMS\(*\),)f(COMM)p 979 1388 V 16 w(NEW,)h(IERROR)75 1474 y(MPI)p 150 1474 V 17 w(MAP)p 239 1474 V 17 w(CART\(COMM)p 472 1474 V 15 w(OLD,)g(NDIMS,)f(DIMS,)g(PERIOD,)g(NEWRANK,)g(IERROR\))170 1531 y(INTEGER)g(COMM)p 460 1531 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g (PERIOD\(*\),)g(NEWRANK,)f(IERROR)75 1617 y(MPI)p 150 1617 V 17 w(MAP)p 239 1617 V 17 w(GRAPH\(COMM)p 496 1617 V 15 w(OLD,)i(NNODES,)f(INDEX,)g(EDGES,)g(NEWRANK,)f(IERROR\))170 1674 y(INTEGER)h(COMM)p 460 1674 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g (EDGES\(*\),)f(NEWRANK,)h(IERROR)75 1847 y Fq(A.12)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Environmental)d(Inquiry)75 1948 y FB(MPI)p 150 1948 V 17 w(GET)p 239 1948 V 17 w(VALID)p 376 1948 V 16 w(TAG)p 464 1948 V 17 w(RANGE\(LOW,)k(HIGH,)h(IERROR\)) 170 2005 y(INTEGER)g(LOW,)h(HIGH,)f(IERROR)75 2091 y(MPI)p 150 2091 V 17 w(GET)p 239 2091 V 17 w(PROCESSOR)p 472 2091 V 15 w(NAME\(NAME,)g(IERROR\))170 2148 y(CHARACTER*\(*\))g(NAME) 170 2204 y(INTEGER)g(IERROR)75 2291 y(MPI)p 150 2291 V 17 w(GET)p 239 2291 V 17 w(HOST\(COMM,)f(RANK,)h(IERROR\))170 2347 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 2433 y(MPI)p 150 2433 V 17 w(GET)p 239 2433 V 17 w(BUFFER)p 400 2433 V 16 w(PARAMS\(NUMSEND,)f(SIZESEND,)g(NUMRECV,)h(SIZERECV,)g (TOTALNUMSEDND,)393 2490 y(TOTALSIZESEN,)f(TOTALNUMRECV,)g (TOTALSIZERECV,)g(IERROR\))170 2546 y(INTEGER)h(NUMSEND,)g(SIZESEND,)g (NUMRECV,)g(SIZERECV,)f(TOTALNUMSEDND,)170 2603 y(TOTALSIZESEN,)h (TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 222 229 222 228 bop 75 -100 a FD(222)987 b Fp(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y FB(MPI)p 150 45 15 2 v 17 w(SUGGEST)p 335 45 V 16 w(BUFFER)p 495 45 V 16 w(PARAMS\(NUMSEND,)22 b(SIZESEND,)h(NUMRECV,)g(SIZERECV,)393 102 y(TOTALNUMSEND,)f(TOTALSIZESEN,)g(TOTALNUMRECV,)g(TOTALSIZERECV,)g (FLAG,)393 158 y(IERROR\))170 214 y(INTEGER)h(NUMSEND,)g(SIZESEND,)g (NUMRECV,)g(SIZERECV,)f(TOTALNUMSEND,)170 271 y(TOTALSIZESEN,)h (TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)170 327 y(LOGICAL)h(FLAG)75 414 y(MPI)p 150 414 V 17 w(USER)p 263 414 V 16 w(SPECIFIES)p 495 414 V 16 w(BUFFER\(COMM,)g(BUFFER,)f(SIZE,)i(IERROR\))170 470 y()f(BUFFER\(*\))170 527 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR) 75 613 y(MPI)p 150 613 V 17 w(GET)p 239 613 V 17 w(ERRORMODE\(COMM,)e (MODE,)j(IERROR\))170 670 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)75 756 y(MPI)p 150 756 V 17 w(SET)p 239 756 V 17 w(ERRORMODE\(COMM,)e (MODE,)j(IERROR\))170 813 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)75 899 y(MPI)p 150 899 V 17 w(SET)p 239 899 V 17 w(ERRHANDLER\(COMM,)e (HANDLER,)i(IERROR\))170 955 y(EXTERNAL)g(HANDLER)170 1012 y(INTEGER)g(COMM,)h(IERROR)75 1098 y(MPI)p 150 1098 V 17 w(ERROR)p 287 1098 V 16 w(STRING\(ERRORCODE,)e(STRING,)h(IERROR\)) 170 1155 y(INTEGER)g(ERRORCODE,)g(IERROR)170 1211 y(CHARACTER*\(*\))g (STRING)75 1298 y(MPI)p 150 1298 V 17 w(INIT\(IERROR\))170 1354 y(INTEGER)g(IERROR)75 1441 y(MPI)p 150 1441 V 17 w(FINALIZE\(IERROR\))170 1497 y(INTEGER)g(IERROR)75 1584 y(MPI)p 150 1584 V 17 w(INITIALIZED\(FLAG,)e(IERROR\))170 1640 y(LOGICAL)i(FLAG)170 1696 y(INTEGER)g(IERROR)75 1783 y(MPI)p 150 1783 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1839 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 223 230 223 229 bop 75 283 a Fy(MPI)40 b(F)-10 b(unction)40 b(Index)75 490 y FD(MPI)p 167 490 14 2 v 16 w(ABOR)l(T,)16 b(190)75 547 y(MPI)p 167 547 V 16 w(ADD)p 287 547 V 16 w(TO)p 371 547 V 17 w(COLLECTION,)g(46)75 603 y(MPI)p 167 603 V 16 w(ADDRESS,)g(66)75 660 y(MPI)p 167 660 V 16 w(ALLGA)l(THER,)g(104) 75 716 y(MPI)p 167 716 V 16 w(ALLGA)l(THER)-5 b(V,)16 b(105)75 773 y(MPI)p 167 773 V 16 w(ALLREDUCE,)g(115)75 829 y(MPI)p 167 829 V 16 w(ALL)l(TO)o(ALL,)h(106)75 886 y(MPI)p 167 886 V 16 w(ALL)l(TO)o(ALL)-5 b(V,)17 b(107)75 942 y(MPI)p 167 942 V 16 w(A)l(TTR)p 312 942 V 17 w(CREA)l(TE)p 520 942 V 17 w(KEY,)e(145)75 999 y(MPI)p 167 999 V 16 w(A)l(TTR)p 312 999 V 17 w(FREE)p 454 999 V 17 w(KEY,)g(146)75 1091 y(MPI)p 167 1091 V 16 w(BARRIER,)i(88)75 1148 y(MPI)p 167 1148 V 16 w(BCAST,)e(89)75 1240 y(MPI)p 167 1240 V 16 w(CANCEL,)g(50)75 1297 y(MPI)p 167 1297 V 16 w(CAR)l(T)p 312 1297 V 17 w(COORDS,)h(171)75 1353 y(MPI)p 167 1353 V 16 w(CAR)l(T)p 312 1353 V 17 w(RANK,)g(171)75 1410 y(MPI)p 167 1410 V 16 w(CAR)l(T)p 312 1410 V 17 w(SHIFT,)f(173)75 1466 y(MPI)p 167 1466 V 16 w(CAR)l(T)p 312 1466 V 17 w(SUB,)g(174)75 1522 y(MPI)p 167 1522 V 16 w(COLLECTION)p 489 1522 V 18 w(CREA)l(TE,)g(45)75 1579 y(MPI)p 167 1579 V 16 w(COLLECTION)p 489 1579 V 18 w(FREE,)g(48)75 1635 y(MPI)p 167 1635 V 16 w(COLLECTION)p 489 1635 V 18 w(SIZE,)g(48)75 1692 y(MPI)p 167 1692 V 16 w(COMM)p 335 1692 V 16 w(DUP)l(,)g(132)75 1748 y(MPI)p 167 1748 V 16 w(COMM)p 335 1748 V 16 w(FREE,)g(134)75 1805 y(MPI)p 167 1805 V 16 w(COMM)p 335 1805 V 16 w(GR)o(OUP)l(,)g(132) 75 1861 y(MPI)p 167 1861 V 16 w(COMM)p 335 1861 V 16 w(MAKE,)g(133)75 1918 y(MPI)p 167 1918 V 16 w(COMM)p 335 1918 V 16 w(RANK,)h(132)75 1974 y(MPI)p 167 1974 V 16 w(COMM)p 335 1974 V 16 w(SIZE,)f(132)75 2031 y(MPI)p 167 2031 V 16 w(COMM)p 335 2031 V 16 w(SPLIT,)h(134)75 2087 y(MPI)p 167 2087 V 16 w(COMM)p 335 2087 V 16 w(TEST)p 473 2087 V 16 w(INTER,)g(136)75 2143 y(MPI)p 167 2143 V 16 w(CREA)l(TE)p 374 2143 V 17 w(ERRHANDLER,)g(186)75 2200 y(MPI)p 167 2200 V 16 w(CREA)l(TE)p 374 2200 V 17 w(RECV,)f(53)75 2256 y(MPI)p 167 2256 V 16 w(CREA)l(TE)p 374 2256 V 17 w(RSEND,)g(52)75 2313 y(MPI)p 167 2313 V 16 w(CREA)l(TE)p 374 2313 V 17 w(SEND,)g(52)75 2369 y(MPI)p 167 2369 V 16 w(CREA)l(TE)p 374 2369 V 17 w(SSEND,)g(53)75 2462 y(MPI)p 167 2462 V 16 w(DELETE)p 372 2462 V 17 w(A)l(TTR,)g(147)75 2518 y(MPI)p 167 2518 V 16 w(DELETE)p 372 2518 V 17 w(FR)o(OM)p 528 2518 V 16 w(COLLECTION,)c(46)75 2611 y(MPI)p 167 2611 V 16 w(ERR)o(OR)p 347 2611 V 18 w(STRING,)16 b(187)75 2704 y(MPI)p 167 2704 V 16 w(FINALIZE,)g(189)1048 490 y(MPI)p 1140 490 V 16 w(GA)l(THER,)f(89)1048 547 y(MPI)p 1140 547 V 16 w(GA)l(THER)-5 b(V,)15 b(92)1048 603 y(MPI)p 1140 603 V 16 w(GET)p 1256 603 V 16 w(A)l(TTR,)g(146)1048 660 y(MPI)p 1140 660 V 16 w(GET)p 1256 660 V 16 w(BUFFER)p 1462 660 V 16 w(P)l(ARAMS,)h(181)1048 716 y(MPI)p 1140 716 V 16 w(GET)p 1256 716 V 16 w(CAR)l(T,)f(170)1048 773 y(MPI)p 1140 773 V 16 w(GET)p 1256 773 V 16 w(CAR)l(TDIM,)g(170) 1048 829 y(MPI)p 1140 829 V 16 w(GET)p 1256 829 V 16 w(COUNT,)g(21,)f(22)1048 886 y(MPI)p 1140 886 V 16 w(GET)p 1256 886 V 16 w(ELEMENTS,)h(71)1048 942 y(MPI)p 1140 942 V 16 w(GET)p 1256 942 V 16 w(ERRHANDLER,)h(187)1048 999 y(MPI)p 1140 999 V 16 w(GET)p 1256 999 V 16 w(GRAPH,)f(169)1048 1055 y(MPI)p 1140 1055 V 16 w(GET)p 1256 1055 V 16 w(GRAPHDIMS,)g(169) 1048 1111 y(MPI)p 1140 1111 V 16 w(GET)p 1256 1111 V 16 w(HOST,)g(179)1048 1168 y(MPI)p 1140 1168 V 16 w(GET)p 1256 1168 V 16 w(INTER)o(COMM,)g(141)1048 1224 y(MPI)p 1140 1224 V 16 w(GET)p 1256 1224 V 16 w(PR)o(OCESSOR)p 1552 1224 V 18 w(NAME,)f(179)1048 1281 y(MPI)p 1140 1281 V 16 w(GET)p 1256 1281 V 16 w(SOUR)o(CE,)h(20)1048 1337 y(MPI)p 1140 1337 V 16 w(GET)p 1256 1337 V 16 w(T)l(A)o(G,)f(21)1048 1394 y(MPI)p 1140 1394 V 16 w(GET)p 1256 1394 V 16 w(V)-5 b(ALID)p 1414 1394 V 17 w(T)l(A)o(G)p 1529 1394 V 16 w(RANGE,)15 b(178)1048 1450 y(MPI)p 1140 1450 V 16 w(GRAPH)p 1324 1450 V 17 w(NEIGHBORS,)h(172)1048 1507 y(MPI)p 1140 1507 V 16 w(GRAPH)p 1324 1507 V 17 w(NEIGHBORS)p 1617 1507 V 17 w(COUNT,)10 b(171)1048 1563 y(MPI)p 1140 1563 V 16 w(GR)o(OUP)p 1324 1563 V 17 w(DIFFERENCE,)k(128)1048 1620 y(MPI)p 1140 1620 V 16 w(GR)o(OUP)p 1324 1620 V 17 w(EX)o(CL,)g(129)1048 1676 y(MPI)p 1140 1676 V 16 w(GR)o(OUP)p 1324 1676 V 17 w(FREE,)g(131)1048 1732 y(MPI)p 1140 1732 V 16 w(GR)o(OUP)p 1324 1732 V 17 w(INCL,)h(129)1048 1789 y(MPI)p 1140 1789 V 16 w(GR)o(OUP)p 1324 1789 V 17 w(INTERSECTION,)h(128)1048 1845 y(MPI)p 1140 1845 V 16 w(GR)o(OUP)p 1324 1845 V 17 w(RANGE)p 1509 1845 V 16 w(EX)o(CL,)f(130)1048 1902 y(MPI)p 1140 1902 V 16 w(GR)o(OUP)p 1324 1902 V 17 w(RANGE)p 1509 1902 V 16 w(INCL,)h(130)1048 1958 y(MPI)p 1140 1958 V 16 w(GR)o(OUP)p 1324 1958 V 17 w(RANK,)f(127)1048 2015 y(MPI)p 1140 2015 V 16 w(GR)o(OUP)p 1324 2015 V 17 w(SIZE,)g(127)1048 2071 y(MPI)p 1140 2071 V 16 w(GR)o(OUP)p 1324 2071 V 17 w(TRANSLA)l(TE)p 1622 2071 V 17 w(RANKS,)c(127)1048 2128 y(MPI)p 1140 2128 V 16 w(GR)o(OUP)p 1324 2128 V 17 w(UNION,)k(128)1048 2226 y(MPI)p 1140 2226 V 16 w(INIT,)h(189)1048 2282 y(MPI)p 1140 2282 V 16 w(INITIALIZED,)g(190)1048 2338 y(MPI)p 1140 2338 V 16 w(INTER)o(COMM)p 1454 2338 V 16 w(FINISH,)g(139)1048 2395 y(MPI)p 1140 2395 V 16 w(INTER)o(COMM)p 1454 2395 V 16 w(MAKE,)f(141)1048 2451 y(MPI)p 1140 2451 V 16 w(INTER)o(COMM)p 1454 2451 V 16 w(MER)o(GE,)g(140)1048 2508 y(MPI)p 1140 2508 V 16 w(INTER)o(COMM)p 1454 2508 V 16 w(NAME)p 1611 2508 V 17 w(ST)l(AR)l(T,)10 b(142,)1214 2564 y(143)1048 2621 y(MPI)p 1140 2621 V 16 w(INTER)o(COMM)p 1454 2621 V 16 w(ST)l(AR)l(T,)16 b(139)1048 2677 y(MPI)p 1140 2677 V 16 w(IPR)o(OBE,)g(49)941 2828 y(223)p eop %%Page: 224 231 224 230 bop 75 -100 a FD(224)1326 b Fp(MPI)15 b(F)l(unction)h(Index)75 45 y FD(MPI)p 167 45 14 2 v 16 w(IRECV,)g(36)75 102 y(MPI)p 167 102 V 16 w(IRSEND,)g(35)75 158 y(MPI)p 167 158 V 16 w(ISEND,)g(35)75 214 y(MPI)p 167 214 V 16 w(ISSEND,)g(36)75 312 y(MPI)p 167 312 V 16 w(MAKE)p 325 312 V 17 w(CAR)l(T,)f(165)75 369 y(MPI)p 167 369 V 16 w(MAKE)p 325 369 V 17 w(DIMS,)g(166)75 425 y(MPI)p 167 425 V 16 w(MAKE)p 325 425 V 17 w(GRAPH,)g(167)75 482 y(MPI)p 167 482 V 16 w(MAP)p 290 482 V 16 w(CAR)l(T,)h(174)75 538 y(MPI)p 167 538 V 16 w(MAP)p 290 538 V 16 w(GRAPH,)f(175)75 636 y(MPI)p 167 636 V 16 w(P)l(A)o(CK,)g(81)75 693 y(MPI)p 167 693 V 16 w(P)l(A)o(CK)p 311 693 V 17 w(SIZE,)g(86)75 749 y(MPI)p 167 749 V 16 w(PR)o(OBE,)h(50)75 806 y(MPI)p 167 806 V 16 w(PTIME,)f(188)75 862 y(MPI)p 167 862 V 16 w(PUT)p 281 862 V 17 w(A)l(TTR,)g(146)75 960 y(MPI)p 167 960 V 16 w(RECV,)g(19)75 1016 y(MPI)p 167 1016 V 16 w(REDUCE,)g(109)75 1073 y(MPI)p 167 1073 V 16 w(REDUCE)p 380 1073 V 17 w(SCA)l(TTER,)g(116)75 1129 y(MPI)p 167 1129 V 16 w(REQUEST)p 405 1129 V 18 w(FREE,)f(55)75 1186 y(MPI)p 167 1186 V 16 w(RSEND,)i(32)75 1284 y(MPI)p 167 1284 V 16 w(SCAN,)f(118)75 1340 y(MPI)p 167 1340 V 16 w(SCA)l(TTER,)h(99)75 1397 y(MPI)p 167 1397 V 16 w(SCA)l(TTER)-5 b(V,)16 b(100)75 1453 y(MPI)p 167 1453 V 16 w(SEND,)f(16)75 1510 y(MPI)p 167 1510 V 16 w(SENDRECV,)g(56)75 1566 y(MPI)p 167 1566 V 16 w(SENDRECV)p 439 1566 V 17 w(REPLA)o(CE,)g(57)75 1623 y(MPI)p 167 1623 V 16 w(SET)p 272 1623 V 17 w(ERRHANDLER,)h(186)75 1679 y(MPI)p 167 1679 V 16 w(SSEND,)f(32)75 1735 y(MPI)p 167 1735 V 16 w(ST)l(AR)l(T,)h(53)75 1792 y(MPI)p 167 1792 V 16 w(ST)l(AR)l(T)l(ALL,)h(54)75 1848 y(MPI)p 167 1848 V 16 w(SUGGEST)p 403 1848 V 16 w(BUFFER)p 609 1848 V 17 w(P)l(ARAMS,)10 b(182)75 1946 y(MPI)p 167 1946 V 16 w(TEST,)15 b(38)75 2003 y(MPI)p 167 2003 V 16 w(TEST)p 305 2003 V 17 w(CANCELLED,)g(51)75 2059 y(MPI)p 167 2059 V 16 w(TEST)p 305 2059 V 17 w(SER)-5 b(VER,)15 b(143)75 2116 y(MPI)p 167 2116 V 16 w(TEST)l(ALL,)h(42,)e(47)75 2172 y(MPI)p 167 2172 V 16 w(TEST)l(ANY,)h(41,)g(47)75 2229 y(MPI)p 167 2229 V 16 w(TESTSOME,)g(44)75 2285 y(MPI)p 167 2285 V 16 w(TOPO)p 317 2285 V 17 w(ST)l(A)l(TUS,)h(169)75 2341 y(MPI)p 167 2341 V 16 w(TYPE)p 312 2341 V 17 w(COMMIT,)e(68)75 2398 y(MPI)p 167 2398 V 16 w(TYPE)p 312 2398 V 17 w(CONTIGUOUS,)h(60)75 2454 y(MPI)p 167 2454 V 16 w(TYPE)p 312 2454 V 17 w(COUNT,)g(67)75 2511 y(MPI)p 167 2511 V 16 w(TYPE)p 312 2511 V 17 w(EXTENT,)f(66)75 2567 y(MPI)p 167 2567 V 16 w(TYPE)p 312 2567 V 17 w(FREE,)g(69)75 2624 y(MPI)p 167 2624 V 16 w(TYPE)p 312 2624 V 17 w(HINDEXED,)g(64)75 2680 y(MPI)p 167 2680 V 16 w(TYPE)p 312 2680 V 17 w(HVECTOR,)h(62)1048 45 y(MPI)p 1140 45 V 16 w(TYPE)p 1285 45 V 16 w(INDEXED,)g(63)1048 102 y(MPI)p 1140 102 V 16 w(TYPE)p 1285 102 V 16 w(SIZE,)g(67)1048 158 y(MPI)p 1140 158 V 16 w(TYPE)p 1285 158 V 16 w(STR)o(UCT,)g(65)1048 214 y(MPI)p 1140 214 V 16 w(TYPE)p 1285 214 V 16 w(VECTOR,)g(61)1048 312 y(MPI)p 1140 312 V 16 w(UNP)l(A)o(CK,)g(82)1048 369 y(MPI)p 1140 369 V 16 w(UNP)l(A)o(CK)p 1352 369 V 17 w(SIZE,)g(86)1048 425 y(MPI)p 1140 425 V 16 w(USER)p 1279 425 V 17 w(ALLREDUCE,)h(115)1048 482 y(MPI)p 1140 482 V 16 w(USER)p 1279 482 V 17 w(ALLREDUCEA,)g(116)1048 538 y(MPI)p 1140 538 V 16 w(USER)p 1279 538 V 17 w(REDUCE,)f(112)1048 595 y(MPI)p 1140 595 V 16 w(USER)p 1279 595 V 17 w(REDUCE)p 1493 595 V 16 w(SCA)l(TTER,)h(117)1048 651 y(MPI)p 1140 651 V 16 w(USER)p 1279 651 V 17 w(REDUCE)p 1493 651 V 16 w(SCA)l(TTERA,)g(117)1048 708 y(MPI)p 1140 708 V 16 w(USER)p 1279 708 V 17 w(REDUCEA,)f(114)1048 764 y(MPI)p 1140 764 V 16 w(USER)p 1279 764 V 17 w(SCAN,)g(118)1048 821 y(MPI)p 1140 821 V 16 w(USER)p 1279 821 V 17 w(SCANA,)g(119)1048 877 y(MPI)p 1140 877 V 16 w(USER)p 1279 877 V 17 w(SPECIFIES)p 1534 877 V 17 w(BUFFER,)g(184)1048 975 y(MPI)p 1140 975 V 16 w(W)-5 b(AIT,)15 b(37)1048 1031 y(MPI)p 1140 1031 V 16 w(W)-5 b(AIT)l(ALL,)16 b(42,)e(47)1048 1088 y(MPI)p 1140 1088 V 16 w(W)-5 b(AIT)l(ANY,)15 b(40,)g(46)1048 1144 y(MPI)p 1140 1144 V 16 w(WTIME,)f(188)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .