%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: mpi-report.dvi %%CreationDate: Thu Mar 31 10:40:26 1994 %%Pages: 236 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips -o mpi-report.ps mpi-report %DVIPSSource: TeX output 1994.03.31:1038 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /@rigin{isls{[ 0 -1 1 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{Resolution hsize -72 div mul 0 TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{ /nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{ /sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0] N df-tail}B /E{pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get}B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{ 128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]} if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{ cc 1 add D}B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258431 52099146 1000 300 300 (/ogi/staff/otto/papers/Editor/MPI/draftFinal/mpi-report.dvi) @start /Fa 2 63 df<0000038000000F0000003C000000F0000003C000000F0000003C 000000F0000003C000000F0000003C000000F0000000F00000003C0000000F00000003C0 000000F00000003C0000000F00000003C0000000F00000003C0000000F00000003801918 7D9520>60 D 62 D E /Fb 17 118 df<387C7E7E3E0E1E1C78F060070B798416>44 D<1FE0003FF0007FF800783C00300E00000E00000E0003FE001FFE003E0E00700E00E00E 00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<03F80FFC1FFE3C1E78 0C7000E000E000E000E000E000F000700778073E0E1FFC0FF803F010127D9116>99 D<003F00007F00003F0000070000070000070000070003C7000FF7001FFF003C1F00780F 00700700E00700E00700E00700E00700E00700E00700700F00700F003C1F001FFFE00FE7 F007C7E014197F9816>I<03E00FF81FFC3C1E780E7007E007FFFFFFFFFFFFE000E00070 0778073C0F1FFE0FFC03F010127D9116>I<001F00007F8000FF8001E78001C30001C000 01C0007FFF00FFFF00FFFF0001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C0003FFE007FFF003FFE0011197F9816>I<018003C003C00180 00000000000000007FC07FC07FC001C001C001C001C001C001C001C001C001C001C001C0 01C07FFFFFFF7FFF101A7D9916>105 D<003000780078003000000000000000001FF81F F81FF8003800380038003800380038003800380038003800380038003800380038003800 38003800386070F0F0FFE07FC03F800D237E9916>I108 DI<7E3C00FEFE007FFF000F 87800F03800E03800E03800E03800E03800E03800E03800E03800E03800E03800E03807F C7F0FFE7F87FC7F01512809116>I<03E0000FF8001FFC003C1E00780F00700700E00380 E00380E00380E00380E00380F00780700700780F003C1E001FFC000FF80003E00011127E 9116>I<7E3E00FEFF007FFF800F83C00F00E00E00E00E00700E00700E00700E00700E00 700E00700E00E00F01E00F83C00FFF800EFF000E3C000E00000E00000E00000E00000E00 000E00007FC000FFE0007FC000141B809116>I114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007F8003C600EE0 0EF00EF81EFFFCFFF8C7E00F127D9116>I<0300000700000700000700000700007FFF00 FFFF00FFFF00070000070000070000070000070000070000070000070100070380070380 07038007870003FE0001FC0000F80011177F9616>I<7E1F80FE3F807E1F800E03800E03 800E03800E03800E03800E03800E03800E03800E03800E03800E03800E0F800FFFF007FB F803E3F01512809116>I E /Fc 1 1 df0 D E /Fd 1 106 df<040C0000000000705898983030606464683006127E910B>105 D E /Fe 11 90 df22 D<0038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 38003800380038003800380038FFF8FFF8FFF80D63808118>I<00001800003000006000 00E00000C0000180000380000700000600000E00000C00001C0000380000380000700000 700000E00000E00001E00001C00001C0000380000380000380000780000700000700000F 00000E00000E00001E00001E00001E00001C00001C00003C00003C00003C00003C000038 0000780000780000780000780000780000780000780000780000700000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0000070 00007800007800007800007800007800007800007800007800003800003C00003C00003C 00003C00001C00001C00001E00001E00001E00000E00000E00000F000007000007000007 800003800003800003800001C00001C00001E00000E00000E00000700000700000380000 3800001C00000C00000E000006000007000003800001800000C00000E000006000003000 0018157C768121>32 DI<00001C00003C00 00F80001E00003C0000780000F00000E00001E00003C00003C00003C0000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 F00000F00000F00001E00001E00003C0000380000700000E00001C0000780000E00000E0 00007800001C00000E000007000003800003C00001E00001E00000F00000F00000F00000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 7800007800003C00003C00003C00001E00000E00000F000007800003C00001E00000F800 003C00001C167C7B8121>40 D<0018007800F001E003C007800F001F001E003E003C007C 007C007800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F8 00F800F800F800F800F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C00F800F800F800F001F001E003E003C0078007000E001C 003800F000C000F00038001C000E000700078003C003E001E001F000F000F800F800F800 7C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C0E4D798025>60 D62 D80 D89 D E /Ff 7 116 df<60F0F070101020204040040A7D830A>59 D<003E000C000C000C000C0018001800180018073018F0307060706060C060C060C06080 C080C480C4C1C446C838700F177E9612>100 D<03000380030000000000000000000000 00001C002400460046008C000C0018001800180031003100320032001C0009177F960C> 105 D<00180038001000000000000000000000000001C002200430043008600060006000 6000C000C000C000C001800180018001806300E300C60078000D1D80960E>I<383C1E00 44C6630047028100460301008E0703000C0603000C0603000C060600180C0600180C0620 180C0C20180C0C4030180440301807801B0E7F8D1F>109 D<383C0044C6004702004602 008E06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15 >I<07C00C201870187038001E000FC003E000606060E060C0C0C1803F000C0E7E8D10> 115 D E /Fg 12 94 df<07C018303018701C600C600CE00EE00EE00EE00EE00EE00EE0 0EE00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF000700070007 00070007000700070007000700070007000700070007000700070007007FF00C157E9412 >I<0F8030E040708030C038E0384038003800700070006000C00180030006000C080808 10183FF07FF0FFF00D157E9412>I<0FE030306018701C701C001C00180038006007E000 300018000C000E000EE00EE00EC00C401830300FE00F157F9412>I<00300030007000F0 01F001700270047008701870107020704070C070FFFE0070007000700070007003FE0F15 7F9412>I<20303FE03FC0240020002000200020002F8030E020700030003800384038E0 38E0388030406020C01F000D157E9412>I<01F00608080C181C301C70006000E000E3E0 EC30F018F00CE00EE00EE00E600E600E300C3018183007C00F157F9412>I<40007FFE7F FC7FF8C008801080200040008000800100010003000200060006000E000E000E000E000E 0004000F167E9512>I<07E018302018600C600C700C78183E101F600FC00FF018F8607C 601EC00EC006C006C004600C38300FE00F157F9412>I61 D91 D93 D E /Fh 5 82 df0 D<020002000200C218F2 783AE00F800F803AE0F278C2180200020002000D0E7E8E12>3 D<000300030006000600 0C000C00180018003000300060006000C000C00180018003000300060006000C000C0018 0018003000300060006000C0004000101E7B9600>54 D<01FFE00FFFF818E0FC20E03E60 E01EC0E00E00E00E00E00C00C00C00C01801C01001C02001804001818003BE0003780003 00000700000600000600000E00000C00000C000018000017187F9619>80 D<001FC0007FE00181F00200F804007808003C18003C30001C30001C60001C60001CE000 18E00018E00030E00030F00060F808407C30803FE1001F02000004000038003FE0067FE0 0C0FFC0800FFF0001FC0171B7C961D>I E /Fi 61 122 df<01F1C003F1C007F1C00701 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFF1C0FFF1C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C0121D809C16>13 D<00F0000001F8000003FC0000039C0000070E0000070E0000 070E0000070E0000070E0000071E0000073C0000073800000778000003F0000003E00300 07C003000F8006001FC006003DC0060078E00C0070F00C00F0701800E0783800E03C3000 E01C6000E00EC000700F800070078040381FE1C01FF8FFC007E03F001A1F7E9D1F>38 D<00E001C0038007000E000E001C001C003800380038007000700070007000E000E000E0 00E000E000E000E000E000E000E000E000E00070007000700070003800380038001C001C 000E000E000700038001C000E00B2A7E9E10>40 DI<018001C0018001806186F99F7DBE1FF807E007E01FF87DBEF99F618601800180 01C0018010127E9E15>I<00060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFE0FFFFFFE000 060000000600000006000000060000000600000006000000060000000600000006000000 0600000006000000060000000600001B1C7E9720>II< FFE0FFE00B0280890E>I<03C00FF01FF83C3C381C700E700E700EE007E007E007E007E0 07E007E007E007E007E007E007E007E007700E700E700E381C3C3C1FF80FF007E0101D7E 9B15>48 D<010007003F00FF00C700070007000700070007000700070007000700070007 000700070007000700070007000700070007000700FFF8FFF80D1C7C9B15>I<07C01FF0 3878701C601EE00EC00F400F400700070007000F000E001E001C003C007800F001E001C0 038007000E001C0038007000FFFFFFFF101C7E9B15>I58 D<7FFFFFC0FFFFFFE0000000000000000000000000000000 0000000000000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<001C0000003E0000003E0000002E0000006700000067000000E7800000C7800000C380 0001C3C0000183C0000181C0000381E0000381E0000700F0000700F0000600F0000E0078 000FFFF8000FFFF8001C003C001C003C0018003C0038001E0038001E0070001F0070000F 0070000F00E0000780191D7F9C1C>65 DI<003FC000FFF003C0F00780300F00001E00003C00003C0000780000780000780000 F00000F00000F00000F00000F00000F00000F00000F00000F00000780000780000780000 3C00003C00001E00000F000807801803C07800FFF0003F80151F7D9D1B>IIII<003F8001FFF003C0F80780 380F00181E00003C00003C0000780000780000780000F00000F00000F00000F00000F000 00F00000F007F8F007F8F000387800387800387800383C00383C00381E00380F00380780 3803C0F801FFF0003F80151F7D9D1C>III75 DIII<003F000001FFE00003FFF00007C0F800 0F807C001E001E003E001F003C000F00780007807800078078000780F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F80007C07800078078000780 7C000F803C000F003E001F001F003E000F807C0007C0F80003FFF00001FFE000003F0000 1A1F7E9D1F>II<003F000001FFE0 0003FFF00007C0F8000F807C001F003E003E001F003C000F007800078078000780780007 80F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003 C07800078078000780780E07803C0F0F003E079F001E03DE000F83FC0007C1F80003FFF0 0001FFF800003F780000003C0000003E0000001F0000000F801A237E9D1F>II<03F8000FFE001C0F00380700700300600000 E00000E00000E00000E00000F000007800007F00003FE0001FFC0007FE0001FF00001F80 0007800003C00003C00001C00001C00001C00001C0C00180E00380F007007C0E001FFC00 07F000121F7E9D17>IIII I<78000E007C001E003C003C001E0038000F0070000F00F0000781E00003C1C00001C3C0 0001E7800000F70000007E0000003E0000003C0000003C0000007E00000077000000E780 0001E3800003C1C0000381E0000700F0000F00F8000E0078001C003C003C003E0078001F 0070000F00F0000F80191D7F9C1C>II<7FFFF07FFF F00001E00003E00003C00007C0000780000F00001F00001E00003E00003C0000780000F8 0000F00001F00001E00003C00007C0000780000F80000F00001E00003E00003C00007C00 00780000FFFFF0FFFFF0141D7E9C19>I<0FC03FF07FF87038401C001C001C00FC0FFC3F FC781CE01CE01CE01CF07C7FFC7FDC3F1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000 E000E000E000700070043C1C1FFC0FF807E00E127E9112>I<000E000E000E000E000E00 0E000E000E000E000E000E0F8E1FEE3FFE7C3E700E700EE00EE00EE00EE00EE00EE00EF0 0E701E7C3E3FFE1FEE0F8E0F1D7E9C15>I<07C01FE03FF078787018601CFFFCFFFCFFFC E000E000E000700070043C1C3FFC1FF807E00E127E9112>I<00FC01FC03FC07000E000E 000E000E000E000E000E00FFE0FFE00E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E1D809C0D>I<03C3C00FFFC01FFFC01C3800381C00381C00 381C00381C00381C001C38001FF8001FF0003BC0003800003800001FFC001FFF003FFF80 700780E001C0E001C0E001C0F003C07C0F803FFF001FFE0007F800121B7F9115>III107 DIII< 03F0000FFC001FFE003C0F00780780700380E001C0E001C0E001C0E001C0E001C0F003C0 7003807807803C0F001FFE000FFC0003F00012127F9115>II114 D<1FC03FF07FF0F030E000E000F0007F 003FC01FE000F0003800388038F078FFF07FE01FC00D127F9110>I<1C001C001C001C00 1C001C00FFE0FFE01C001C001C001C001C001C001C001C001C001C001C001C001C201FF0 0FF007C00C187F970F>IIII<70038078 07003C0E001C1C000E1C0007380003F00001E00001C00001E00003F0000738000E18000E 1C001C0E00380700700380F003C01212809113>II E /Fj 64 122 df<003F83F800FF8FF801FF9FF803E03E0003803800 0780780007807800078078000780780007807800078078000780780007807800FFFC7FC0 FFFC7FC0FFFC7FC007807800078078000780780007807800078078000780780007807800 078078000780780007807800078078000780780007807800078078000780780007807800 0780780007807800078078001D2380A21C>11 D<000078007C7800FC7801FC7803C00003 8000078000078000078000078000078000078000078000078000FFFC78FFFC78FFFC7807 807807807807807807807807807807807807807807807807807807807807807807807807 8078078078078078078078078078078078078078152480A31A>I<0000C018000000C018 000000C01800000180300000018030000001803000000180300000030060000003006000 000300600000030060000003006000000600C000000600C000000600C000000600C00000 0C018000FFFFFFFFC0FFFFFFFFC000180300000018030000001803000000180300000030 060000003006000000300600000030060000FFFFFFFFC0FFFFFFFFC000600C000000C018 000000C018000000C018000000C018000001803000000180300000018030000001803000 0003006000000300600000030060000003006000000600C000000600C000000600C00000 222D7DA229>35 D<787878781830306060E0050A7CA20E>39 D<007000E001C003C00780 07000F000E001E001C003C003C00380078007800780078007000F000F000F000F000F000 F000F000F000F000F000F000F000F000F0007000780078007800780038003C003C001C00 1E000E000F000700078003C001C000E000700C327DA413>II<787878781830306060E0050A 7C830E>44 DII<00C001C007C0FF C0FFC0FBC003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C003C003C0FFFFFFFFFFFF10227CA118>49 D<03F0000FFC001FFE003C1F003007807007C06003C0E003E0C001E04001E04001E00001 E00001E00001E00003C00003C0000780000780000F00001E00003C0000780000F00001E0 0001C0000380000700000E00001C0000380000700000FFFFE0FFFFE0FFFFE013227EA118 >I<01F00007FC001FFF003E0F003807807003C02003C02003C00003C00003C00003C000 0780000780000F00001E0003FC0003F80003FE00000F000007800003C00003C00001E000 01E00001E00001E00001E08001E0C003C0E003C07007803C0F801FFF000FFC0003F00013 237EA118>I<001F00001F00002F00002F00006F0000EF0000CF0001CF0001CF00038F00 038F00078F00070F000F0F000E0F001E0F003C0F003C0F00780F00780F00F00F00FFFFF8 FFFFF8FFFFF8000F00000F00000F00000F00000F00000F00000F00000F00000F0015217F A018>I<3FFF803FFF803FFF803C00003C00003C00003C00003C00003C00003C00003C00 003C00003CF8003FFE003FFF003F0F803E07803C03C03803C00001E00001E00001E00001 E00001E00001E00001E04003C04003C0E003C07007807C1F003FFE000FFC0003F0001322 7EA018>I<007E0001FF0003FF0007C1000F00001E00001E00003C00003C000078000078 000078F800F1FE00F7FF00FF0F80FC0780F803C0F803C0F801C0F001E0F001E0F001E0F0 01E0F001E07001E07001E07801E07803C03803C03C03801C07801F0F000FFE0007FC0001 F00013237EA118>II<01F00007FC000FFE001E0F003C07803C07807803C07803C07803C07803 C07803C03803803C07801E0F000F1E0007FC0003F8000FFE001E0F003C07807803C07803 C0F001E0F001E0F001E0F001E0F001E0F001E07803C07803C03C07803E0F801FFF0007FC 0001F00013237EA118>I<01F00007FC000FFE001E0F003C07003807807803807803C0F0 03C0F001C0F001E0F001E0F001E0F001E0F001E0F001E07003E07803E07803E03C07E03E 1FE01FFDE00FF1E003E3C00003C00003C0000380000780000700000F00101E00183C003F F8001FF00007C00013237EA118>II<001F0000001F0000003F8000003F8000003B8000007BC0000073C000 0071C00000F1E00000F1E00000E0E00001E0F00001E0F00001C0F00003C0780003C07800 0380780007803C0007803C0007003C000F001E000F001E000FFFFE001FFFFF001FFFFF00 1C000F003C0007803C00078038000780780003C0780003C0700003C0F00001E0F00001E0 E00001E01B237EA220>65 DI<000FF000007FFC0000FFFF0001F01F0003C00700 078000000F0000001E0000003E0000003C0000003C000000780000007800000078000000 F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000 7800000078000000780000003C0000003C0000003E0000001E0000000F00000007800080 03C0038001F00F8000FFFF00007FFC00000FF00019257DA31F>IIII<000FF000003FFE0000FFFF8001F80F8003E00380078000000F0000001E0000001E00 00003C0000003C000000780000007800000078000000F0000000F0000000F0000000F000 0000F0000000F0000000F000FFC0F000FFC0F000FFC0780003C0780003C0780003C03C00 03C03C0003C01E0003C01E0003C00F0003C0078003C003E003C001F807C000FFFFC0003F FF00000FF8001A257DA321>III76 DII<001FC000007FF00001FFFC0003F07E0007C01F000F800F801F0007C0 1E0003C03C0001E03C0001E0780000F0780000F0780000F070000070F0000078F0000078 F0000078F0000078F0000078F0000078F0000078F0000078F0000078780000F0780000F0 780000F07C0001F03C0001E03E0003E01E0003C01F0007C00F800F8007C01F0003F07E00 01FFFC00007FF000001FC0001D257DA324>II82 D<00FE0003FFC007FFE00F81E01E00603C00003C00007800007800007800007800007800 007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007E00001F00000F00000 F8000078000078000078000078000078000078C000F0E000F0F801E07E07C03FFF800FFF 0001FC0015257EA31B>IIII<780003E07C0003C03E0007C0 1E000F800F000F000F801F0007C03E0003C03C0003E07C0001F0780000F0F0000079F000 007DE000003FC000001FC000001F8000000F0000001F8000001FC000003FC000007DE000 0079F00000F0F00001F0780001E03C0003C03E0007C01E0007800F000F000F801F000780 1E0003C03C0003E07C0001F0780000F0F80000F81D237FA220>88 D<07E01FF83FFC381E201E000F000F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF0 0FF83F7FFF3FEF1F8F10167E9517>97 DI<01FC0007FF000FFF801F03803C0180 780000780000700000F00000F00000F00000F00000F00000F00000780000780000780000 3C00401F03C00FFFC007FF8001FC0012167E9516>I<0003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C003E3C00FFBC01FFFC03F0F C03C07C07803C07803C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07803 C07803C03C07C03E0FC01FFFC00FFBC003E3C012237EA219>I<03F00007FC001FFE003E 0F003C0780780380780380F001C0FFFFC0FFFFC0FFFFC0F00000F00000F0000070000078 00007800003C00801F07800FFF8007FF0001F80012167E9516>I<003F00FF01FF03C003 8007800780078007800780078007800780FFF8FFF8FFF807800780078007800780078007 80078007800780078007800780078007800780078007800780102380A20F>I<01F07807 FFF80FFFF81F1F001E0F003C07803C07803C07803C07803C07801E0F001F1F000FFE001F FC0019F0003800003800003C00001FFE001FFFC01FFFE03FFFF07801F07800F8F00078F0 0078F00078F000787800F03E03E01FFFC00FFF8001FC0015217F9518>III<00F000F000F000F000000000000000000000000000000000000000F000F000 F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000 F000F000F000F000F000F000F080F0E3E0FFE07FC01F000C2D83A20D>IIIII<01FC0007FF000FFF801F07C03C01E07800F07800F0700070F00078F00078 F00078F00078F00078F000787800F07800F07C01F03E03E01F07C00FFF8007FF0001FC00 15167F9518>II<03E3C007FBC00FFFC01F0FC03C07C07C03C07803C0F803C0F003C0F003C0F003C0F0 03C0F003C0F003C0F803C07803C07803C03C07C03F0FC01FFFC00FFBC003E3C00003C000 03C00003C00003C00003C00003C00003C00003C00003C00003C012207E9519>II<07F01FFC3FFE3C0E7806780078007C003F003FF01FF80FFC 01FE001F000F000F000FC00FF81EFFFE3FFC0FF010167F9513>I<0F000F000F000F000F 000F00FFF8FFF8FFF80F000F000F000F000F000F000F000F000F000F000F000F000F000F 000F080F1C07FC07F803E00E1C7F9B12>IIII<7801F07C01E03E03C01E07C00F0780 078F0007DE0003FC0001FC0000F80000700000F80001FC0003DC00039E00078F000F0780 1E07801E03C03C01E07800F0F800F81516809516>II E /Fk 60 124 df<007E1F0001C1B1800303E3 C00703C3C00E03C1800E01C0000E01C0000E01C0000E01C0000E01C0000E01C000FFFFFC 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0007F87FC001A1D80 9C18>11 D<007E0001C1800301800703C00E03C00E01800E00000E00000E00000E00000E 0000FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C07F87F8151D809C17>I<6060F0F0F8F8686808 0808080808101010102020404080800D0C7F9C15>34 D<004000800100020006000C000C 0018001800300030007000600060006000E000E000E000E000E000E000E000E000E000E0 00E000E000600060006000700030003000180018000C000C00060002000100008000400A 2A7D9E10>40 D<800040002000100018000C000C00060006000300030003800180018001 8001C001C001C001C001C001C001C001C001C001C001C001C00180018001800380030003 00060006000C000C00180010002000400080000A2A7E9E10>I<60F0F070101010102020 4080040C7C830C>44 DI<60F0F06004047C830C>I<0001000300 0600060006000C000C000C0018001800180030003000300060006000C000C000C0018001 800180030003000300060006000C000C000C001800180018003000300030006000600060 00C000C00010297E9E15>I<03C00C301818300C300C700E60066006E007E007E007E007 E007E007E007E007E007E007E007E007E00760066006700E300C300C18180C3007E0101D 7E9B15>I<030007003F00C7000700070007000700070007000700070007000700070007 0007000700070007000700070007000700070007000F80FFF80D1C7C9B15>I<07C01830 201C400C400EF00FF80FF807F8077007000F000E000E001C001C00380070006000C00180 030006010C01180110023FFE7FFEFFFE101C7E9B15>I<60F0F060000000000000000000 0060F0F06004127C910C>58 D<60F0F0600000000000000000000060F0F0701010101020 204080041A7C910C>I<7FFFFFC0FFFFFFE0000000000000000000000000000000000000 0000000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<000600000006000000060000000F0000000F0000000F00000017800000178000001780 000023C0000023C0000023C0000041E0000041E0000041E0000080F0000080F0000180F8 000100780001FFF80003007C0002003C0002003C0006003E0004001E0004001E000C001F 001E001F00FF80FFF01C1D7F9C1F>65 D<001F808000E0618001801980070007800E0003 801C0003801C00018038000180780000807800008070000080F0000000F0000000F00000 00F0000000F0000000F0000000F0000000F0000000700000807800008078000080380000 801C0001001C0001000E000200070004000180080000E03000001FC000191E7E9C1E>67 DIII<001F808000E0618001801980070007800E0003801C00 03801C00018038000180780000807800008070000080F0000000F0000000F0000000F000 0000F0000000F0000000F000FFF0F0000F80700007807800078078000780380007801C00 07801C0007800E00078007000B800180118000E06080001F80001C1E7E9C21>I73 D76 DII<003F800000E0E0000380380007 001C000E000E001C0007003C00078038000380780003C0780003C0700001C0F00001E0F0 0001E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0700001C0780003C078 0003C0380003803C0007801C0007000E000E0007001C000380380000E0E000003F80001B 1E7E9C20>II82 D<07E0801C1980300580700380600180E00180E00080E00080E00080F00000F8 00007C00007FC0003FF8001FFE0007FF0000FF80000F800007C00003C00001C08001C080 01C08001C0C00180C00180E00300D00200CC0C0083F800121E7E9C17>I<7FFFFFC0700F 01C0600F00C0400F0040400F0040C00F0020800F0020800F0020800F0020000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000001F800003FFFC001B1C 7F9B1E>II89 D<08081010202040404040808080808080B0B0F8F8787830300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC000F1C00381C00701C00601C 00E01C40E01C40E01C40603C40304E801F870012127E9115>97 DI<07E00C301878307870306000E000E000E000E000E000E0 0060007004300418080C3007C00E127E9112>I<003F0000070000070000070000070000 070000070000070000070000070000070003E7000C1700180F00300700700700600700E0 0700E00700E00700E00700E00700E00700600700700700300700180F000C370007C7E013 1D7E9C17>I<03E00C301818300C700E6006E006FFFEE000E000E000E000600070023002 18040C1803E00F127F9112>I<00F8018C071E061E0E0C0E000E000E000E000E000E00FF E00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E007FE00F 1D809C0D>I<00038003C4C00C38C01C3880181800381C00381C00381C00381C00181800 1C38000C300013C0001000003000001800001FF8001FFF001FFF803003806001C0C000C0 C000C0C000C06001803003001C0E0007F800121C7F9215>II<18003C003C0018000000000000000000000000000000FC001C 001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00FF80091D7F 9C0C>I<00C001E001E000C000000000000000000000000000000FE000E000E000E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E060E0F0C0 F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180E001C0E001C0E001C0E001C0 E001C0E001C06001807003803003001806000E1C0003F00012127F9115>II<03C1000C3300180B00300F00700700700700E00700E00700E00700E0 0700E00700E00700600700700700300F00180F000C370007C70000070000070000070000 0700000700000700000700003FE0131A7E9116>II<1F9030704030C010 C010E010F8007F803FE00FF000F880388018C018C018E010D0608FC00D127F9110>I<04 000400040004000C000C001C003C00FFE01C001C001C001C001C001C001C001C001C001C 101C101C101C101C100C100E2003C00C1A7F9910>III< FF3FCFE03C0F03801C0701801C0701001C0B01000E0B82000E0B82000E1182000711C400 0711C4000720C40003A0E80003A0E80003C0680001C0700001C070000180300000802000 1B127F911E>I<7F8FF00F03800F030007020003840001C80001D80000F0000070000078 0000F800009C00010E00020E000607000403801E07C0FF0FF81512809116>II<7FFC70386038407040F040E041C003C0038007000F040E041C043C 0C380870087038FFF80E127F9112>II E /Fl 58 124 df<00003FE00000E0100001803800038078000300780007003000070000000700 0000070000000E0000000E0000000E000000FFFFE0000E00E0001C01C0001C01C0001C01 C0001C01C0001C0380003803800038038000380380003807000038070000700700007007 1000700E2000700E2000700E2000E00E2000E0064000E0038000E0000000C0000001C000 0001C000003180000079800000F3000000620000003C0000001D29829F1A>12 D<0E1F3F3F1D0102020404081020C0080E779F0E>39 D<00010002000400080010002000 6000C0018001800300070006000E000C001C0018003800380030007000700060006000E0 00E000C000C000C000C000C000C000C000C000C000C000C000C000C00040006000600020 00100010000800102E79A113>I<00100000080000040000060000020000030000030000 030000010000018000018000018000018000018000018000018000038000038000038000 0300000300000300000700000700000600000600000E00000C00000C00001C0000180000 380000300000700000600000E00000C0000180000100000300000600000C000018000030 0000600000800000112E80A113>I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I<70F8F8F0E005057B840E>I<000F800030E000E07001C0 700380300380380700380F00780F00780E00781E00781E00703C00F03C00F03C00F03C00 F07801E07801E07801E07801C07003C0F003C0F00380F00780F00700700700700E00701C 003038001870000FC000151F7C9D17>48 D<000200020006000E003C00DC031C001C0038 003800380038007000700070007000E000E000E000E001C001C001C001C0038003800380 03800780FFF80F1E7B9D17>I<001F000061800080E00100E00200700220700420700410 700820F00820F00820F00840E00881E00703C0000380000700000C000018000060000080 000300000400000800401000401000802001807E030047FF0041FE0080FC00807800141F 7C9D17>I<0000600000E00000E00000E00001C00001C00001C000038000038000030000 0700000700000600000E00000C0000180000180000300000300000630000C70000870001 0700030700060E00040E00080E003F8E00607C00801FC0001C00001C0000380000380000 380000380000700000700000600013277E9D17>52 D<001F0000718000C0C00180C00380 E00700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C01C03C01C03C01C07C01C0F 800C0F8006378003C700000F00000E00000E00001C00601C00F03800F07000E0600080C0 004380003E0000131F7B9D17>57 D<00000200000006000000060000000E0000001E0000 001E0000003F0000002F0000004F0000004F0000008F0000010F0000010F0000020F0000 020F0000040F00000C0F0000080F0000100F0000100F0000200F80003FFF800040078000 C007800080078001000780010007800200078002000780060007801E000F80FF807FF81D 207E9F22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C003C00 3C003C003C003C0078007800780078007800F0007801E000F0078000FFFE0000F00F8000 F003C001E001C001E001E001E001E001E001E003C001E003C001E003C001E003C001C007 8003C00780078007800F0007801E000F007800FFFFE0001E1F7D9E20>I<0000FE020007 8186001C004C0038003C0060003C00C0001C01C0001803800018070000180F0000181E00 00101E0000103C0000003C00000078000000780000007800000078000000F0000000F000 0000F0000000F0000000F000008070000080700000807000010038000100380002001800 04000C001800060020000381C00000FE00001F217A9F21>I<01FFFF80001E00E0001E00 70001E0038001E001C003C001C003C000E003C000E003C000E0078000E0078000E007800 0E0078000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01E000 7803C0007003C0007003C000E003C001C0078001C00780038007800E0007801C000F0070 00FFFFC0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C0004 003C0004003C0004003C00040078080800780800007808000078180000F0300000FFF000 00F0300000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C00040 03C00040078000800780018007800100078007000F001F00FFFFFE001F1F7D9E1F>I<01 FFFFFC001E0038001E0018001E0008001E0008003C0008003C0008003C0008003C000800 78001000780800007808000078080000F0100000F0300000FFF00000F0300001E0200001 E0200001E0200001E0200003C0000003C0000003C0000003C00000078000000780000007 800000078000000F800000FFF800001E1F7D9E1E>I<0000FC040007030C001C00980030 007800E0007801C000380380003003800030070000300E0000301E0000201E0000203C00 00003C00000078000000780000007800000078000000F0000000F000FFF0F0000780F000 0780F0000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E000E00 CC000383040000FC00001E217A9F23>I<01FFF3FFE0001F003E00001E003C00001E003C 00001E003C00003C007800003C007800003C007800003C007800007800F000007800F000 007800F000007800F00000F001E00000FFFFE00000F001E00000F001E00001E003C00001 E003C00001E003C00001E003C00003C007800003C007800003C007800003C00780000780 0F000007800F000007800F000007800F00000F801F0000FFF1FFE000231F7D9E22>I<01 FFF0001F00001E00001E00001E00003C00003C00003C00003C0000780000780000780000 780000F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C00003 C0000780000780000780000780000F8000FFF800141F7D9E12>I<001FFF0000F80000F0 0000F00000F00001E00001E00001E00001E00003C00003C00003C00003C0000780000780 000780000780000F00000F00000F00000F00001E00001E00301E00781E00F83C00F83C00 F0780080700040E00021C0001F000018207D9E18>I<01FFF03FE0001F000F80001E000E 00001E000800001E001000003C002000003C004000003C010000003C0200000078040000 00780800000078100000007830000000F0F0000000F1F8000000F278000000F478000001 E83C000001F03C000001E03C000001E01E000003C01E000003C01E000003C00F000003C0 0F000007800F00000780078000078007800007800780000F8007C000FFF03FF800231F7D 9E23>I<01FFF800001F0000001E0000001E0000001E0000003C0000003C0000003C0000 003C00000078000000780000007800000078000000F0000000F0000000F0000000F00000 01E0000001E0000001E0000001E0008003C0010003C0010003C0030003C0020007800600 0780060007800C0007801C000F007800FFFFF800191F7D9E1D>I<01FE00007FC0001E00 00FC00001E0000F80000170001780000170001780000270002F00000270004F000002700 04F00000270008F00000470009E00000470011E00000470021E00000470021E000008700 43C00000838043C00000838083C00000838083C000010381078000010382078000010382 0780000103840780000203840F00000203880F00000203900F00000203900F00000401E0 1E00000401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0002A1F7D 9E29>I<01FF007FE0001F000F00001F0004000017800400001780040000278008000023 C008000023C008000023C008000041E010000041E010000041F010000040F010000080F0 200000807820000080782000008078200001003C400001003C400001003C400001001E40 0002001E800002001E800002000F800002000F800004000F000004000700000400070000 0C000700001C00020000FF80020000231F7D9E22>I<0001FC0000070700001C01C00030 00E000E0006001C000700380007007800038070000380E0000381E0000381C0000383C00 00383C00003878000078780000787800007878000078F00000F0F00000F0F00000E0F000 01E0F00001C0F00003C0700003807000070078000F0038001E0038003C001C0070000E00 E0000783800001FC00001D217A9F23>I<01FFFF80001E00E0001E0070001E0038001E00 3C003C003C003C003C003C003C003C003C0078007800780078007800F0007800E000F003 C000F00F0000FFFC0000F0000001E0000001E0000001E0000001E0000003C0000003C000 0003C0000003C00000078000000780000007800000078000000F800000FFF000001E1F7D 9E1F>I<01FFFF00001E03C0001E00E0001E0070001E0078003C0078003C0078003C0078 003C0078007800F0007800F0007801E0007801C000F0070000F01E0000FFF00000F03800 01E01C0001E01E0001E00E0001E00F0003C01E0003C01E0003C01E0003C01E0007803C00 07803C0807803C0807803C100F801C10FFF00C20000007C01D207D9E21>82 D<0007E040001C18C0003005800060038000C0038001C001800180010003800100038001 00038001000380000003C0000003C0000003F8000001FF800001FFE000007FF000001FF0 000001F80000007800000078000000380000003800200038002000380020003000600070 00600060006000E0007000C000E8038000C606000081F800001A217D9F1A>I<0FFFFFF0 1E0780E0180780201007802020078020200F0020600F0020400F0020400F0020801E0040 001E0000001E0000001E0000003C0000003C0000003C0000003C00000078000000780000 007800000078000000F0000000F0000000F0000000F0000001E0000001E0000001E00000 01E0000003E00000FFFF00001C1F789E21>I<7FFC1FF807C003C0078001000780010007 8001000F0002000F0002000F0002000F0002001E0004001E0004001E0004001E0004003C 0008003C0008003C0008003C00080078001000780010007800100078001000F0002000F0 002000F0002000F0004000F0004000700080007001000030020000380400000C18000007 E000001D20779E22>I I<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E20F01C40F01C40703C40705C40308C800F070013147C9317>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207B9F15>I< 007E0001C1000300800E07801E07801C07003C0200780000780000780000F00000F00000 F00000F00000F0000070010070020030040018380007C00011147C9315>I<0000780003 F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1C00389 C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E 20F01C40F01C40703C40705C40308C800F070015207C9F17>I<007C01C207010E011C01 3C013802780C7BF07C00F000F000F000F0007000700170023804183807C010147C9315> I<00007800019C00033C00033C000718000700000700000E00000E00000E00000E00000E 0001FFE0001C00001C00001C00001C000038000038000038000038000038000070000070 0000700000700000700000700000E00000E00000E00000E00000C00001C00001C0000180 003180007B0000F300006600003C00001629829F0E>I<003C6000E27001C1E00380E007 00E00F00E00E01C01E01C01E01C01E01C03C03803C03803C03803C03803C07003C07001C 0F001C17000C2E0003CE00000E00000E00001C00001C00301C00783800F0700060E0003F 8000141D7E9315>I<01E0000FE00001C00001C00001C00001C000038000038000038000 038000070000070000071E000763000E81800F01C00E01C00E01C01C03801C03801C0380 1C0380380700380700380700380E10700E20700C20701C20700C40E00CC060070014207D 9F17>I<00C001E001E001C000000000000000000000000000000E003300230043804300 470087000E000E000E001C001C001C003840388030807080310033001C000B1F7C9E0E> I<01E0000FE00001C00001C00001C00001C0000380000380000380000380000700000700 000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC00038E0003870 00387000383840707080707080707080703100E03100601E0013207D9F15>107 D<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C001C 0038003800380038007000700070007100E200E200E200E200640038000A207C9F0C>I< 1C0F80F0002630C318004740640C004780680E004700700E004700700E008E00E01C000E 00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C070803803 8071003803806100380380E10038038062007007006600300300380021147C9325>I<1C 0F802630C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C01C 01C01C03843803883803083807083803107003303001C016147C931A>I<007C0001C300 0301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780 F00700700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C0478 1C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380 700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C00 00FFC000171D809317>I<00F0400388C00705800E03801C03803C038038070078070078 0700780700F00E00F00E00F00E00F00E00F01C00F01C00703C00705C0030B8000F380000 380000380000700000700000700000700000E00000E0000FFE00121D7C9315>I<1C1E00 2661004783804787804707804703008E00000E00000E00000E00001C00001C00001C0000 1C000038000038000038000038000070000030000011147C9313>I<00FC030206010C03 0C070C060C000F800FF007F803FC003E000E700EF00CF00CE008401020601F8010147D93 13>I<018001C0038003800380038007000700FFF007000E000E000E000E001C001C001C 001C003800380038003820704070407080708031001E000C1C7C9B0F>I<0E00C03300E0 2301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C0710 1C0E20180E20180E201C1E200C264007C38014147C9318>I<0E03803307802307C04383 C04301C04700C08700800E00800E00800E00801C01001C01001C01001C02001C02001C04 001C04001C08000E300003C00012147C9315>I<0E00C1C03300E3C02301C3E04381C1E0 4301C0E04701C060870380400E0380400E0380400E0380401C0700801C0700801C070080 1C0701001C0701001C0602001C0F02000C0F04000E13080003E1F0001B147C931E>I<03 83800CC4401068E01071E02071E02070C040E00000E00000E00000E00001C00001C00001 C00001C040638080F38080F38100E5810084C60078780013147D9315>I<0E00C03300E0 2301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C0700 1C0E00180E00180E001C1E000C3C0007DC00001C00001C00003800F03800F07000E06000 C0C0004380003E0000131D7C9316>I123 D E /Fm 51 121 df<00030004000800100020006000C0018001000300060006000C000C00 1C0018001800380030003000700070006000600060006000E000E000E000E000E000E000 60006000600060006000300030001000180008000C00040002000100102E7CA112>40 D<008000C000600020003000180018000C000C000C000E00060006000600060006000600 060006000600060006000E000E000E000C000C001C001800180038003000300060006000 C000C001800100030006000C0018003000600080000F2E7FA112>I<1C3E7E7E3A020204 0408081020C0070E7D840D>44 DI<3078F8787005057C840D>I< 00000400000C00000C0000180000180000300000300000600000600000C00000C0000180 000180000300000300000600000600000C00000C00001800001800003000003000006000 00600000C00000C0000180000180000300000300000600000600000600000C00000C0000 180000180000300000300000600000600000C00000C00000800000162D7EA117>I<007E 0001C3000301800701C00E00C00E00E01C00E01C00E03C01E03801E07801E07801E07801 E07801E07801E0F003C0F003C0F003C0F003C0F003C0F00380F00780E00780E00700E007 00E00E00600E00701C003038003870000FC000131F7C9D17>I<000C001C00FC0F380038 003800380038003800700070007000700070007000E000E000E000E000E000E001C001C0 01C001C001C001C0038003C0FFFE0F1E7C9D17>I<003F8000C1E00100F0020078040078 0400780F007C0F807C0F807C0F00780600780000F80000F00001E00001C0000380000700 000E00001C0000380000600000C0000180000300200600200800401000403FFFC07FFF80 FFFF80161E7E9D17>I<007F000183C00201E00400F00700F00F00F00F01F00F01F00001 E00001E00003C0000380000700000E0000F800000E000007000007800007C00003C00007 C03007C07807C0F807C0F807C0F00780800F00400E00201C0018780007E000141F7D9D17 >I<0000600000600000E00001C00003C00005C0000DC00009C00011C000238000438000 C380008380010380020380040700080700180700100700200700400700FFFFF0000E0000 0E00000E00000E00000E00001C00001E0001FFE0141E7E9D17>I<01803001FFE003FFC0 03FF0003FC00020000020000020000040000040000040000047C000587000603800C0180 0801C00001C00001E00001E00001E00001E07003C0F803C0F003C0E00380800780400700 400E00201C0018700007C000141F7D9D17>I<000F8000704000C0200180E00301E00701 E00E00C01E00001C00003C000038000078F800790E007A07007C0300F80380F80380F003 C0F003C0F003C0F003C0F00780E00780E00780E00700E00F00600E00701C003018001870 0007C000131F7C9D17>I<2000003FFFE07FFFC07FFF80400100C0020080020080040000 08000010000020000040000040000080000180000300000300000700000600000E00000E 00001E00001C00001C00003C00003C00003C0000780000780000780000300000131F799D 17>I<003F0000C1C00100600200600400300C00300C00300C00300C00600E00600F80C0 0FC18007F60003FC0001FC0001FF00063F800C0F801007C03003C06001C06000C0C000C0 C000C0C000C0C00080C0010060030030040018180007E000141F7D9D17>I<007E0001C3 000301800601C00E01C01C00C03C00E03C00E03C01E07801E07801E07801E07801E07803 E07803E03803C03807C01C0BC00C13C003E380000780000780000700000E00600E00F01C 00F01800E0300080600041C0003F0000131F7C9D17>I<07E01838201C401C701CF03CF0 3C603C0038007000E001C001800300020006000400040008000800080008000000000000 000000000030007800F800780070000E20799F15>63 D<00001000000018000000380000 00380000007800000078000000FC000001BC0000013C0000033C0000023C0000063C0000 043E0000081E0000081E0000101E0000101E0000201E0000200F0000400F0000400F0000 FFFF0000800F0001000F8001000780020007800200078004000780040007800C0007C03E 0007C0FF807FFC1E207E9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F800 7800F8007800F8007800F8007800F8007800F000F001F000F001E000F003C000F00F8000 FFFE0000F00F0001E007C001E003C001E003E001E001E001E001E001E001E003C001E003 C003E003C003E003C003C003C007C003C00F8007800F0007803E00FFFFF0001D1F7E9E20 >I<0001F808000E061800380138007000F801E0007803C0007007800030078000300F00 00301F0000301E0000303E0000203C0000007C0000007C0000007C0000007C000000F800 0000F8000000F8000000F8000000F80000007800004078000080780000803C0000803C00 01001C0002000E00020006000C000300100001C0E000003F00001D217B9F21>I<07FFFF 00007C01E0003C00F0003C00780078003C0078003C0078001E0078001E0078001E007800 1F00F0001F00F0001F00F0001F00F0001F00F0001F00F0001F01E0001E01E0003E01E000 3E01E0003E01E0003C01E0007C03C0007803C000F003C000F003C001E003C003C003C007 8007800F0007803C00FFFFE000201F7E9E23>I<07FFFFF8007C0078003C0038003C0018 00780018007800080078000800780008007800080078080800F0100000F0100000F01000 00F0300000FFF00000F0700001E0200001E0200001E0200001E0200001E0000801E00010 03C0001003C0001003C0002003C0002003C0006003C000C0078001C0078007C0FFFFFF80 1D1F7E9E1F>I<07FFFFF8007C0078003C0038003C001800780018007800080078000800 780008007800080078000800F0100000F0100000F0100000F0300000F0700000FFF00001 E0600001E0200001E0200001E0200001E0200001E0000003C0000003C0000003C0000003 C0000003C0000003C000000780000007C00000FFFE00001D1F7E9E1E>I<0001FC04000F 030C003C009C0070007C00E0003C01C0003803800018078000180F0000181F0000181E00 00183E0000103C0000007C0000007C0000007C0000007C000000F8000000F8000000F800 7FFCF80003E0780001E0780001E0780003C0780003C03C0003C03C0003C01C0003C00E00 07C007000B800380118001E06080003F80001E217B9F24>I<07FFC7FFC0007C00F80000 3C007800003C007800007800F000007800F000007800F000007800F000007800F0000078 00F00000F001E00000F001E00000F001E00000F001E00000FFFFE00000F001E00001E003 C00001E003C00001E003C00001E003C00001E003C00001E003C00003C007800003C00780 0003C007800003C007800003C007800003C007800007800F000007C00F8000FFF8FFF800 221F7E9E22>I<07FFE0007C00003C00003C000078000078000078000078000078000078 0000F00000F00000F00000F00000F00000F00001E00001E00001E00001E00001E00001E0 0003C00003C00003C00003C00003C00003C00007800007C000FFFC00131F7F9E10>I<07 FFE07FE0007C001F00003C000C00003C0018000078001000007800400000780080000078 0100000078020000007804000000F008000000F010000000F060000000F0F0000000F1F0 000000F278000001E478000001E878000001F03C000001E03C000001E01E000001E01E00 0003C00F000003C00F000003C00F000003C007800003C007800003C003C000078003C000 07C007E000FFFC3FFC00231F7E9E23>75 D<07FFF000007E0000003C0000003C00000078 0000007800000078000000780000007800000078000000F0000000F0000000F0000000F0 000000F0000000F0000001E0000001E0000001E0000001E0000001E0008001E0010003C0 010003C0010003C0030003C0020003C0060003C0060007801E0007807C00FFFFFC00191F 7E9E1C>I<07FC0000FFC0007C0000F800003C00017800003C00017800004E0002F00000 4E0002F000004E0004F000004E0004F000004E0008F000004E0008F00000870011E00000 870011E00000870021E00000870021E00000870041E00000838041E00001038083C00001 038083C00001038103C00001038203C0000101C203C0000101C403C0000201C407800002 01C80780000201C80780000201D00780000200F00780000600E00780000600E00F00000F 00C00F8000FFE0C1FFF8002A1F7E9E2A>I<07FC01FFC0003E003E00003E001800003E00 1800004F001000004F001000004780100000478010000043C010000043C010000083C020 000081E020000081E020000080F020000080F020000080782000010078400001007C4000 01003C400001003C400001001E400001001E400002000F800002000F800002000F800002 000780000200078000060003800006000300000F00010000FFE0010000221F7E9E22>I< 0003F800001E0E000038070000E0038001C001C003C001E0078000E00F0000F00F0000F0 1E0000F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C0000F8F80001F0 F80001F0F80001F0F80001F0F80003E0780003E0780003C0780007C07C0007803C000F00 3C001E001E001C000E0038000700F00003C3C00000FE00001D217B9F23>I<07FFFF0000 7C03C0003C01E0003C00F0007800F0007800F8007800F8007800F8007800F8007800F000 F001F000F001E000F003C000F0078000F00F0000FFF80001E0000001E0000001E0000001 E0000001E0000001E0000003C0000003C0000003C0000003C0000003C0000003C0000007 80000007C00000FFFC00001D1F7E9E1F>I<0003F800001E0E000038070000F0038001E0 01C003C001E0078001E00F0000F00F0000F01F0000F01E0000F83E0000F83C0000F87C00 00F87C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0F80003E07800 03E0780003C0780007C0781E07803C210F003C409E001E409C000E80B8000740F00003C1 C04000FEC0400000C0400000C0800000E1800000FF800000FF000000FF0000007E000000 3C001D297B9F23>I<07FFFC00007C0700003C03C0003C01E0007801E0007801F0007801 F0007801F0007801F0007801E000F003E000F003C000F0078000F00F0000F03C0000FFF0 0001E0300001E0380001E01C0001E01C0001E01C0001E01E0003C03E0003C03E0003C03E 0003C03E0003C03E0003C03E0207803E0407C01F04FFFC0F18000003E01F207E9E21>I< 003F040060CC01803C03801C03001C0700180600080E00080E00080E00080E00000F0000 0F80000FE00007FE0003FF8001FFC0007FE00007E00001E00000E00000F00000F04000E0 4000E04000E04000E06000C0600180E00380F80300C60C0081F80016217D9F19>I<3FFF FFF03C0780F03007803060078030400F0010400F0010C00F0010800F0010800F0010800F 0010001E0000001E0000001E0000001E0000001E0000001E0000003C0000003C0000003C 0000003C0000003C0000003C000000780000007800000078000000780000007800000078 000000F0000001F800007FFFE0001C1F7A9E21>IIII<03FFC0FFC0007F007E00003E0038 00001E003000001E002000000F004000000F008000000F81000000078200000007C60000 0003C400000003E800000001F000000001F000000000F000000000F800000000F8000000 017C000000023C000000043C0000000C1E000000081E000000101F000000200F00000040 0F800000C0078000008007C000010003C000070003E0001F8007E000FFE01FFE00221F7F 9E22>II<03FFFFC003E0078003 80078007000F0006001E0004003C000C003C00080078000800F0000801F0000001E00000 03C0000007800000078000000F0000001E0000003C0000003C00000078000000F0010001 E0010001E0020003C00200078002000F8006000F0004001E000C003C001C007C003C0078 00F800FFFFF8001A1F7D9E1C>I<01FC07060E0F1C0F380E78007000F000F000F000F000 E000E000E000E000F0027004300818300FC010147C9314>99 D<0000700003F00000F000 00700000700000E00000E00000E00000E00000E00000E00001C000F9C00305C00E03C01C 03C03801C0780380700380F00380F00380F00380F00380E00700E00700E00700E00700E0 0700700F00301E00186F000F8FE014207C9F19>I<00F800070E000E07001C0700380380 780380700380F00380F00380FFFF80F00000E00000E00000E00000E00000F00100700200 3004001C180007E00011147D9314>I<01C003E003E003E001C000000000000000000000 000003801F800780038003800700070007000700070007000E000E000E000E000E000E00 1C001E00FF800B1F7F9E0C>105 D<038F801F90E007A0E003C0600380600780E00700E0 0700E00700E00700E00700E00E01C00E01C00E01C00E01C00E01C00E01C01C03801E03C0 FFCFF815147E9319>110 D<00FC000387000E01801C00C03800E03800E07000F0F000F0 F000F0F000F0F000F0E001E0E001E0E001C0E003C0F00380700700380E001C1C0007E000 14147D9317>I<0080010001000100030007000F001E00FFF80E000E000E000E001C001C 001C001C001C001C00380038203820382038203840384018800F000D1C7C9B12>116 D<1C0380FC1F803C07801C03801C0380380700380700380700380700380700380700700E 00700E00700E00700E00701E00701E00703C00305E001F9FC012147B9319>I<1FF0FF03 C07801C06001C04000E08000E180007300007600003C00003C00001C00002E00004E0000 87000107000203800603800C01C03E03E0FF07FC18147F9318>120 D E /Fn 64 123 df<000FF01FE0007FF07FE000FFF0FFE001FFF1FFE003F013F02003E0 03E00007C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007C007 C00007C007C000FFFF07FF00FFFF07FF00FFFF07FF00FFFF07FF0007C007C00007C007C0 0007C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007C007C000 07C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007 C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007C007C0002329 7FA823>11 D<00001F80003F9F80007F9F8000FF9F8001FF9F8003E09F8003C0000007C0 000007C0000007C0000007C0000007C0000007C0000007C0000007C00000FFFF8F80FFFF 8F80FFFF8F80FFFF8F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C0 0F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C0 0F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F80192A7FA920> I<001E003C007800F001F001E003C007C007800F800F001F001F001E003E003E003E007C 007C007C007C007C007800F800F800F800F800F800F800F800F800F800F800F800F800F8 00F800F80078007C007C007C007C007C003E003E003E001E001F001F000F000F80078007 C003C001E001F000F00078003C001E0F3D7CAC17>40 DI<7E7E7E7E7E7E1E3C3C387870F0070D7C8511>44 DII<00000600000E0000 0E00001C00001C00001C0000380000380000380000700000700000E00000E00000E00001 C00001C00001C0000380000380000380000700000700000700000E00000E00000E00001C 00001C00001C0000380000380000700000700000700000E00000E00000E00001C00001C0 0001C0000380000380000380000700000700000700000E00000E00000E00001C00001C00 00380000380000380000700000700000700000E00000E00000C00000173C7DAC1E>I<00 7F000001FFC00007FFF0000FFFF8000FC1F8001F007C003F007E003E003E003C001E007C 001F007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F80F8000F80F8 000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8 000F8078000F007C001F007C001F007C001F003E003E003E003E003F007E001F80FC000F C1F8000FFFF80007FFF00001FFC000007F000019297EA71E>I<00180000380000F80007 F800FFF800FFF800FFF800F8F80000F80000F80000F80000F80000F80000F80000F80000 F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000 F80000F80000F80000F80000F80000F80000F80000F80000F8007FFFF07FFFF07FFFF07F FFF014287CA71E>I<00FE0003FFC00FFFE01FFFF03F03F83C00FC78007E78007EF0003E F0003F70003F60001F20001F00001F00003F00003F00003E00003E00007C00007C0000F8 0001F00003E00007E00007C0000F80001E00003C0000780000F00001E00003C000078000 0F00001E00003C00007FFFFF7FFFFF7FFFFF7FFFFF18287EA71E>I<007F000003FFC000 07FFF0000FFFF8001F81F8003E00FC007C007E0038007E0030003E0010003E0000007E00 00007E0000007C0000007C000000FC000001F8000007F00000FFE00000FFC00000FFE000 00FFF0000001F80000007C0000007E0000003F0000003F0000001F8000001F8000001F80 00001F8000001F8040001F8060003F00F0003F00F8007F007C007E003F81FC001FFFF800 0FFFF00003FFC000007F000019297EA71E>I<0003F0000007F0000007F000000DF00000 0DF000001DF000003DF0000039F0000079F0000079F00000F1F00000F1F00001F1F00003 E1F00003E1F00007C1F00007C1F0000F81F0000F81F0001F01F0001F01F0003E01F0007C 01F0007C01F000F801F000FFFFFF80FFFFFF80FFFFFF80FFFFFF800001F0000001F00000 01F0000001F0000001F0000001F0000001F0000001F0000001F0000001F00019277EA61E >I<3FFFFC3FFFFC3FFFFC3FFFFC3E00003E00003E00003E00003E00003E00003E00003E 00003E00003E00003E3F003EFFC03FFFE03FFFF03FC1F83F80FC3F007C3E007E00003E00 003F00003F00003F00003F00003F00003F00003F20003F60007E70007EF800FC7C01FC3F 03F83FFFF00FFFE007FF8000FE0018287EA61E>I<000FF000003FFC0000FFFC0001FFFC 0003F80C0007E000000FC000000F8000001F0000001E0000003E0000003C0000007C0000 007C0000007C3FE000F8FFF000F9FFF800FBFFFC00FF807E00FF003E00FE003F00FC001F 00FC001F00FC000F80F8000F80F8000F80F8000F80F8000F8078000F807C000F807C000F 807C000F003E001F003E001F001F003E001F807C000FC1FC0007FFF80003FFF00001FFC0 00007F000019297EA71E>II<007F000001 FFC00007FFF0000FFFF8001FC1FC003F007E003E003E007E003F007C001F007C001F007C 001F007C001F007C001F003E003E003E003E001F007C000FC1F80007FFF00003FFE00003 FFE0000FFFF8001FC1FC003F007E003E003E007C001F007C001F00F8000F80F8000F80F8 000F80F8000F80F8000F80F8000F807C001F007C001F007E003F003F007E001FC1FC000F FFF80007FFF00003FFE000007F000019297EA71E>I<007F000001FFC00003FFE0000FFF F0000FC1F8001F007C003E007C007C003E007C001E007C001F00F8001F00F8001F00F800 0F00F8000F80F8000F80F8000F80F8000F80F8001F807C001F807C001F807E003F803E00 7F803F00FF801FFFEF800FFFCF8007FF8F8003FE1F0000001F0000001F0000001E000000 3E0000003E0000007C0000007C000000F8001801F0001E07E0003FFFC0001FFF80000FFE 000003F8000019297EA71E>I<03FC001FFF807FFFC0FFFFE0FE03F07801F06000F80000 F80000F80000F80000F80001F00003F00007E0000FC0001F80003F00003E00007C00007C 0000F80000F80000F80000F80000F80001F80001F80001F8000000000000000000000000 0000000000000000000001F80001F80001F80001F80001F80001F80015297DA81C>63 D<0001F000000003F800000003F800000007FC00000007FC000000077C0000000F3E0000 000F3E0000000F3E0000001E3F0000001E1F0000003E1F8000003C1F8000003C1F800000 7C0FC00000780FC00000780FC00000F807E00000F007E00000F007E00001F003F00001E0 03F00003E003F80003E001F80003C001F80007C000FC0007FFFFFC0007FFFFFC000FFFFF FE000FFFFFFE000F00007E001F00003F001F00003F003E00003F803E00001F803C00001F 807C00001FC07C00000FC07800000FC0F8000007E0F8000007E023297EA828>65 DI<0001FF00000FFFE0 003FFFF8007FFFF801FF03F803F8007007F0003007E000100FC000001F8000001F800000 3F0000003F0000007E0000007E0000007E0000007E000000FC000000FC000000FC000000 FC000000FC000000FC000000FC000000FC000000FC0000007E0000007E0000007E000000 7E0000003F0000003F0000001F8000001F8000000FC0000007E0000407F0000C03F8003C 01FF00FC007FFFFC003FFFF8000FFFE00001FF001E2B7CA926>IIII<0001FF00000FFFE0003FFFFC007FFFFE01FF01FE03FC003E07F0001C07E000 0C0FC000041F8000001F8000003F0000003F0000007E0000007E0000007E0000007E0000 00FC000000FC000000FC000000FC000000FC000000FC000000FC001FFEFC001FFEFC001F FE7E001FFE7E00003E7E00003E7E00003E3F00003E3F00003E1F80003E1F80003E0FC000 3E07E0003E07F0003E03FC003E01FF01FE007FFFFE003FFFFC000FFFE00001FF001F2B7C A928>III76 DII<0001FC0000000F FF8000003FFFE00000FFFFF80001FE03FC0003F800FE0007F0007F000FE0003F800FC000 1F801F80000FC01F000007C03F000007E03E000003E07E000003F07E000003F07E000003 F07C000001F0FC000001F8FC000001F8FC000001F8FC000001F8FC000001F8FC000001F8 FC000001F8FC000001F8FC000001F8FE000003F87E000003F07E000003F07E000003F03F 000007E03F000007E01F80000FC01F80000FC00FC0001F800FE0003F8007F0007F0003F8 00FE0001FF07FC0000FFFFF800003FFFE000000FFF80000001FC0000252B7DA92C>II82 D<007FC00001FFF80007FFFE000FFFFF001FC0 7F003F001F007E000E007C00060078000200F8000000F8000000F8000000F8000000FC00 0000FC0000007E0000007F8000003FF800001FFF80000FFFE00007FFF80003FFFC0000FF FE00000FFF000000FF0000003F8000001F8000000FC000000FC0000007C0000007C00000 07C0000007C0400007C060000F8070000F80F8001F00FE003F00FF80FE007FFFFC001FFF F80007FFE00000FF80001A2B7DA921>IIIII<00FE0007FF801FFFC03FFFE03E03F03801F03001F82000F80000 F80000F80000F80000F8007FF807FFF81FFFF83FE0F87F00F8FC00F8F800F8F800F8F800 F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8151B7E9A1D>97 DI<007F8001FFE007FFF80FFFFC1FC07C1F001C3E00087C00 007C00007C0000F80000F80000F80000F80000F80000F80000F800007C00007C00007E00 003E00041F001C1FC07C0FFFFC07FFFC01FFF0007F80161B7E9A1B>I<00003E00003E00 003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00 FC3E03FF3E07FFFE0FFFFE1FC1FE3F007E3E003E7C003E7C003EFC003EF8003EF8003EF8 003EF8003EF8003EF8003EF8003EFC003E7C003E7C003E3E007E3F00FE1FC1FE0FFFFE07 FFBE03FF3E00FC3E17297EA81F>I<007E0003FF8007FFC00FFFE01F83F03F00F03E0078 7C00787C003878003CFFFFFCFFFFFCFFFFFCFFFFFCF80000F80000F800007800007C0000 7C00003E00003F000C1FC07C0FFFFC07FFFC01FFF0007F80161B7E9A1B>I<001FC0007F C000FFC001FFC003E04003C00007C00007C00007C00007C00007C00007C00007C00007C0 00FFFE00FFFE00FFFE00FFFE0007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00012297FA812>I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0 003E03E0003E03E0007C01F0007C01F0007C01F0007C01F0007C01F0007C01F0003E03E0 003E03E0001F07C0001FFFC0003FFF80003BFE000038F8000078000000780000003C0000 003FFFC0003FFFF8001FFFFC001FFFFE003FFFFF007C007F00F8001F80F8000F80F8000F 80F8000F80FC001F807E003F003F80FE003FFFFE000FFFF80007FFF00000FF80001A287E 9A1E>III107 DI II<007F000001FFC00007FFF0000FFFF8001FC1FC 003F007E003E003E007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F80F8000F807C001F007C001F007E003F003E003E003F007E001FC1FC 000FFFF80007FFF00001FFC000007F0000191B7E9A1E>II<00FC3E03FF3E07FFBE0FFFFE1FC1FE3F80FE3E007E7E003E7C003E7C003EF800 3EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C003E7E003E3E007E3F007E1FC1 FE1FFFFE0FFFBE03FF3E01F83E00003E00003E00003E00003E00003E00003E00003E0000 3E00003E00003E00003E00003E17277E9A1F>II<03FC001FFF803FFFC07FFFC07C07C0F80180F80000F80000F80000 FC00007F80007FF8003FFE001FFF0007FF8000FFC0000FE00007E00003E04003E06003E0 F007E0FC0FC0FFFFC07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C00007C0 0007C00007C00007C000FFFFC0FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C04007E1C003FFE003FFE001FF8000FC0013227FA116>IIII<7C000FC03E001F803F001F001F803E000F807C0007C0FC0003 E0F80001F1F00001FBE00000FFC000007FC000003F8000001F0000001F0000003F800000 7FC00000FBC00000F3E00001F1F00003E0F80007C07C000F807C000F803E001F001F003E 000F807E000FC0FC0007E01B1B809A1C>I II E /Fo 18 121 df<00001E000000003E000000 00FE00000003FE0000003FFE0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00007FFFFF FFC07FFFFFFFC07FFFFFFFC07FFFFFFFC0223879B731>49 D<0007FE0000007FFFE00001 FFFFF80003FFFFFE0007F01FFF000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC0 007FF0FFE0007FF8FFF0003FF8FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE000 3FFC3FC0003FFC1F80003FFC0000003FFC0000003FF80000007FF80000007FF00000007F F0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00000007F800 00000FF00000001FC00000003F800000007F00000000FC00000001F800000001F0003C00 03E0003C0007C0003C000F000078001E000078003C00007800780000F800F00000F801FF FFFFF803FFFFFFF007FFFFFFF00FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFF FFF0FFFFFFFFE0FFFFFFFFE0FFFFFFFFE026387BB731>I<0003FF8000001FFFF000007F FFFE0000FE03FF0001F000FF8003C000FFC00780007FE00FF0007FF00FF8007FF01FFC00 7FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF80FFC007FF007F8007F F003F0007FF0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00 00001FF000000FFFC000000FFF8000000FFFF800000003FE00000000FF800000007FE000 00003FF00000003FF80000003FFC0000001FFC0000001FFE0000001FFE0200001FFF1FC0 001FFF3FE0001FFF7FF0001FFF7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF800 1FFEFFF0001FFE7FF0003FFC7FE0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FF C003FFFFFF0001FFFFFE00003FFFF0000007FF800028397CB731>I<00000007C0000000 000FC0000000000FC0000000001FC0000000003FC0000000007FC000000000FFC0000000 00FFC000000001FFC000000003FFC000000007FFC00000000FFFC00000000FFFC0000000 1EFFC00000003CFFC00000007CFFC0000000F8FFC0000000F0FFC0000001E0FFC0000003 C0FFC0000007C0FFC000000F80FFC000000F00FFC000001E00FFC000003C00FFC000007C 00FFC00000F800FFC00000F000FFC00001E000FFC00003C000FFC00007C000FFC0000F80 00FFC0000F0000FFC0001E0000FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFF FFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC0000007FFFFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A37 7DB631>I<04000000C00F800007C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000F FFFFFC000FFFFFF8000FFFFFF0000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F80 0000000F800000000F800000000F800000000F800000000F800000000F800000000F8000 00000F81FF00000F8FFFE0000FBFFFF8000FFE03FE000FF000FF000FC000FF800F80007F C00F00007FE00700007FF00000003FF00000003FF80000003FF80000003FF80000003FFC 0000003FFC0600003FFC1F80003FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFF F0003FFCFFF0003FF8FFE0003FF8FFE0003FF87FC0007FF07F00007FF03C00007FE03E00 00FFC01F0000FF800FC003FF0007F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC 000026397BB731>I<00000FF80000007FFF000003FFFF80000FFC07C0001FE001E0003F C001F0007F0007F000FF000FF001FE001FF803FC003FF807FC003FF80FFC003FF80FF800 3FF81FF8001FF01FF8000FE03FF80007C03FF00000003FF00000007FF00000007FF00000 007FF00000007FF07FF000FFF0FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0 FFFC001FF0FFFC000FF8FFF8000FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFF F0000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0 000FFF3FF0000FFF3FF0000FFE3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF800 1FF807FC001FF003FC001FF001FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF8 000000FFC00028397CB731>I<1E00000000001F00000000001FF0000000001FFFFFFFFF C01FFFFFFFFFC01FFFFFFFFFC03FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE 003FFFFFFFFC003FFFFFFFF8003FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC0 007800001F80007800001F00007800003E0000F000007E0000F00000FC0000F00001F800 00000003F00000000003E00000000007E0000000000FC0000000000F80000000001F8000 0000003F80000000003F00000000007F00000000007F0000000000FF0000000000FE0000 000001FE0000000001FE0000000003FE0000000003FE0000000003FE0000000007FC0000 000007FC0000000007FC000000000FFC000000000FFC000000000FFC000000000FFC0000 00000FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000 00001FFC000000001FFC000000001FFC000000001FFC000000000FF80000000007F00000 000003E00000002A3B7BB931>I<0001FF8000000FFFF800003FFFFE00007F00FF0000F8 003F8001F0000FC003E0000FE007C00007F007C00007F00FC00003F80FC00003F81FC000 03F81FC00003F81FE00003F81FF00003F81FF80003F81FFC0007F01FFF0007F01FFFC007 E00FFFF00FE00FFFFC1FC007FFFE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE00 007FFFFF00003FFFFF80003FFFFFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1F C007FFFC3F8001FFFE3F8000FFFE7F00003FFF7F00000FFFFE000003FFFE000001FFFE00 0000FFFE000000FFFE0000007FFE0000007FFE0000007EFE0000007E7F0000007E7F0000 00FC3F800000FC3FC00001F81FE00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE 00003FFFF8000003FF800028397CB731>I<00000001F80000000000000001F800000000 00000003FC0000000000000003FC0000000000000007FE0000000000000007FE00000000 00000007FE000000000000000FFF000000000000000FFF000000000000001FFF80000000 0000001FFF800000000000001FFF800000000000003FFFC00000000000003FFFC0000000 0000007FFFE00000000000007DFFE00000000000007DFFE0000000000000FDFFF0000000 000000F8FFF0000000000001F8FFF8000000000001F07FF8000000000001F07FF8000000 000003F07FFC000000000003E03FFC000000000007E03FFE000000000007C01FFE000000 00000FC01FFF00000000000F801FFF00000000000F800FFF00000000001F800FFF800000 00001F0007FF80000000003F0007FFC0000000003E0003FFC0000000003E0003FFC00000 00007E0003FFE0000000007C0001FFE000000000FC0001FFF000000000F80000FFF00000 0000F80000FFF000000001FFFFFFFFF800000001FFFFFFFFF800000003FFFFFFFFFC0000 0003FFFFFFFFFC00000003E000003FFC00000007E000003FFE00000007C000001FFE0000 000FC000001FFF0000000F8000000FFF0000001F8000000FFF8000001F0000000FFF8000 001F00000007FF8000003F00000007FFC000003E00000003FFC000007E00000003FFE000 00FF00000001FFE000FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0 FFFFFC0001FFFFFFF0443B7DBA4B>65 D<0000001FFF000030000001FFFFE000F000000F FFFFFC01F000007FFFFFFE03F00001FFFE007F87F00003FFE0000FCFF0000FFF000003FF F0001FFC000001FFF0003FF80000007FF0007FF00000003FF000FFC00000003FF001FFC0 0000001FF003FF800000000FF007FF000000000FF00FFF0000000007F00FFE0000000007 F01FFE0000000003F01FFE0000000003F03FFC0000000003F03FFC0000000001F03FFC00 00000001F07FFC0000000001F07FF80000000001F07FF80000000000007FF80000000000 00FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF800 0000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF80000000000 00FFF8000000000000FFF80000000000007FF80000000000007FF80000000000007FF800 00000000007FFC0000000000F03FFC0000000000F03FFC0000000000F03FFC0000000000 F01FFE0000000000F01FFE0000000001E00FFE0000000001E00FFF0000000001E007FF00 00000003C003FF8000000003C001FFC0000000078000FFE00000000F00007FF00000001F 00003FF80000003E00001FFC0000007C00000FFF000001F8000003FFE00007F0000001FF FE003FC00000007FFFFFFF000000000FFFFFFC0000000001FFFFF000000000001FFF0000 003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007FE00000FF801 FF00001FFC00FF80001FFC007FC0001FFC007FE0001FFC003FE0000FF8003FF0000FF800 3FF00007F0003FF00001C0003FF0000000003FF0000000003FF0000000003FF0000000FF FFF000000FFFFFF000007FF83FF00001FF803FF00007FE003FF0000FF8003FF0001FF000 3FF0003FE0003FF0007FE0003FF0007FE0003FF000FFC0003FF000FFC0003FF000FFC000 3FF000FFC0003FF000FFC0007FF0007FE0007FF0007FE000DFF0003FF0039FF8001FFC0F 0FFFF007FFFE0FFFF001FFFC07FFF0003FE000FFF02C267DA530>97 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007FC001FC00FF8000FE0 0FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE00003FC7FE00003FC7F E00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE0000000FFE0000000FFE0 000000FFE00000007FE00000007FE00000007FE00000003FE00000003FF000003C1FF000 003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E000FFC03FC0003FFFFF 000007FFFC000000FFE00026267DA52D>101 D<00FF00000000FFFF00000000FFFF0000 0000FFFF00000000FFFF0000000007FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF007FC00003FF01FFF80003FF07FFFC0003FF0F03FE0003FF1C01 FF0003FF3001FF8003FF6000FF8003FFE000FFC003FFC000FFC003FF8000FFC003FF8000 FFC003FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3F FFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF303C7CBB37>104 D<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FFFE1C01FF0007FE3001FF 8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003FF8000FFC003FF8000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFF FFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<00FF01FF8000FFFF0FFFF000FFFF 3FFFFC00FFFFFE03FF00FFFFF000FF8003FFC0007FC003FF80003FE003FF00003FF003FF 00001FF803FF00001FFC03FF00000FFC03FF00000FFE03FF00000FFE03FF000007FE03FF 000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF 000007FF03FF000007FF03FF000007FF03FF000007FE03FF000007FE03FF00000FFE03FF 00000FFC03FF00000FFC03FF00001FF803FF00001FF803FF00003FF003FF80003FE003FF C0007FC003FFF001FF8003FFFC07FF0003FF3FFFFC0003FF0FFFF00003FF01FF000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 00000000FFFFFC000000FFFFFC000000FFFFFC000000FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80FFFE707FC007FE60FFE003FEE0FFE0 03FEC0FFE003FFC0FFE003FF807FC003FF807FC003FF803F8003FF800E0003FF00000003 FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF000000FFFFFE0000FFFFFE00 00FFFFFE0000FFFFFE000023267DA529>114 D<00078000000780000007800000078000 00078000000F8000000F8000000F8000000F8000001F8000001F8000003F8000003F8000 007F800000FF800001FF800007FF80001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C00FF8078 00FFC078007FC070003FE0E0001FFFC00007FF800001FF001E377EB626>116 D120 D E /Fp 31 123 df<007E01C007000E001C003C003800780078007FF0F000F000F0007000700070003000 18000C1807E00F147E9312>15 D<70F8F8F87005057C840D>58 D<70F8FCFC7404040408 0810102040060E7C840D>I<000001C00000078000001E00000078000001E00000078000 000E00000038000000F0000003C000000F0000003C000000F0000000F00000003C000000 0F00000003C0000000F0000000380000000E0000000780000001E0000000780000001E00 00000780000001C01A1A7C9723>I62 D<000002000000060000000E0000000E0000 001E0000001F0000002F0000002F0000004F0000008F0000008F0000010F0000010F0000 020F0000040F0000040F0000080F8000080780001007800020078000200780007FFF8000 40078000800780018007800100078002000780020007C0040003C00C0003C01E0007C0FF 807FFC1E207E9F22>65 D<0FFFFFFC1E03C0381803C0181003C0082003C0082007800860 0780084007800840078008800F0010000F0000000F0000000F0000001E0000001E000000 1E0000001E0000003C0000003C0000003C0000003C000000780000007800000078000000 78000000F0000000F0000000F0000000F0000001F000007FFFC0001E1F7F9E1B>84 D<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E10F01C20F01C20703C20705C40308C400F078014147E9318>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207E9F14>I< 007C01C207010E0F1E0F1C0E3C04780078007800F000F000F000F000F000700170023004 18380FC010147E9314>I<0000780003F80000700000700000700000700000E00000E000 00E00000E00001C00001C000F1C00389C00707800E03801C03803C038038070078070078 0700780700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F078015 207E9F18>I<007C01C207010E011C013C013802780C7BF07C00F000F000F000F0007000 700170023004183807C010147E9315>I<00007C0000CE00019E00039E00030C00070000 0700000700000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C0000 1C00001C0000380000380000380000380000380000700000700000700000700000700000 E00000E00000E00000E00000C00001C000318000798000F300006200003C000017297E9F 16>I<001E3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E01C0 1E01C01E01C01E01C01E03801E03800E07800E0B8006170001E700000700000700000E00 000E00300E00781C00F038006070003FC000151D809316>I<00E001E001E000C0000000 00000000000000000000000E00130023804380438043808700070007000E000E001C001C 001C20384038403840388019000E000B1F7E9E10>105 D<0000C00001E00001E00001C0 000000000000000000000000000000000000000000001E00006300004380008380010380 010380020700000700000700000700000E00000E00000E00000E00001C00001C00001C00 001C0000380000380000380000380000700000700030700078E000F1C0006380003E0000 1328819E13>I<01E0000FE00001C00001C00001C00001C0000380000380000380000380 000700000700000701E00706100E08700E10F00E20F00E40601C80001D00001E00001FC0 00387000383800383800381C20703840703840703840701880E01880600F0014207E9F18 >I<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C00 1C0038003800380038007000700070007100E200E200E200E200640038000A207E9F0E> I<1E07C07C00231861860023A032030043C0340300438038038043803803808700700700 0700700700070070070007007007000E00E00E000E00E00E000E00E00E000E00E01C101C 01C01C201C01C038201C01C038401C01C0184038038018801801800F0024147E9328>I< 1E07802318C023A06043C0704380704380708700E00700E00700E00700E00E01C00E01C0 0E01C00E03821C03841C07041C07081C03083803101801E017147E931B>I<007C0001C3 000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F007 80F00700700F00700E0030180018700007C00013147E9316>I<03C1E004621804741C08 781C08701E08701E10E01E00E01E00E01E00E01E01C03C01C03C01C03C01C03803807803 80700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C 0000FFC000171D819317>I<1E1E0023210023C38043C780438780438300870000070000 0700000700000E00000E00000E00000E00001C00001C00001C00001C0000380000180000 11147E9315>114 D<007C018203010603060706060E00078007F803FC01FE001F000770 07F006F006E004400820301FC010147E9315>I<00C000E001C001C001C001C003800380 FFF8038007000700070007000E000E000E000E001C001C001C001C103820382038203840 18800F000D1C7F9B10>I<0F00601180702180E021C0E041C0E04380E08381C00701C007 01C00701C00E03800E03800E03800E03840E07080C07080C07080E0F1006131003E1E016 147E931A>I<0F01801183C02183E021C1E041C0E0438060838040070040070040070040 0E00800E00800E00800E01000E01000C02000E04000E040006180001E00013147E9316> I<0F006060118070F02180E0F821C0E07841C0E0384380E0188381C0100701C0100701C0 100701C0100E0380200E0380200E0380200E0380400E0380400E0380800E078080060781 000709860001F078001D147E9321>I<03C1C00C62201034701038F02038F02038604070 0000700000700000700000E00000E00000E00000E02061C040F1C040F1C080E2C0804463 00383C0014147E931A>I<0F00601180702180E021C0E041C0E04380E08381C00701C007 01C00701C00E03800E03800E03800E03800E07000C07000C07000E0F00061E0003EE0000 0E00000E00001C0078180078380070700060600021C0001F0000141D7E9316>I<01E020 03F04007F8C00C1F8008010000020000040000080000100000600000C000010000020000 0400800801001003003F060061FC0040F80080700013147E9315>I E /Fq 56 122 df<000FF07F00007FF9FF8000F83FC7C001E07F8FC003E07F0FC007C07F 0FC007C03F078007C01F000007C01F000007C01F000007C01F000007C01F0000FFFFFFF8 00FFFFFFF80007C01F000007C01F000007C01F000007C01F000007C01F000007C01F0000 07C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007 C01F000007C01F000007C01F00003FF8FFF0003FF8FFF0002220809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C0007C07C0007C07C0007C0380007C000 0007C0000007C0000007C1FE00FFFFFE00FFFFFE0007C03E0007C03E0007C03E0007C03E 0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E 0007C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>I<000FF03FFC00007F FDFFFC0000F83FE0FC0001E03F81FC0003E07F81FC0007C07F00FC0007C03F00FC0007C0 1F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C00FFFFFFFFFC00FFFF FFFFFC0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C0 1F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C0 1F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C003FF8FFE3FF803FF8 FFE3FF802920809F2C>15 D<387CFEFFFF7F3B030306060E0C18702008107C860F>44 DI<387CFEFEFE7C3807077C860F>I<01FC0007FF001F 07C01E03C03E03E07C01F07C01F07C01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC 01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F87C01F07C01F07C01F03E03E01E03C01F 8FC007FF0001FC00151D7E9C1A>48 D<00E00001E0000FE000FFE000F3E00003E00003E0 0003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E0 0003E00003E00003E00003E00003E00003E00003E00003E000FFFF80FFFF80111D7C9C1A >I<07F0001FFE00383F007C1F80FE0FC0FE0FC0FE0FE0FE07E07C07E03807E0000FE000 0FC0000FC0001F80001F00003E0000780000F00000E00001C0000380600700600E00601C 00E01FFFC03FFFC07FFFC0FFFFC0FFFFC0131D7D9C1A>I<01FC0007FF000E0F801E0FC0 3F07E03F07E03F07E03F07E01E0FC0000FC0000F80001F0001FC0001FC00000F800007C0 0003E00003F00003F83803F87C03F8FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF80 03FC00151D7E9C1A>I<0001C00003C00007C00007C0000FC0001FC0003BC00073C00063 C000C3C00183C00383C00703C00E03C00C03C01803C03803C07003C0E003C0FFFFFEFFFF FE0007C00007C00007C00007C00007C00007C000FFFE00FFFE171D7F9C1A>I<3803803F FF803FFF003FFE003FFC003FF0003F800030000030000030000030000033F80037FE003C 1F00380F801007C00007C00007E00007E07807E0FC07E0FC07E0FC07E0FC07C0780FC060 0F80381F001FFC0007F000131D7D9C1A>I<003F0001FFC007E0E00F81E01F03F01E03F0 3E03F07C03F07C01E07C0000FC1000FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8 FC01F8FC01F87C01F87C01F87C01F83C01F03E01F01E03E00F07C007FF8001FE00151D7E 9C1A>I<6000007FFFF87FFFF87FFFF07FFFE07FFFC0E00180C00300C00300C00600000C 0000180000380000380000780000700000F00000F00001F00001F00001F00001F00003F0 0003F00003F00003F00003F00003F00001E00000C000151E7D9D1A>I<01FC0007FF000F 07801E03C01C01E03C01E03C01E03E01E03F01E03FC3C01FE3801FFF000FFE0007FF8007 FFC01FFFE03C3FF0780FF07803F8F001F8F000F8F00078F00078F000707800707C00E03E 03C00FFF8003FC00151D7E9C1A>I<01FC000FFF001F07803E03C07C03E07C01E0FC01F0 FC01F0FC01F0FC01F8FC01F8FC01F8FC01F87C03F87C03F83E05F81FFDF807F9F80041F8 0001F03C01F07E01F07E03E07E03E07E07C03C0780381F001FFC0007F000151D7E9C1A> I<7FFFFFFF80FFFFFFFFC0FFFFFFFFC00000000000000000000000000000000000000000 00000000000000000000000000000000000000000000000000FFFFFFFFC0FFFFFFFFC07F FFFFFF80220F7D9229>61 D<0000E000000000E000000001F000000001F000000001F000 000003F800000003F800000006FC00000006FC0000000EFE0000000C7E0000000C7E0000 00183F000000183F000000303F800000301F800000701FC00000600FC00000600FC00000 C007E00000FFFFE00001FFFFF000018003F000018003F000030001F800030001F8000600 01FC00060000FC000E0000FE00FFE00FFFE0FFE00FFFE0231F7E9E28>65 DI<0007FC02003FFF0E00FE03DE 03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E0000067E000006 FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E0000007E000006 7F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000FE01C0003FFF80 0007FC001F1F7D9E26>I69 DI<0007FC0200003FFF0E0000FE 03DE0003F000FE0007E0003E000FC0001E001F80001E003F00000E003F00000E007F0000 06007E000006007E00000600FE00000000FE00000000FE00000000FE00000000FE000000 00FE003FFFE0FE003FFFE07E00007E007E00007E007F00007E003F00007E003F00007E00 1F80007E000FC0007E0007E0007E0003F000FE0000FE01FE00003FFF8E000007FC060023 1F7D9E29>I73 D76 DI 80 D<001FF80000FFFF0001F81F8007E007E00FC003F01F8001F81F8001F83F0000FC7F 0000FE7F0000FE7E00007EFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE 00007FFE00007FFE00007F7E00007E7E00007E7F0000FE3F0000FC3F87C1FC1F8FE1F80F D833F007F81FE001F81F8000FFFF00001FFE0300000E0300000F0700000FFF000007FF00 0007FE000007FE000003FC000001F8000000F020287D9E27>I<03FC080FFF381E03F838 00F8700078700038F00038F00018F00018F80000FC00007FC0007FFE003FFF801FFFE00F FFF007FFF000FFF80007F80000FC00007C00003CC0003CC0003CC0003CE00038E00078F8 0070FE01E0E7FFC081FF00161F7D9E1D>83 D<7FFFFFFC7FFFFFFC7C07E07C7007E01C60 07E00C6007E00CE007E00EC007E006C007E006C007E006C007E0060007E0000007E00000 07E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00000 07E0000007E0000007E0000007E0000007E0000007E00003FFFFC003FFFFC01F1E7E9D24 >I86 D<07FC001FFF003F0F803F07C03F03E0 3F03E00C03E00003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F803E0FC05E0 7E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000 FC0000FC0000FC0000FC0000FC00007C00007E00003E00603F00C01F81C007FF0001FC00 13147E9317>I<0007F80007F80000F80000F80000F80000F80000F80000F80000F80000 F80000F80000F801F8F80FFEF81F83F83E01F87E00F87C00F87C00F8FC00F8FC00F8FC00 F8FC00F8FC00F8FC00F87C00F87C00F87E00F83E01F81F07F80FFEFF03F8FF18207E9F1D >I<01FE0007FF800F83C01E01E03E00F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC 0000FC00007C00007C00003E00181E00180F807007FFE000FF8015147F9318>I<001F80 00FFC001F3E003E7E003C7E007C7E007C3C007C00007C00007C00007C00007C000FFFC00 FFFC0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000 07C00007C00007C00007C00007C0003FFC003FFC0013207F9F10>I<01FC3C07FFFE0F07 9E1E03DE3E03E03E03E03E03E03E03E03E03E01E03C00F07800FFF0009FC001800001800 001C00001FFF800FFFF007FFF81FFFFC3C007C70003EF0001EF0001EF0001E78003C7800 3C3F01F80FFFE001FF00171E7F931A>II<1C003E003F007F003F003E001C0000000000000000000000 0000FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F00 1F00FFE0FFE00B217EA00E>I<0038007C00FE00FE00FE007C0038000000000000000000 00000001FE01FE003E003E003E003E003E003E003E003E003E003E003E003E003E003E00 3E003E003E003E003E003E303E783EFC3CFC7C78783FF01FC00F2A83A010>IIIII<01FF0007FFC01F83F03E00F83E00F87C 007C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C007C7C007C3E00F83E00F81F 83F007FFC001FF0017147F931A>I I<01F81807FE381F87783F01F83E01F87E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00 F8FC00F87C00F87E00F87E00F83F01F81F87F80FFEF803F8F80000F80000F80000F80000 F80000F80000F80000F80007FF0007FF181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07FF8 3FFC1FFE03FE001FC007C007E007F006F81EFFFCC7F010147E9315>I<01800180018003 800380038007800F803F80FFFCFFFC0F800F800F800F800F800F800F800F800F800F800F 860F860F860F860F8607CC03F801F00F1D7F9C14>IIIIII E /Fr 37 122 df<00000007FF800000000001FFFFF0000000000FFFFFFC000000003FFFFFFE00 000000FFFC00FF00000001FFC0003F80000007FF00007FC000000FFE0001FFC000001FFC 0001FFE000001FF80003FFE000003FF00003FFE000003FF00003FFE000007FE00003FFE0 00007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00001FFC000007FE0 0000FF8000007FE000003E0000007FE00000000000007FE00000000000007FE000000000 00007FE00000000000007FE00000000000007FE00000000000007FE0003FFFF000FFFFFF FFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF0 00007FF00000FFF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFF E03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC007FFE00FFFE00FFFF00FFFF 00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E780000780000780000F00000F 00000F00000F00001E00001E00003C00003C0000780000F80000F00001F00003E00007C0 000F80001F00001E00000C00001124788F21>44 DI<00000000 03E000000000000000000007F000000000000000000007F00000000000000000000FF800 00000000000000000FF80000000000000000000FF80000000000000000001FFC00000000 00000000001FFC0000000000000000003FFE0000000000000000003FFE00000000000000 00007FFF0000000000000000007FFF0000000000000000007FFF000000000000000000FF FF800000000000000000FFFF800000000000000001FFFFC00000000000000001FFFFC000 00000000000001FFFFC00000000000000003FFFFE00000000000000003EFFFE000000000 00000007EFFFF00000000000000007CFFFF00000000000000007C7FFF000000000000000 0FC7FFF8000000000000000F83FFF8000000000000001F83FFFC000000000000001F03FF FC000000000000001F01FFFC000000000000003F01FFFE000000000000003E00FFFE0000 00000000007E00FFFF000000000000007C007FFF00000000000000FC007FFF8000000000 0000F8007FFF80000000000000F8003FFF80000000000001F8003FFFC0000000000001F0 001FFFC0000000000003F0001FFFE0000000000003E0001FFFE0000000000003E0000FFF E0000000000007E0000FFFF0000000000007C00007FFF000000000000FC00007FFF80000 0000000F800007FFF800000000000F800003FFF800000000001F800003FFFC0000000000 1F000001FFFC00000000003FFFFFFFFFFE00000000003FFFFFFFFFFE00000000007FFFFF FFFFFF00000000007FFFFFFFFFFF00000000007FFFFFFFFFFF0000000000FC0000007FFF 8000000000F80000003FFF8000000001F80000003FFFC000000001F00000003FFFC00000 0001F00000001FFFC000000003F00000001FFFE000000003E00000000FFFE000000007E0 0000000FFFF000000007C00000000FFFF000000007C000000007FFF00000000FC0000000 07FFF80000000F8000000003FFF80000001F8000000003FFFC0000001F0000000003FFFC 0000007FC000000001FFFC0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FF FFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF8051487CC7 5A>65 DI<000000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E000003FFFFFFFE 00FE00000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001FFFC000003 FFFE0007FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003FFF00000000 1FFE007FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FFF000000000 03FE03FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFFC000000000 00FE0FFFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF8000000000 007E3FFF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF0000000000 003E7FFF000000000000007FFF00000000000000FFFF00000000000000FFFF0000000000 0000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF0000000000 0000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF0000000000 0000FFFF00000000000000FFFF00000000000000FFFF000000000000007FFF0000000000 00007FFF000000000000007FFF000000000000007FFF8000000000003E3FFF8000000000 003E3FFF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFFC000000000 003E0FFFC000000000007C0FFFC000000000007C07FFE000000000007C07FFE000000000 00F803FFF00000000000F803FFF00000000001F801FFF80000000001F000FFFC00000000 03E0007FFE0000000007E0003FFF000000000FC0001FFF800000001F80000FFFC0000000 3F000007FFF0000000FE000001FFFC000001FC000000FFFF80000FF80000003FFFF8007F F00000000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000000FFFFFE0 0000000000003FFE000000474979C756>I69 DI<000000 003FFE00000E000000000FFFFFC0001E000000007FFFFFF8003E00000003FFFFFFFE00FE 0000000FFFFFFFFF81FE0000003FFFF800FFC3FE000000FFFF80000FF7FE000001FFFC00 0003FFFE000007FFF0000001FFFE00000FFFC00000007FFE00001FFF800000003FFE0000 3FFF000000001FFE00007FFE000000000FFE0000FFFC0000000007FE0001FFF800000000 07FE0003FFF00000000003FE0003FFF00000000001FE0007FFE00000000001FE0007FFE0 0000000000FE000FFFC00000000000FE000FFFC000000000007E001FFFC000000000007E 001FFF8000000000007E003FFF8000000000007E003FFF8000000000003E003FFF800000 0000003E007FFF8000000000003E007FFF0000000000003E007FFF00000000000000007F FF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000 000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF00 00000000000000FFFF0000000000000000FFFF0000000000000000FFFF00000000000000 00FFFF0000000000000000FFFF0000000000000000FFFF0000007FFFFFFFFE7FFF000000 7FFFFFFFFE7FFF0000007FFFFFFFFE7FFF8000007FFFFFFFFE7FFF8000007FFFFFFFFE3F FF8000000000FFFE003FFF8000000000FFFE003FFF8000000000FFFE001FFF8000000000 FFFE001FFFC000000000FFFE000FFFC000000000FFFE000FFFC000000000FFFE0007FFE0 00000000FFFE0007FFE000000000FFFE0003FFF000000000FFFE0003FFF800000000FFFE 0001FFF800000000FFFE0000FFFC00000000FFFE00007FFE00000000FFFE00003FFF0000 0000FFFE00001FFF80000000FFFE00000FFFE0000001FFFE000007FFF0000003FFFE0000 01FFFC000007FFFE000000FFFF80001FFFFE0000003FFFFC00FFC7FE0000000FFFFFFFFF 83FE00000003FFFFFFFE00FE000000007FFFFFF8003E000000000FFFFFE0000E00000000 003FFE000000004F4979C75D>I73 D76 DI80 D<3FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFF FFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFF0003FFF8003FFF007FF8 0003FFF80007FF807FE00003FFF80001FF807FC00003FFF80000FF807F800003FFF80000 7F807F000003FFF800003F807F000003FFF800003F807E000003FFF800001F807E000003 FFF800001F807E000003FFF800000F807C000003FFF800000F807C000003FFF800000F80 7C000003FFF800000F807C000003FFF800000F80FC000003FFF800000FC0F8000003FFF8 000007C0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007C0F800 0003FFF8000007C000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF8000000000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFF FFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800004A467CC553>84 D<0007FFFC000000007FFFFFC0000001FFFFFFF8000003FFFFFFFE000007FE001FFF0000 07FF0003FFC0000FFF8001FFE0000FFF8000FFF0000FFF80007FF0000FFF80007FF8000F FF80007FF80007FF00003FFC0007FF00003FFC0003FE00003FFC0000F800003FFC000000 00003FFC00000000003FFC00000000003FFC00000000003FFC00000007FFFFFC000000FF FFFFFC000007FFFFFFFC00003FFFE03FFC0000FFFE003FFC0003FFF0003FFC0007FFC000 3FFC000FFF00003FFC001FFE00003FFC003FFC00003FFC007FF800003FFC007FF800003F FC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC 00FFF000007FFC007FF80000FFFC007FF80001EFFC003FFC0003EFFC003FFF0007CFFF00 0FFFC03F8FFFF807FFFFFF07FFFC01FFFFFC03FFFC007FFFF001FFFC0003FF80007FF836 2E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000 000000FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC00FFC00000001FFC07FFFC0000001FFC3FFFFF0000001FFCFFFFFFC000001FFDFF00F FF000001FFFF8003FF800001FFFE0001FFC00001FFF800007FE00001FFF000007FF00001 FFE000003FF80001FFE000001FFC0001FFE000001FFC0001FFE000001FFE0001FFE00000 1FFE0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE00000 1FFE0001FFE000001FFE0001FFE000001FFC0001FFE000001FFC0001FFE000003FF80001 FFF000003FF80001FFF800007FF00001FFFC0000FFE00001FFFE0001FFC00001FFBF0007 FF800001FF1FE01FFE000001FE0FFFFFFC000001FC03FFFFF0000001F800FFFF80000001 F0001FF800000039487CC742>I<00001FFFC0000000FFFFF8000007FFFFFE00001FFFFF FF80007FFC00FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF0003FFE00FFE0003 FFE00FFE0003FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC00003E007FF80000 00007FF8000000007FF800000000FFF800000000FFF800000000FFF800000000FFF80000 0000FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF80000 00007FF8000000007FF8000000007FFC000000003FFC000000003FFC000000001FFC0000 00F81FFE000000F80FFE000000F80FFF000001F007FF800003F003FFC00007E001FFE000 0FC000FFF0001F80007FFE00FF00001FFFFFFE000007FFFFF8000000FFFFE00000001FFE 00002D2E7CAD35>I<00000000007FC00000000000FFFFC00000000000FFFFC000000000 00FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 0000000FFC01FFC0000000FFFF81FFC0000007FFFFE1FFC000001FFFFFF9FFC000007FFC 03FFFFC00000FFF0007FFFC00001FFC0001FFFC00003FF80000FFFC00007FF000007FFC0 000FFE000003FFC0000FFE000003FFC0001FFC000003FFC0001FFC000003FFC0003FFC00 0003FFC0003FFC000003FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC0 00FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF800 0003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC0 00FFF8000003FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC0003FF800 0003FFC0003FFC000003FFC0003FFC000003FFC0001FFC000003FFC0001FFC000003FFC0 000FFE000007FFC00007FF00000FFFC00003FF00001FFFC00001FFC0003FFFC00000FFE0 00FFFFE000007FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FFFF800001FFFF03FFFF 8000001FF803FFFF8039487CC742>I<00001FFE00000001FFFFE0000007FFFFF800001F FFFFFE00007FFC07FF0000FFE001FF8001FFC0007FC003FF80003FE007FF00003FF00FFE 00001FF01FFE00000FF81FFC00000FF83FFC00000FFC3FFC000007FC7FFC000007FC7FF8 000007FC7FF8000007FE7FF8000007FEFFF8000007FEFFF8000007FEFFFFFFFFFFFEFFFF FFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFCFFF800000000FFF800000000FFF800000000FFF8 000000007FF8000000007FF8000000007FFC000000003FFC000000003FFC000000003FFC 0000001C1FFE0000003E0FFE0000003E07FF0000007E07FF000000FC03FF800001F801FF C00003F0007FF0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE000000FFFFF8000000 0FFF80002F2E7DAD36>I<000000FFC000000007FFF80000003FFFFC000000FFFFFF0000 01FFC1FF000007FF03FF80000FFC03FF80000FF807FFC0001FF807FFC0003FF007FFC000 3FF007FFC0003FE003FF80007FE003FF80007FE001FF00007FE000FE00007FE000380000 7FE0000000007FE0000000007FE0000000007FE0000000007FE0000000007FE000000000 7FE0000000007FE0000000007FE0000000007FE0000000FFFFFFFE0000FFFFFFFE0000FF FFFFFE0000FFFFFFFE0000FFFFFFFE0000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF00000003FFFFFF800003FFFFFF800003FFFFFF800003F FFFFF800003FFFFFF800002A487DC724>I<00000000001F8000007FF000FFE00007FFFF 03FFF0001FFFFFC7FFF0007FFFFFFFC7F800FFE03FFE0FF801FF800FFC0FF803FF0007FE 0FF807FE0003FF07F007FE0003FF07F00FFC0001FF81C00FFC0001FF80000FFC0001FF80 001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC000 1FFC0001FFC0000FFC0001FF80000FFC0001FF80000FFC0001FF800007FE0003FF000007 FE0003FF000003FF0007FE000001FF800FFC000000FFE03FF8000001FFFFFFF0000001DF FFFFC0000003C7FFFF00000003C07FF000000007C0000000000007C0000000000007C000 0000000007C0000000000007E0000000000007F0000000000007F8000000000007FFFFFF F0000007FFFFFFFF000003FFFFFFFFE00003FFFFFFFFF80001FFFFFFFFFE0001FFFFFFFF FF0000FFFFFFFFFF80007FFFFFFFFF8003FFFFFFFFFFC00FFFFFFFFFFFC01FF800001FFF E03FE0000001FFE07FC00000007FF07FC00000003FF0FF800000001FF0FF800000001FF0 FF800000001FF0FF800000001FF0FF800000001FF07FC00000003FE07FC00000003FE03F E00000007FC03FF0000000FFC01FFC000003FF800FFF00000FFF0003FFF000FFFC0000FF FFFFFFF000003FFFFFFFC0000007FFFFFE000000003FFFC0000035447DAE3B>I<007FC0 0000000000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC000000000 00FFFFC0000000000003FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC001FFC0000001FFC0 0FFFF8000001FFC03FFFFE000001FFC0FFFFFF000001FFC1FC07FF800001FFC3E003FFC0 0001FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001FFFC 0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF00001FFF00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFFC07FFFFF E0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B487C C742>I<00FC0001FF0003FF8007FFC00FFFC01FFFE01FFFE01FFFE01FFFE01FFFE01FFF E00FFFC007FFC003FF8001FF0000FC000000000000000000000000000000000000000000 00000000000000000000000000007FC0FFFFC0FFFFC0FFFFC0FFFFC0FFFFC003FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0FFFFFFFFFFFFFFFF FFFFFFFFFFFFFF18497CC820>I<007FC000000000FFFFC000000000FFFFC000000000FF FFC000000000FFFFC000000000FFFFC00000000003FFC00000000001FFC00000000001FF C00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC0 0000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000 00000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000 000001FFC00000000001FFC00000000001FFC0007FFFF801FFC0007FFFF801FFC0007FFF F801FFC0007FFFF801FFC0007FFFF801FFC0000FFE0001FFC00007F00001FFC0000FE000 01FFC0003FC00001FFC0007F800001FFC000FE000001FFC001FC000001FFC007F8000001 FFC00FF0000001FFC01FC0000001FFC03F80000001FFC0FF00000001FFC1FE00000001FF C3FF00000001FFCFFF80000001FFDFFFC0000001FFFFFFC0000001FFFFFFE0000001FFFF FFF0000001FFFCFFF8000001FFF87FFC000001FFE03FFC000001FFC01FFE000001FFC01F FF000001FFC00FFF800001FFC007FFC00001FFC003FFC00001FFC001FFE00001FFC001FF F00001FFC000FFF80001FFC0007FFC0001FFC0003FFC0001FFC0001FFE0001FFC0000FFF 0001FFC0000FFF8001FFC0000FFFC0FFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFF FFFFFF807FFFFFFFFFFF807FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FF FFC000FFFFC000FFFFC00003FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC000FFFFFF80FFFFFF80FFFFFF80FFFFFF80FFFFFF8019487CC720>I<007FC001FFC0 0000FFE00000FFFFC00FFFF80007FFFC0000FFFFC03FFFFE001FFFFF0000FFFFC0FFFFFF 007FFFFF8000FFFFC1FC07FF80FE03FFC000FFFFC3E003FFC1F001FFE00003FFC7C001FF C3E000FFE00001FFCF0001FFE78000FFF00001FFDE0000FFEF00007FF00001FFDC0000FF EE00007FF00001FFFC0000FFFE00007FF80001FFF80000FFFC00007FF80001FFF00000FF F800007FF80001FFF00000FFF800007FF80001FFF00000FFF800007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF800FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFF FFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFF FFE03FFFFFF05C2E7CAD63>I<007FC001FFC00000FFFFC00FFFF80000FFFFC03FFFFE00 00FFFFC0FFFFFF0000FFFFC1FC07FF8000FFFFC3E003FFC00003FFC7C001FFC00001FFCF 0001FFE00001FFDE0000FFE00001FFDC0000FFE00001FFFC0000FFF00001FFF80000FFF0 0001FFF00000FFF00001FFF00000FFF00001FFF00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFF C07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B2E7CAD42>I<00000FFF00000000 00FFFFF000000007FFFFFE0000001FFFFFFF8000003FFC03FFC00000FFE0007FF00001FF 80001FF80003FF00000FFC0007FE000007FE000FFE000007FF000FFC000003FF001FFC00 0003FF803FFC000003FFC03FF8000001FFC03FF8000001FFC07FF8000001FFE07FF80000 01FFE07FF8000001FFE0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001 FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FF F0FFF8000001FFF07FF8000001FFE07FF8000001FFE07FF8000001FFE07FF8000001FFE0 3FFC000003FFC03FFC000003FFC01FFC000003FF801FFE000007FF800FFE000007FF0007 FF00000FFE0003FF80001FFC0001FFC0003FF80000FFE0007FF000007FFC03FFE000001F FFFFFF80000007FFFFFE00000000FFFFF0000000000FFF000000342E7DAD3B>I<007FC0 0FFC000000FFFFC07FFFC00000FFFFC3FFFFF00000FFFFCFFFFFFC0000FFFFDFF01FFF00 00FFFFFF8007FF800003FFFE0001FFC00001FFF80000FFE00001FFF00000FFF00001FFE0 00007FF80001FFE000003FFC0001FFE000003FFC0001FFE000003FFE0001FFE000001FFE 0001FFE000001FFF0001FFE000001FFF0001FFE000001FFF0001FFE000000FFF0001FFE0 00000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF 8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE0 00000FFF8001FFE000000FFF0001FFE000001FFF0001FFE000001FFF0001FFE000001FFE 0001FFE000001FFE0001FFE000003FFC0001FFE000003FFC0001FFE000007FF80001FFF0 00007FF80001FFF80000FFF00001FFFC0001FFE00001FFFE0003FFC00001FFFF0007FF80 0001FFFFE03FFE000001FFEFFFFFFC000001FFE3FFFFF0000001FFE0FFFF80000001FFE0 1FF800000001FFE0000000000001FFE0000000000001FFE0000000000001FFE000000000 0001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0 000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE000000000 0001FFE0000000000001FFE00000000000FFFFFFC000000000FFFFFFC000000000FFFFFF C000000000FFFFFFC000000000FFFFFFC00000000039427CAD42>I<00FF803F8000FFFF 80FFF000FFFF83FFFC00FFFF87FFFE00FFFF8FC3FF00FFFF8F07FF0003FF9E0FFF8001FF BC0FFF8001FFB80FFF8001FFF80FFF8001FFF00FFF8001FFF007FF0001FFF007FF0001FF E003FE0001FFE000F80001FFE000000001FFE000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC0000000FFFFFFE00000FFFFFFE00000FFFF FFE00000FFFFFFE00000FFFFFFE00000292E7CAD31>114 D<000FFF00E0007FFFF3E001 FFFFFFE007FFFFFFE00FF800FFE01FC0001FE03F80000FE03F000007E07F000003E07F00 0003E0FF000003E0FF000003E0FF800003E0FFC0000000FFF0000000FFFE000000FFFFF8 00007FFFFFC0007FFFFFF0003FFFFFFC001FFFFFFF000FFFFFFF8007FFFFFFC003FFFFFF E000FFFFFFF0003FFFFFF00003FFFFF800001FFFF8000000FFFC0000001FFC7800000FFC F8000007FCF8000003FCFC000003FCFC000003FCFE000003F8FE000003F8FF000003F8FF 800007F0FFC0000FF0FFF0001FE0FFFC00FFC0FFFFFFFF80FC7FFFFE00F81FFFF800E003 FF8000262E7CAD2F>I<0001F000000001F000000001F000000001F000000001F0000000 01F000000003F000000003F000000003F000000007F000000007F000000007F00000000F F00000000FF00000001FF00000003FF00000003FF00000007FF0000001FFF0000003FFF0 00000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC000FFF0000000FFF000 0000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF00000 00FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000 FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF0 01F000FFF001F0007FF001E0007FF803E0003FF803E0003FFC07C0001FFE0F80000FFFFF 800007FFFE000001FFFC0000001FF00024427EC12E>I<007FE000003FF000FFFFE0007F FFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF00003 FFE00001FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00001FFF00001FFE00001FFF00001FFE00001FFF00001FFE00003FFF00000FFE00007 FFF00000FFE0000F7FF000007FE0001F7FF000007FF0003E7FF800003FFC00FC7FFFE000 1FFFFFF87FFFE00007FFFFE07FFFE00001FFFF807FFFE000003FFE007FFFE03B2E7CAD42 >III<7FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFF FF007FFFFF801FFFFF00007FF80001FF0000007FFC0001FC0000003FFE0003F80000001F FF0007F00000000FFF0007E000000007FF800FC000000003FFC01F8000000003FFE03F80 00000001FFF07F0000000000FFF8FE00000000007FF9FC00000000003FFFF80000000000 3FFFF000000000001FFFE000000000000FFFC0000000000007FFC0000000000003FFC000 0000000001FFE0000000000001FFF0000000000001FFF8000000000003FFFC0000000000 03FFFE000000000007FFFE00000000000FEFFF00000000001FCFFF80000000003F87FFC0 000000007F03FFE000000000FE01FFE000000001FC00FFF000000001F8007FF800000003 F0007FFC00000007F0003FFE0000000FE0001FFF0000001FC0000FFF0000003F800007FF 800000FF800007FFC000FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFF F8003FFFFFC0FFFFF8003FFFFFC03A2E7EAD3F>I<7FFFFFC000FFFF807FFFFFC000FFFF 807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF8000FFF000000FE00000FFF8 00000FC00000FFF800000FC000007FFC00000F8000007FFC00001F8000003FFC00001F00 00003FFE00003F0000001FFE00003E0000001FFF00007E0000000FFF00007C0000000FFF 8000FC00000007FF8000F800000007FFC001F800000003FFC001F000000003FFE003F000 000003FFE003F000000001FFF003E000000001FFF007E000000000FFF007C000000000FF F80FC0000000007FF80F80000000007FFC1F80000000003FFC1F00000000003FFE3F0000 0000001FFE3E00000000001FFF7E00000000000FFF7C00000000000FFFFC00000000000F FFFC000000000007FFF8000000000007FFF8000000000003FFF0000000000003FFF00000 00000001FFE0000000000001FFE0000000000000FFC0000000000000FFC0000000000000 7F800000000000007F800000000000003F000000000000003F000000000000003F000000 000000003E000000000000007E000000000000007C00000000000000FC000000001F8000 F8000000003FC001F8000000007FE001F000000000FFF003F000000000FFF003E0000000 00FFF007E000000000FFF00FC000000000FFF01F8000000000FFF03F80000000007FE07F 00000000007F43FE00000000003FFFF800000000001FFFF0000000000007FFC000000000 0001FE00000000000039427EAD3F>I E /Fs 10 58 df<1F00318060C04040C060C060C0 60C060C060C060C060C060404060C031801F000B107F8F0F>48 D<0C003C00CC000C000C 000C000C000C000C000C000C000C000C000C000C00FF8009107E8F0F>I<1F00618040C0 8060C0600060006000C00180030006000C00102020207FC0FFC00B107F8F0F>I<1F0021 8060C060C000C0008001800F00008000400060C060C060804060801F000B107F8F0F>I< 0300030007000F000B001300330023004300C300FFE003000300030003001FE00B107F8F 0F>I<20803F002C002000200020002F0030802040006000600060C06080C061801F000B 107F8F0F>I<0780184030C060C06000C000CF00F080E040C060C060C060406060C03080 1F000B107F8F0F>I<40007FE07FC08080808001000200040004000C0008000800180018 001800180018000B117E900F>I<1F00318060C060C060C071803F000F00338061C0C060 C060C060404060801F000B107F8F0F>I<1F00318060C0C040C060C060C06040E021E01E 600060004060C0608043003E000B107F8F0F>I E /Ft 16 104 df0 D<70F8F8F87005057C8D0D>I<400004C0000C6000183000301800600C00 C006018003030001860000CC0000780000300000300000780000CC000186000303000601 800C00C0180060300030600018C0000C40000416187A9623>I<01800180018001800180 C183F18F399C0FF003C003C00FF0399CF18FC1830180018001800180018010147D9417> I<0001FE00000007FF8000001E01E000007800780000E0001C0001800006000300000300 06000001800C000000C00C000000C0180000006030000000303000000030300000003060 0000001860000000186000000018C00000000CC00000000CC00000000CC00000000CC000 00000CC00000000CC00000000CC00000000CC00000000C60000000186000000018600000 001830000000303000000030300000003018000000600C000000C00C000000C006000001 800300000300018000060000E0001C000078007800001E01E0000007FF80000001FE0000 262B7DA02D>13 D<03C00FF01C38300C60066006C003C003C003C003C003C00360066006 300C1C380FF003C010127D9317>I<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFF FFFFFF7FFE7FFE3FFC1FF80FF003C010127D9317>I<000000C0000003C000000F000000 3C000000F0000003C00000070000001C00000078000001E00000078000001E0000007800 0000E0000000780000001E0000000780000001E0000000780000001C0000000700000003 C0000000F00000003C0000000F00000003C0000000C00000000000000000000000000000 00000000000000000000000000007FFFFF80FFFFFFC01A247C9C23>20 DI<003FF800FFF803C0000700000C000018000030000030000060000060 0000C00000C00000C00000FFFFF8FFFFF8C00000C00000C0000060000060000030000030 00001800000C000007000003C00000FFF8003FF8151C7C981E>50 D<00000C00000C0000180000180000300000300000600000600000C00000C00001800001 80000180000300000300000600000600000C00000C000018000018000030000030000060 0000600000C00000C0000180000180000300000300000600000600000600000C00000C00 00180000180000300000300000600000600000C00000400000162C7AA000>54 D<003FFFC001FFFFF0071E03FC081E00FE181E003E701E001F701E001FC01E000F001C00 0F001C000F003C000E003C000E003C001C0038001C003800380078003000780040007000 800070030000F01C0000F3F00000E7800000E0000001E0000001C0000001C0000003C000 00038000000380000007000000070000000E0000000C00000020217F9E20>80 D<0000FE000007FF80001C1FC0002007E000C003F0018001F0030000F0060000F80E0000 F81C0000781C00007838000078380000787000007870000078F0000070F0000070F00000 F0F00000E0F00000E0F80001C0F80001C07C0003807E0083003F0707001FFC0E000FE00C 0000001000000020000000C000000F00001FFC000C7FF8001807FF0030007FF060000FFF C00000FF001E257B9F25>I<400002C00006C00006C00006C00006C00006C00006C00006 C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006 C0000660000C60000C3000181C00700F01E003FF8000FE00171C7D9A1E>91 D<000F0038006000E001C001C001C001C001C001C001C001C001C001C001C001C001C001 C001C0038007001E00F8001E000700038001C001C001C001C001C001C001C001C001C001 C001C001C001C001C001C000E000600038000F102D7DA117>102 DI E /Fu 89 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8F870000000000070F8F8F87005 1C779B18>33 D<4010E038F078E038E038E038E038E038E038E038E038E038E03860300D 0E7B9C18>I<030600078F00078F00078F00078F00078F00078F007FFFC0FFFFE0FFFFE0 7FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFFC0FFFFE0FFFFE07FFFC01E3C00 1E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18>I<3803007C07807C0780EE0F 80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C007C3C00387C0000780000780000F8 0000F00001F00001E00001E00003E00003C00003C00007C0000783800787C00F87C00F0E E00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07C018038013247E9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C38001C73F01C73F01CE3F00FE3 800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00E0F800E0F800E07070E078 7070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C7C7E3E0E0E0E1C1C38F8F0C0070E 789B18>I<007000F001E003C007800F001E001C00380038007000700070007000E000E0 00E000E000E000E000E000E0007000700070007000380038001C001E000F00078003C001 F000F000700C24799F18>I<6000F00078003C001E000F000780038001C001C000E000E0 00E000E00070007000700070007000700070007000E000E000E000E001C001C003800780 0F001E003C007800F00060000C247C9F18>I<01C00001C00001C00001C000C1C180F1C7 80F9CF807FFF001FFC0007F00007F0001FFC007FFF00F9CF80F1C780C1C18001C00001C0 0001C00001C00011147D9718>I<00600000F00000F00000F00000F00000F00000F00000 F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F00000F00000F00000F00000F00000 600013147E9718>I<1C3E7E7F3F1F070E1E7CF860080C788518>I<7FFF00FFFF80FFFF80 7FFF0011047D8F18>I<3078FCFC78300606778518>I<000300000780000780000F80000F 00001F00001E00001E00003E00003C00007C0000780000780000F80000F00001F00001E0 0003E00003C00003C00007C0000780000F80000F00000F00001F00001E00003E00003C00 003C00007C0000780000F80000F00000F0000060000011247D9F18>I<01F00007FC000F FE001F1F001C07003803807803C07001C07001C0E000E0E000E0E000E0E000E0E000E0E0 00E0E000E0E000E0E000E0F001E07001C07001C07803C03803801C07001F1F000FFE0007 FC0001F000131C7E9B18>I<01800380038007800F803F80FF80FB804380038003800380 03800380038003800380038003800380038003800380038003807FFCFFFE7FFC0F1C7B9B 18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0F000E06000E00000E00000E0 0001C00001C00003C0000780000F00001E00003C0000780000F00001E00007C0000F8000 1E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8001FFE003FFF007807807803 C07801C03001C00001C00003C0000380000F0003FF0003FE0003FF000007800003C00001 C00000E00000E00000E0F000E0F000E0F001C0F003C07C07803FFF001FFE0003F800131C 7E9B18>I<001F00003F0000770000770000E70001E70001C7000387000787000707000E 07001E07003C0700380700780700F00700FFFFF8FFFFF8FFFFF800070000070000070000 0700000700000700007FF000FFF8007FF0151C7F9B18>I<1FFF803FFF803FFF80380000 3800003800003800003800003800003800003800003BF8003FFE003FFF003C07801803C0 0001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F803FFF001FFC0003F000 131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03C03801803800007000007000 00E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000E0F000E07000E07000E07000 E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18>II<03F8000FFE001FFF003E0F803803807001C07001C07001C0 7001C03803803C07801FFF0007FC000FFE001F1F003C07807001C0F001E0E000E0E000E0 E000E0E000E07001C07803C03E0F801FFF000FFE0003F800131C7E9B18>I<03F0000FFC 001FFE003C0F00780780700380E001C0E001C0E001C0E001E0E001E07001E07803E03C0F E01FFFE00FFEE003F0E00000E00001C00001C00001C0300380780780780F00783E003FFC 001FF00007C000131C7E9B18>I<3078FCFC783000000000000000003078FCFC78300614 779318>I<183C7E7E3C180000000000000000183C7E7E3E1E0E1C3C78F060071A789318> I<000300000780001F80003F00007E0001FC0003F00007E0001FC0003F00007E0000FC00 00FC00007E00003F00001FC00007E00003F00001FC00007E00003F00001F800007800003 0011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE0000000000000000000000000FFFFE0FF FFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00007E00003F00001FC00007E000 03F00001FC00007E00003F00001F80001F80003F00007E0001FC0003F00007E0001FC000 3F00007E0000FC0000F0000060000011187D9918>I<007C0001FE0007FF000F87801E03 C03C1DC0387FC070FFE071E3E071C1E0E1C1E0E380E0E380E0E380E0E380E0E380E0E380 E0E1C1C071C1C071E3C070FF80387F003C1C001E00E00F83E007FFC001FF80007E00131C 7E9B18>64 D<00700000F80000F80000D80000D80001DC0001DC0001DC00018C00038E00 038E00038E00038E000306000707000707000707000707000FFF800FFF800FFF800E0380 0E03801C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>II<00F8E003FEE007FFE00F07E01E03E03C01E03800E07000E07000E070 0000E00000E00000E00000E00000E00000E00000E00000E000007000007000E07000E038 00E03C00E01E01C00F07C007FF8003FE0000F800131C7E9B18>I<7FF800FFFE007FFF00 1C0F801C03C01C03C01C01E01C00E01C00E01C00F01C00701C00701C00701C00701C0070 1C00701C00701C00701C00F01C00E01C00E01C01E01C01C01C03C01C0F807FFF00FFFE00 7FF800141C7F9B18>III<01F1C003FDC00FFFC01F0FC01C03C03803C03801C07001C0 7001C0700000E00000E00000E00000E00000E00000E00FF0E01FF0E00FF07001C07001C0 7003C03803C03803C01C07C01F0FC00FFFC003FDC001F1C0141C7E9B18>I<7F07F0FF8F F87F07F01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01FFFC01FFF C01FFFC01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C07F07 F0FF8FF87F07F0151C7F9B18>I<7FFF00FFFF807FFF0001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C0007FFF00FFFF807FFF00111C7D9B18>I<7F07F0 FF87F87F07F01C03C01C07801C07001C0E001C1E001C3C001C38001C70001CF0001DF000 1DF0001FB8001FB8001F1C001E1C001C0E001C0E001C07001C07001C03801C03801C01C0 7F03F0FF87F87F03F0151C7F9B18>75 D<7FE000FFE0007FE0000E00000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E 00000E00000E00700E00700E00700E00700E00707FFFF0FFFFF07FFFF0141C7F9B18>I< FC01F8FE03F8FE03F83B06E03B06E03B06E03B06E03B8EE03B8EE0398CE0398CE039DCE0 39DCE039DCE038D8E038D8E038F8E03870E03870E03800E03800E03800E03800E03800E0 3800E0FE03F8FE03F8FE03F8151C7F9B18>I<7E07F0FF0FF87F07F01D81C01D81C01D81 C01DC1C01CC1C01CC1C01CE1C01CE1C01CE1C01C61C01C71C01C71C01C31C01C39C01C39 C01C39C01C19C01C19C01C1DC01C0DC01C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18 >I<0FF8003FFE007FFF00780F00700700F00780E00380E00380E00380E00380E00380E0 0380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380F0078070 0700780F007FFF003FFE000FF800111C7D9B18>II<0FF8003FFE007FFF00780F00700700F00780E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380E00380E00380E00380E00380E1E380E1E380F0E7 8070F700787F007FFF003FFE000FFC00001C00001E00000E00000F000007000007001122 7D9B18>I<7FF800FFFE007FFF001C0F801C03801C03C01C01C01C01C01C01C01C03C01C 03801C0F801FFF001FFE001FFE001C0F001C07001C03801C03801C03801C03801C03801C 039C1C039C1C039C7F01F8FF81F87F00F0161C7F9B18>I<03F3801FFF803FFF807C0F80 700780E00380E00380E00380E000007000007800003F00001FF00007FE0000FF00000F80 0003C00001C00000E00000E06000E0E000E0E001E0F001C0F80780FFFF80FFFE00E7F800 131C7E9B18>I<7FFFF8FFFFF8FFFFF8E07038E07038E07038E070380070000070000070 000070000070000070000070000070000070000070000070000070000070000070000070 0000700000700000700007FF0007FF0007FF00151C7F9B18>IIII<7F8FE07F9FE07F8FE00E07000F0700070E00078E00039C0003 DC0001F80001F80000F00000F00000700000F00000F80001F80001DC00039E00038E0007 0F000707000E07800E03801E03C07F07F0FF8FF87F07F0151C7F9B18>II<3FFFE07FFFE07FFFE07001C07003C0700780700700000F 00001E00001C00003C0000780000700000F00001E00001C00003C0000780000700000F00 001E00E01C00E03C00E07800E07000E0FFFFE0FFFFE0FFFFE0131C7E9B18>II<600000F00000F00000F800007800007C00003C00003C00003E00001E00001F0000 0F00000F00000F800007800007C00003C00003C00003E00001E00001F00000F00000F800 007800007800007C00003C00003E00001E00001E00001F00000F00000F80000780000780 00030011247D9F18>II<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C38070E789E18>I<1FE0003FF8007FFC00781E00300E 0000070000070000FF0007FF001FFF007F0700780700E00700E00700E00700F00F00781F 003FFFF01FFBF007E1F014147D9318>I<7E0000FE00007E00000E00000E00000E00000E 00000E00000E3E000EFF800FFFC00FC1E00F80E00F00700E00700E00380E00380E00380E 00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF80063E00151C809B18>I< 01FE0007FF001FFF803E0780380300700000700000E00000E00000E00000E00000E00000 E000007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80003F 80001F8000038000038000038000038000038003E3800FFB801FFF803C1F80380F807007 80700380E00380E00380E00380E00380E00380E00380700780700780380F803C1F801FFF F00FFBF803E3F0151C7E9B18>I<01F00007FC001FFE003E0F00380780700380700380E0 01C0E001C0FFFFC0FFFFC0FFFFC0E000007000007001C03801C03E03C01FFF8007FF0001 FC0012147D9318>I<001F80007FC000FFE000E1E001C0C001C00001C00001C0007FFFC0 FFFFC0FFFFC001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C0007FFF007FFF007FFF00131C7F9B18>I<01E1F007FFF80FFF F81E1E301C0E003807003807003807003807003807001C0E001E1E001FFC001FF80039E0 003800001C00001FFE001FFFC03FFFE07801F0700070E00038E00038E00038E000387800 F07E03F01FFFC00FFF8001FC00151F7F9318>I<7E0000FE00007E00000E00000E00000E 00000E00000E00000E3E000EFF800FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B 18>I<03800007C00007C00007C0000380000000000000000000000000007FC000FFC000 7FC00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C000FFFF00FFFF80FFFF00111D7C9C18>I<0038007C007C007C003800 000000000000000FFC1FFC0FFC001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C6038F078FFF07FE03F800E277E9C18 >II<7FE000FFE0007FE00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E0007FFFC0FFFFE07FFFC0131C7E 9B18>I<7CE0E000FFFBF8007FFFF8001F1F1C001E1E1C001E1E1C001C1C1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 7F1F1F00FFBFBF807F1F1F001914819318>I<7E3E00FEFF807FFFC00FC1C00F80E00F00 E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3 FCFFE7FE7FC3FC1714809318>I<01F0000FFE001FFF003E0F803803807001C07001C0E0 00E0E000E0E000E0E000E0E000E0F001E07001C07803C03C07803E0F801FFF000FFE0001 F00013147E9318>I<7E3E00FEFF807FFFC00FC1E00F80E00F00700E00700E00380E0038 0E00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF800E3E000E0000 0E00000E00000E00000E00000E00000E00007FC000FFE0007FC000151E809318>I<01E3 8007FB801FFF803E1F80380F80700780700780E00380E00380E00380E00380E00380E003 80700780700780380F803C1F801FFF800FFB8003E3800003800003800003800003800003 80000380000380003FF8003FF8003FF8151E7E9318>I<7F87E0FF9FF07FBFF803F87803 F03003E00003C00003C00003800003800003800003800003800003800003800003800003 80007FFE00FFFF007FFE0015147F9318>I<07F7003FFF007FFF00780F00E00700E00700 E007007C00007FE0001FFC0003FE00001F00600780E00380E00380F00380F80F00FFFF00 FFFC00E7F00011147D9318>I<0180000380000380000380000380007FFFC0FFFFC0FFFF C00380000380000380000380000380000380000380000380000380000380400380E00380 E00380E001C1C001FFC000FF80003E0013197F9818>I<7E07E0FE0FE07E07E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E01E00F 03E007FFFC03FFFE01FCFC1714809318>I<7F8FF0FF8FF87F8FF01E03C00E03800E0380 0E0380070700070700070700038E00038E00038E00038E0001DC0001DC0001DC0000F800 00F80000700015147F9318>II<7F8FF07F9FF07F8FF0070700078E00039E0001DC0001F80000F80000 700000F00000F80001DC00039E00038E000707000F07807F8FF0FF8FF87F8FF015147F93 18>I<7F8FF0FF8FF87F8FF00E01C00E03800E0380070380070700070700038700038600 038E0001CE0001CE0000CC0000CC0000DC00007800007800007800007000007000007000 00F00000E00079E0007BC0007F80003F00001E0000151E7F9318>I<3FFFF07FFFF07FFF F07001E07003C0700780000F00001E00003C0000F80001F00003C0000780000F00701E00 703C0070780070FFFFF0FFFFF0FFFFF014147F9318>I<0007E0001FE0007FE000780000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00001E0007F C000FF8000FF80007FC00001E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E000007800007FE0001FE00007E013247E9F18>I<60F0F0F0F0F0 F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0600424769F18> I<7C0000FF0000FFC00003C00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000F000007FC0003FE0003FE0007FC000F00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00003C000FFC000FF00007C00 0013247E9F18>I E /Fv 78 123 df<007F07F001FF1FF003FF3FF00780780007007000 0F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000FFF8FF80FFF8FF80 FFF8FF800F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000 0F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000 1C20809F1B>11 D<0000F000F8F001F8F003F8F00780000700000F00000F00000F00000F 00000F00000F00000F0000FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F 00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F014 2180A018>I<00F8F001F8F003F8F00780F00700F00F00F00F00F00F00F00F00F00F00F0 0F00F00F00F0FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0 0F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01420809F18> I<783C783C783C783C180C3018301860306030E0700E0A7F9F17>34 D<007C000000FE000001FF000003C78000038780000383C0000783C0000703C0000703C0 000783C00007878000078F8000078F0000079E000003BC008003F800E003F001E003E001 C007E001C00FF003C01EF003803CF8078078780700787C0F00F03E0E00F01F1E00F01F3C 00F00FF800F807F0007803E0007C0FF8303FFFFFF01FFE3FF007F00FC01C227EA022>38 D<007000E001C00380078007000E001E001E003C003C003C0078007800780078007000F0 00F000F000F000F000F000F000F000F000F000F000F000700078007800780078003C003C 003C001E001E000E0007000780038001C000E000700C2E7EA112>40 DI<018001C001800180 C183E187F99F7DBE1FF807E007E01FF87DBEF99FE187C1830180018001C0018010147DA1 17>I<000600000006000000060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFF0FFFFFFF000 060000000600000006000000060000000600000006000000060000000600000006000000 06000000060000000600000006000000060000000600001C207D9A23>I<787878781830 306060E0050A7D830D>III<03F0 0007F8001FFE001E1E003C0F00380700780780780780700380F003C0F003C0F003C0F003 C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07807807807 807807803807003C0F001E1E001FFE000FFC0003F00012207E9E17>48 D<00C001C00FC0FFC0FFC0F3C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C003C003C003C0FFFEFFFEFFFE0F1F7C9E17>I<07F0 000FFC001FFE00383F00700F00600780E00780E003C04003C04003C00003C00003C00003 C0000780000780000F00000E00001C00003C0000780000E00001C0000380000700000E00 001C0000380000700000FFFFC0FFFFC0FFFFC0121F7E9E17>I<03F0000FFC001FFE003C 1F00780F00300780200780000780000780000780000F00000F00003E0003FC0003F80003 FC00001E00000F000007800003800003C00003C00003C00003C08003C0C003C0C0078070 0F807C1F003FFE000FFC0003F00012207E9E17>I<003E00003E00005E00005E0000DE00 01DE00019E00039E00039E00079E00071E000F1E000E1E001E1E003C1E003C1E00781E00 781E00F01E00FFFFF0FFFFF0FFFFF0001E00001E00001E00001E00001E00001E00001E00 001E00141E7F9D17>I<7FFF007FFF007FFF007800007800007800007800007800007800 0078000079F0007FFC007FFE007F1F007C07007C07807807800003C00003C00003C00003 C00003C00003C00003C0400780600780F00F007C1E003FFC001FF80007E000121F7E9D17 >I<007C0001FE0003FE000782000F00001E00003C00003C000078000078000078F800F3 FC00F7FE00FE1F00FC0F80F80780F80780F003C0F003C0F003C0F003C0F003C07003C078 03C07803C07807803807803C0F001E1E000FFE0007F80001F00012207E9E17>II<03F0000FFC001FFE003E1F003C0F 007807807807807807807807807807807807803C0F001E1E000FFC0007F8000FFC001F3E 003C0F00780780780780F003C0F003C0F003C0F003C0F003C0F003C07807807C0F803E1F 001FFE000FFC0003F00012207E9E17>I<03F00007F8000FFC001E1E003C0F0078070078 0780F00780F00380F003C0F003C0F003C0F003C0F003C0F003C07807C07807C07C0FC03E 1FC01FFBC00FF3C007C780000780000780000700000F00001E00201E00307C007FF8003F F0000FC00012207E9E17>II61 D<001F0000001F0000003F800000 3B8000003B8000007BC0000073C0000071C00000F1E00000E1E00000E0E00001E0F00001 E0F00001C0F00003C0780003C078000380780007803C0007803C0007003C000FFFFE000F FFFE000FFFFE001E000F001E000F003C000F803C0007803C000780780007C0780003C078 0003C0F00003E01B207F9F1E>65 DI<001FC000FFF801FFFC03E03C07800C0F00001E00003E00003C00 007C0000780000780000780000F00000F00000F00000F00000F00000F00000F00000F000 007800007800007800007C00003C00003E00001E00000F000207800E03E03E01FFFC00FF F0001FC017227DA01D>II II<001FE000 FFF801FFFE03E03E07800E0F00001E00003E00003C00007C0000780000780000780000F0 0000F00000F00000F00000F00000F00000F003FEF003FE7803FE78001E78001E7C001E3C 001E3E001E1E001E0F001E07801E03E03E01FFFE00FFF8001FC017227DA01E>III75 DIII<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E00 3C000F003C000F00780007807800078078000780F00003C0F00003C0F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C0F00003C0F80007C0780007807800078078000780 3C000F003C000F001E001E001F003E000F807C0007E1F80003FFF00000FFC000003F0000 1A227DA021>II<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C00 0F00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F000 03C0F00003C0F00003C0F00003C0F00003C07800078078000780781E07803C0F0F003C0F 8F001E079E001F03FE000F83FC0007E1F80003FFF00000FFF800003F780000007C000000 3E0000001E0000001F0000000F801A277DA021>II<01FC0007FF800FFFC01F03C03C00C03C00007800 007800007800007800007800007C00003C00003F00001FE0000FFC0007FE0001FF00003F 800007C00003C00003E00001E00001E00001E00001E00001E00001C0C003C0F007C0FC0F 807FFF001FFE0003F80013227EA019>IIIII<780007807C000F003E001F001E001E00 0F003C000F807C000780780003C0F00003E1F00001F1E00000F3C000007FC000007F8000 003F0000001F0000001E0000003F0000007F8000007FC00000F3C00001F1E00001E0F000 03C0F80007C0780007803C000F003E001F001E001E000F003C000F807C0007C0780003C0 F00003E01B207F9F1E>IIII<381C3018301860306030C060F078F078F078F0780E0A79 9F17>II<07E03FF87FFC701E401F000F000F000F 003F07FF1FFF7E0FF80FF00FF00FF00FF83F7FFF3FEF1F8F10147E9316>97 DI<03F00FFC 1FFE3E0E3C0278007800F000F000F000F000F000F000780078003C013E0F1FFF0FFE03F0 10147E9314>I<0007800007800007800007800007800007800007800007800007800007 8000078000078007C7800FF7801FFF803E1F807C0780780780F80780F00780F00780F007 80F00780F00780F00780F00780780780780F803E1F801FFF800FF78007C78011207E9F17 >I<03F0000FFC001FFE003E1F003C0700780700700380FFFF80FFFF80FFFF80F00000F0 0000F000007000007800003C01003E07001FFF0007FE0001F80011147F9314>I<007E01 FE03FE078007000F000F000F000F000F000F000F00FFF0FFF0FFF00F000F000F000F000F 000F000F000F000F000F000F000F000F000F000F000F000F000F20809F0E>I<03E0F00F FFF01FFFF03E3E003C1E00780F00780F00780F00780F00780F003C1E003E3E001FFC003F F80033E0003000003800003FFE003FFF801FFFC03FFFE07803F0F000F0F000F0F000F0F8 01F07E07E03FFFC00FFF0003FC00141E7F9317>III<01E001E001E001E00000000000 000000000000000000000001E001E001E001E001E001E001E001E001E001E001E001E001 E001E001E001E001E001E001E001E001E001E001E001E001E0C3C0FFC0FF803F000B2983 9F0C>IIIII<01F80007FE001FFF 803F0FC03C03C07801E07801E0F000F0F000F0F000F0F000F0F000F0F000F07801E07801 E03C03C03F0FC01FFF8007FE0001F80014147F9317>II<03C7800FF7801FFF803E1F807C0F80780780780780F00780F00780 F00780F00780F00780F00780F807807807807C0F803E1F801FFF800FF78007C780000780 000780000780000780000780000780000780000780000780111D7E9317>II<07F01FFC3FFC780C7800780078007C003FC01FF00FF803F8007C003C003C C03CF07CFFF87FF00FC00E147F9311>I<1E001E001E001E001E001E00FFF0FFF0FFF01E 001E001E001E001E001E001E001E001E001E001E001E001E001E201FF00FF007C00C1A7F 9910>IIII<7801E07C03C03E07801E0F 000F0F00079E0003FC0003F80001F80000F00001F00001F80003FC00079E000F0F000E0F 001E07803C03C07801E0F801F01414809315>II<7FFF7FFF7FFF003E003C007800F800F001E003E007C007800F001F001E00 3C007C00FFFFFFFFFFFF10147F9314>I E /Fw 1 98 df<001800001800001800003C00 003C00004E00004E00004E000087000087000187800103800103800201C00201C003FFC0 0400E00400E00800700800701800703C0078FE01FF18177F961C>97 D E /Fx 91 128 df<001F83E000F06E3001C078780380F8780300F03007007000070070 000700700007007000070070000700700007007000FFFFFF800700700007007000070070 000700700007007000070070000700700007007000070070000700700007007000070070 000700700007007000070070000700700007007000070070007FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700000700000700 00FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000 E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F04 0001C07C06000380F80F000300F00F000700F00F00070070000007007000000700700000 070070000007007000000700700000FFFFFFFF0007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 07007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C07C0F000380F80F000300 F00700070070070007007007000700700700070070070007007007000700700700070070 0700FFFFFFFF000700700700070070070007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 070070070007007007000700700700070070070007007007007FE3FE3FF02420809F26> I22 D<70F8F8F8F8F8F8F87070707070707070707020202020200000 00000070F8F8F87005217CA00D>33 D<7038F87CFC7EFC7E743A04020402040208040804 10081008201040200F0E7E9F17>I<000300C0000300C0000300C0000300C00006018000 060180000601800006018000060180000C0300000C0300000C0300000C0300000C030000 1806007FFFFFFCFFFFFFFE00300C0000300C0000300C0000300C00006018000060180000 60180000601800FFFFFFFE7FFFFFFC00C030000180600001806000018060000180600001 8060000300C0000300C0000300C0000300C0000300C0000601800006018000060180001F 297D9F26>I<007800000084000001840000030200000702000007020000070200000702 0000070400000704000007080000070800000310000003A00FFC03C003E0038001C001C0 008001C0010003E0010004E0020008F00200187004003078080070380800701C1000F01E 1000F00E2000F0074000F003C0087003C0087801C010380670301C18386007E00F801E22 7EA023>38 D<70F8FCFC74040404080810102040060E7C9F0D>I<002000400080010002 0006000C000C00180018003000300030007000600060006000E000E000E000E000E000E0 00E000E000E000E000E000E0006000600060007000300030003000180018000C000C0006 00020001000080004000200B2E7DA112>I<800040002000100008000C00060006000300 030001800180018001C000C000C000C000E000E000E000E000E000E000E000E000E000E0 00E000E000C000C000C001C001800180018003000300060006000C000800100020004000 80000B2E7DA112>I<000600000006000000060000000600000006000000060000000600 000006000000060000000600000006000000060000000600000006000000060000FFFFFF F0FFFFFFF000060000000600000006000000060000000600000006000000060000000600 00000600000006000000060000000600000006000000060000000600001C207D9A23>43 D<70F8FCFC74040404080810102040060E7C840D>II<70F8F8F8 7005057C840D>I<000100030003000600060006000C000C000C00180018001800300030 003000600060006000C000C000C00180018001800300030003000600060006000C000C00 0C00180018001800300030003000600060006000C000C000C000102D7DA117>I<03F000 0E1C001C0E00180600380700700380700380700380700380F003C0F003C0F003C0F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0700380700380700380 7807803807001806001C0E000E1C0003F000121F7E9D17>I<018003800F80F380038003 800380038003800380038003800380038003800380038003800380038003800380038003 80038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C00100E0020070040078080 0780F007C0F803C0F803C0F803C02007C00007C0000780000780000F00000E00001C0000 380000700000600000C0000180000300000600400C00401800401000803FFF807FFF80FF FF80121E7E9D17>I<03F0000C1C00100E00200F00780F80780780780780380F80000F80 000F00000F00000E00001C0000380003F000003C00000E00000F000007800007800007C0 2007C0F807C0F807C0F807C0F00780400780400F00200E001C3C0003F000121F7E9D17> I<000600000600000E00000E00001E00002E00002E00004E00008E00008E00010E00020E 00020E00040E00080E00080E00100E00200E00200E00400E00C00E00FFFFF0000E00000E 00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I<1803001FFE001FFC001F F8001FE00010000010000010000010000010000010000011F000161C00180E0010070010 07800003800003800003C00003C00003C07003C0F003C0F003C0E0038040038040070020 0600100E000C380003E000121F7E9D17>I<007C000182000701000E03800C07801C0780 380300380000780000700000700000F1F000F21C00F40600F80700F80380F80380F003C0 F003C0F003C0F003C0F003C07003C07003C07003803803803807001807000C0E00061C00 01F000121F7E9D17>I<4000007FFFC07FFF807FFF804001008002008002008004000008 0000080000100000200000200000400000400000C00000C00001C0000180000380000380 00038000038000078000078000078000078000078000078000078000030000121F7D9D17 >I<03F0000C0C001006003003002001806001806001806001807001807803003E03003F 06001FC8000FF00003F80007FC000C7E00103F00300F806003804001C0C001C0C000C0C0 00C0C000C0C000806001802001001002000C0C0003F000121F7E9D17>I<03F0000E1800 1C0C00380600380700700700700380F00380F00380F003C0F003C0F003C0F003C0F003C0 7007C07007C03807C0180BC00E13C003E3C0000380000380000380000700300700780600 780E00700C002018001070000FC000121F7E9D17>I<70F8F8F870000000000000000000 0070F8F8F87005147C930D>I<70F8F8F8700000000000000000000070F0F8F878080808 101010202040051D7C930D>I<7FFFFFE0FFFFFFF0000000000000000000000000000000 0000000000000000000000000000000000FFFFFFF07FFFFFE01C0C7D9023>61 D<0FC0307040384038E03CF03CF03C603C0038007000E000C00180018001000300020002 0002000200020002000000000000000000000007000F800F800F8007000E207D9F15>63 D<000100000003800000038000000380000007C0000007C0000007C0000009E0000009E0 000009E0000010F0000010F0000010F00000207800002078000020780000403C0000403C 0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F0002000780020007 8002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F204F00 07A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000001FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F00 001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004078004 0F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C>II<080410082010201040204020804080408040B85CFC7E FC7E7C3E381C0F0E7B9F17>II<08102020404080 8080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00000E00 000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10386720 0F83C014147E9317>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00780E00 780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E001520 7F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0007000700038 0138011C020E0C03F010147E9314>I<000380003F800003800003800003800003800003 8000038000038000038000038000038003E380061B801C07803803803803807003807003 80F00380F00380F00380F00380F00380F003807003807003803803803807801C07800E1B 8003E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F00380F0 0380FFFF80F00000F00000F000007000007000003800801800800C010007060001F80011 147F9314>I<007C00C6018F038F07060700070007000700070007000700FFF007000700 07000700070007000700070007000700070007000700070007000700070007007FF01020 809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F00780F0038 0E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC03001E060 0070C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I<0E0000 FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E3E00 0E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C001E003E001E00 1C000000000000000000000000000E007E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001F001F000E000 0000000000000000000000007007F000F000700070007000700070007000700070007000 70007000700070007000700070007000700070007000706070F060F0C061803F000C2882 9E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70000EF8000F38 000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF815207F9F18>I<0E 00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B20809F0C>I<0E1F 01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I<0E3E00 FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E001C03 803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000E07000 E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F01C00F00E00E 00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E00F00E00E00F 01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E00000E0000FF E000151D7F9319>I<03E0800619801C05803C0780380380780380700380F00380F00380 F00380F00380F00380F003807003807803803803803807801C0B800E138003E380000380 000380000380000380000380000380000380000380003FF8151D7E9318>I<0E78FE8C0F 1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00F 147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070803880188018 C018C018E030D0608F800D147E9312>I<020002000200060006000E000E003E00FFF80E 000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610031001 E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16147F93 19>III<7FC3FC0F01E00701C007018003810001C20000E40000EC 00007800003800003C00007C00004E000087000107000303800201C00601E01E01E0FF07 FE1714809318>II<3FFF380E200E 201C40384078407000E001E001C00380078007010E011E011C0338027006700EFFFE1014 7F9314>III<30307878F87C787830 300E057C9E17>127 D E /Fy 45 122 df<70F8FCFC7404040404080810102040060F7C 840E>44 DI<70F8F8F87005057C840E>I<01F000071C000C0600 1803003803803803807001C07001C07001C07001C0F001E0F001E0F001E0F001E0F001E0 F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001C07001C07001C0 7803C03803803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038003800380038003800380038007C0FFFE0F217CA018 >I<03F8000C1E001007002007804007C07807C07803C07807C03807C000078000078000 0700000F00000E0000380003F000001C00000F000007800007800003C00003C00003E020 03E07003E0F803E0F803E0F003C04003C0400780200780100F000C1C0003F00013227EA0 18>51 D<000200000600000E00000E00001E00001E00002E00004E00004E00008E00008E 00010E00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E00800E 00FFFFF8000E00000E00000E00000E00000E00000E00000E00001F0001FFF015217FA018 >I<1000801E07001FFF001FFE001FF80013E00010000010000010000010000010000010 000010F800130E001407001803801003800001C00001C00001E00001E00001E00001E070 01E0F001E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013227EA0 18>I<007E0001C1000300800601C00E03C01C03C0180180380000380000780000700000 700000F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0F001E0 F001E07001E07001E07001E03801C03801C01803801C03000C0600070C0001F00013227E A018>I<01F800060E000803001001802001802000C06000C06000C06000C07000C07801 803E01003F02001FC4000FF80003F80003FC00067F00083F80100F803007C06001C06000 E0C000E0C00060C00060C00060C000606000406000C03000801803000E0E0003F0001322 7EA018>56 D<01F000060C000C0600180700380380700380700380F001C0F001C0F001C0 F001E0F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E003E1E0 0001C00001C00001C0000380000380300300780700780600700C002018001030000FC000 13227EA018>I<0001800000018000000180000003C0000003C0000003C0000005E00000 05E000000DF0000008F0000008F0000010F800001078000010780000203C0000203C0000 203C0000401E0000401E0000401E0000800F0000800F0000FFFF00010007800100078003 0007C0020003C0020003C0040003E0040001E0040001E00C0000F00C0000F03E0001F8FF 800FFF20237EA225>65 D<0007E0100038183000E0063001C00170038000F0070000F00E 0000701E0000701C0000303C0000303C0000307C0000107800001078000010F8000000F8 000000F8000000F8000000F8000000F8000000F8000000F800000078000000780000107C 0000103C0000103C0000101C0000201E0000200E000040070000400380008001C0010000 E0020000381C000007E0001C247DA223>67 D69 DI73 D77 DI80 D82 D<03F0200C0C601802603001E07000 E0600060E00060E00060E00020E00020E00020F00000F000007800007F00003FF0001FFE 000FFF0003FF80003FC00007E00001E00000F00000F00000708000708000708000708000 70C00060C00060E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF8780780 7860078018400780084007800840078008C007800C800780048007800480078004800780 040007800000078000000780000007800000078000000780000007800000078000000780 000007800000078000000780000007800000078000000780000007800000078000000780 00000780000007800000078000000FC00003FFFF001E227EA123>I<0FE0001838003C0C 003C0E0018070000070000070000070000FF0007C7001E07003C0700780700700700F007 08F00708F00708F00F087817083C23900FC1E015157E9418>97 D<0E0000FE00001E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F00 0E61C00E80600F00300E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E 0E001E0E001C0E003C0E00380F00700C80600C41C0083F0017237FA21B>I<01FE000703 000C07801C0780380300780000700000F00000F00000F00000F00000F00000F00000F000 007000007800403800401C00800C010007060001F80012157E9416>I<0000E0000FE000 01E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E001 F8E00704E00C02E01C01E03800E07800E07000E0F000E0F000E0F000E0F000E0F000E0F0 00E0F000E07000E07800E03800E01801E00C02E0070CF001F0FE17237EA21B>I<01FC00 0707000C03801C01C03801C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000 F000007000007800203800201C00400E008007030000FC0013157F9416>I<003C00C601 8F038F030F070007000700070007000700070007000700FFF80700070007000700070007 0007000700070007000700070007000700070007000700070007807FF8102380A20F>I< 00007001F198071E180E0E181C07001C07003C07803C07803C07803C07801C07001C0700 0E0E000F1C0019F0001000001000001800001800001FFE000FFFC00FFFE03800F0600030 400018C00018C00018C000186000306000303800E00E038003FE0015217F9518>I<0E00 00FE00001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E1F800E60C00E80E00F00700F00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18237FA21B >I<1C001E003E001E001C00000000000000000000000000000000000E00FE001E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A22 7FA10E>I<01C003E003E003E001C00000000000000000000000000000000001E00FE001 E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000 E000E000E000E000E000E060E0F0C0F18061803E000B2C82A10F>I<0E0000FE00001E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E03 FC0E01F00E01C00E01800E02000E04000E08000E10000E38000EF8000F1C000E1E000E0E 000E07000E07800E03C00E01C00E01E00E00F00E00F8FFE3FE17237FA21A>I<0E00FE00 1E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B237FA20E> I<0E1FC07F00FE60E183801E807201C00F003C00E00F003C00E00E003800E00E003800E0 0E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E 003800E00E003800E00E003800E00E003800E00E003800E00E003800E0FFE3FF8FFE2715 7F942A>I<0E1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F94 1B>I<01FC000707000C01801800C03800E0700070700070F00078F00078F00078F00078 F00078F00078F000787000707800F03800E01C01C00E038007070001FC0015157F9418> I<0E1F00FE61C00E80600F00700E00380E003C0E001C0E001E0E001E0E001E0E001E0E00 1E0E001E0E001E0E003C0E003C0E00380F00700E80E00E41C00E3F000E00000E00000E00 000E00000E00000E00000E00000E00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F 0F0F060F000E000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF010 157F9413>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C 800C800CC00CC008E018D0308FC00E157E9413>I<02000200020002000600060006000E 001E003E00FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E 040E040E040708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E0070 0E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F0 06017003827800FC7F18157F941B>III121 D E /Fz 18 117 df45 D<78FCFCFCFC780000000000 000000000000000000000000000078FCFCFCFC78061F7A9E12>58 D<00000300000000000300000000000300000000000780000000000780000000000FC000 0000000FC0000000000FC00000000017E00000000013E00000000013E00000000023F000 00000021F00000000021F00000000040F80000000040F80000000040F800000000807C00 000000807C00000001807E00000001003E00000001003E00000002003F00000002001F00 000002001F00000004000F80000004000F80000004000F800000080007C00000080007C0 0000180007E000001FFFFFE000001FFFFFE00000200003F00000200001F00000200001F0 0000400001F80000400000F80000400000F800008000007C00008000007C00008000007C 00010000003E00010000003E00030000003F00030000001F00070000001F001F8000003F 80FFE00003FFFCFFE00003FFFC2E327EB132>65 D73 D77 D 80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E0780000E070 0000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F80000007C 0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC0000FFFF0000 0FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F8000000F880 00007880000078800000788000007880000078C0000078C0000070E00000F0E00000E0F0 0000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E003E001E 0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E001F001E 003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E0478003E 047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700 040F00041E001E1C003E3C003E7C003E7C0008780000F80000F80000F80000F80000F800 00F80000F80000F80000F800007800007C00007C00003C00011E00011E00020F00020700 0403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E00000 01E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E00000 01E0000001E0000001E0000001E0000001E0000001E0001F81E000F061E001C019E00780 05E00F0003E00E0003E01E0001E03C0001E03C0001E07C0001E0780001E0F80001E0F800 01E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0780001E07800 01E03C0001E03C0001E01C0001E01E0003E00E0005E0070009E0038011F000E061FF003F 81FF20327DB125>I<003F800000E0E0000380380007003C000E001E001E001E001C000F 003C000F007C000F0078000F8078000780F8000780F8000780FFFFFF80F8000000F80000 00F8000000F8000000F8000000F8000000780000007C0000003C0000003C0000801E0000 800E0001000F0002000780020001C00C0000F03000001FC000191F7E9E1D>I<0007E000 1C1000383800707C00E07C01E07C01C03803C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C000FFFFC0FFFFC003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00007E0007FFF007F FF0016327FB114>I<000000F0007F030801C1C41C0380E81C070070080F0078001E003C 001E003C003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E003C 000F007800070070000780E00009C1C000087F0000180000001800000018000000180000 00180000001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E0300000 F070000070E0000038E0000038E0000038E0000038E00000387000007070000070380000 E01C0001C00700070001C01C00003FE0001E2F7E9F21>I<07000F801F801F800F800700 000000000000000000000000000000000000000000000780FF80FF800F80078007800780 078007800780078007800780078007800780078007800780078007800780078007800780 0780078007800FC0FFF8FFF80D307EAF12>105 D<0780FE0000FF83078000FF8C03C000 0F9001E00007A001E00007A000F00007C000F00007C000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F0000FC001F800FFFC1FFF80FFFC1FFF80211F7E9E 25>110 D<0783E0FF8C18FF907C0F907C07A07C07C03807C00007C00007C00007800007 800007800007800007800007800007800007800007800007800007800007800007800007 80000780000780000780000780000780000FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E00010E00010F00010F800007E00 003FF0001FFF000FFFC003FFE0003FF00001F80000F880003C80003C80001CC0001CC000 1CE0001CE00018F00038F00030CC0060C301C080FE00161F7E9E1A>I<00400000400000 400000400000400000C00000C00000C00001C00001C00003C00007C0000FC0001FFFE0FF FFE003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C01003C01003C01003C01003C01003C01003C01003 C01001C02001E02000E0400078C0001F00142C7FAB19>I E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 340 938 a Fz(MPI:)22 b(A)g(Message-P)n(assing)e(In)n(terface)j (Standard)621 1133 y Fy(Message)16 b(P)o(assing)h(In)o(terface)e(F)l (orum)811 1259 y(Marc)o(h)h(31,)g(1994)77 1317 y(This)h(w)o(ork)f(w)o (as)h(supp)q(orted)g(in)f(part)h(b)o(y)e(ARP)l(A)h(and)h(NSF)e(under)h (gran)o(t)h(ASC-9310330,)i(the)192 1375 y(National)d(Science)f(F)l (oundation)i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q (erativ)o(e)76 1433 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o (y)e(the)h(Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1492 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 166 45 a Fx(The)14 b(Message)e(P)o(assing)h(In)o(terface)h(F)l (orum)f(\(MPIF\),)f(with)i(participation)g(from)f(o)o(v)o(er)f(40)h (organi-)75 102 y(zations,)h(has)f(b)q(een)i(meeting)f(since)g(Jan)o (uary)g(1993)e(to)h(discuss)h(and)g(de\014ne)h(a)e(set)g(of)g(library)i (in)o(terface)75 158 y(standards)h(for)h(message)f(passing.)25 b(MPIF)16 b(is)i(not)e(sanctioned)i(or)e(supp)q(orted)h(b)o(y)g(an)o(y) g(o\016cial)g(stan-)75 214 y(dards)e(organization.)166 271 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o(terface,)g (simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g(used)75 327 y(standard)h(for)g(writing)h(message-passing)g(programs.)29 b(As)19 b(suc)o(h)g(the)g(in)o(terface)g(should)g(establish)h(a)75 384 y(practical,)c(p)q(ortable,)f(e\016cien)o(t,)g(and)h(\015exible)h (standard)e(for)g(message)f(passing.)166 440 y(This)20 b(is)g(the)g(\014nal)h(rep)q(ort,)f(V)l(ersion)g(1.0,)g(of)f(the)h (Message)f(P)o(assing)h(In)o(terface)f(F)l(orum.)33 b(This)75 497 y(do)q(cumen)o(t)18 b(con)o(tains)g(all)h(the)f(tec)o(hnical)i (features)d(prop)q(osed)h(for)g(the)g(in)o(terface.)28 b(This)19 b(cop)o(y)e(of)h(the)75 553 y(draft)d(w)o(as)f(pro)q(cessed)i (b)o(y)f(L)554 547 y Fw(a)575 553 y Fx(T)600 567 y(E)626 553 y(X)g(on)g(Marc)o(h)f(31,)h(1994.)166 610 y(Please)i(send)g(commen) o(ts)f(on)g Fv(MPI)g Fx(to)g Fu(mpi-comments@cs.utk.edu)p Fx(.)21 b(Y)l(our)16 b(commen)o(t)g(will)i(b)q(e)75 666 y(forw)o(arded)c(to)h(MPIF)g(committee)g(mem)o(b)q(ers)g(who)g(will)i (attempt)d(to)h(resp)q(ond.)179 1922 y(c)166 1923 y Ft(\015)p Fx(1993,)f(1994)h(Univ)o(ersit)o(y)h(of)f(T)l(ennessee,)i(Kno)o (xville,)g(T)l(ennessee.)22 b(P)o(ermission)16 b(to)f(cop)o(y)g(with-) 75 1979 y(out)d(fee)g(all)h(or)f(part)g(of)f(this)i(material)g(is)f (gran)o(ted,)g(pro)o(vided)h(the)f(Univ)o(ersit)o(y)h(of)f(T)l (ennessee)h(cop)o(yrigh)o(t)75 2035 y(notice)i(and)f(the)g(title)h(of)e (this)i(do)q(cumen)o(t)f(app)q(ear,)g(and)h(notice)f(is)h(giv)o(en)g (that)e(cop)o(ying)h(is)h(b)o(y)f(p)q(ermis-)75 2092 y(sion)i(of)e(the)i(Univ)o(ersit)o(y)g(of)e(T)l(ennessee.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 3 2 2 bop 75 377 a Fr(Con)m(ten)m(ts)75 645 y Fq(Ac)o(kno)o(wledgmen)o (ts)1346 b(vi)75 747 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(1)143 804 y Fx(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(1)143 860 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(2)143 916 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(3)143 973 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(3)143 1029 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b Fp(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b Fx(3)143 1086 y(1.6)46 b(Organization)16 b(of)e(this)i(Do)q(cumen)o(t)34 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(4)75 1188 y Fq(2)42 b(MPI)17 b(T)l(erms)f(and)i(Con)o(v)o(en)o (tions)1042 b(6)143 1244 y Fx(2.1)46 b(Do)q(cumen)o(t)15 b(Notation)44 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b Fx(6)143 1301 y(2.2)46 b(Pro)q(cedure)16 b(Sp)q(eci\014cation) 44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(6)143 1357 y(2.3)46 b(Seman)o(tic)15 b(T)l(erms)g Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 91 b Fx(7)143 1414 y(2.4)46 b(Data)14 b(T)o(yp)q(es)30 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b Fx(8)248 1470 y(2.4.1)50 b(Opaque)16 b(ob)s(jects)21 b Fp(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(8)248 1526 y(2.4.2)50 b(Arra)o(y)14 b(argumen)o(ts)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(9)248 1583 y(2.4.3)50 b(State)14 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(9)248 1639 y(2.4.4)50 b(Named)15 b(constan)o(ts)23 b Fp(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(9)248 1696 y(2.4.5)50 b(Choice)20 b Fp(:)i(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(9)248 1752 y(2.4.6)50 b(Addresses)28 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(10)143 1809 y(2.5)46 b(Language)15 b(Binding)45 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b Fx(10)248 1865 y(2.5.1)50 b(F)l(ortran)14 b(77)h(Binding)i(Issues)30 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(10)248 1922 y(2.5.2)50 b(C)15 b(Binding)i(Issues)31 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(11)143 1978 y(2.6)46 b(Pro)q(cesses)40 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fx(11)143 2035 y(2.7)46 b(Error)14 b(Handling)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fx(12)143 2091 y(2.8)46 b(Implemen)o(tation)16 b(issues)36 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(13)248 2147 y(2.8.1)50 b(Indep)q(endence)19 b(of)14 b(Basic)i(Run)o(time)g(Routines)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(13)248 2204 y(2.8.2)50 b(In)o(teraction)16 b(with)f(signals)h(in)g(POSIX)42 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)69 b Fx(14)75 2306 y Fq(3)42 b(P)o(oin)o(t-to-P)o(oin)o(t)19 b(Comm)o(unication)975 b(15)143 2362 y Fx(3.1)46 b(In)o(tro)q(duction) 15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b Fx(15)143 2419 y(3.2)46 b(Blo)q(c)o(king)16 b(Send)g(and)g(Receiv)o(e)g(Op)q(erations)24 b Fp(:)e(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(16)248 2475 y(3.2.1)50 b(Blo)q(c)o(king)17 b(send)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(16)248 2532 y(3.2.2)50 b(Message)15 b(data)26 b Fp(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(16)248 2588 y(3.2.3)50 b(Message)15 b(en)o(v)o(elop)q(e)j Fp(:)k(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(18)248 2645 y(3.2.4)50 b(Blo)q(c)o(king)17 b(receiv)o(e)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(19)248 2701 y(3.2.5)50 b(Return)16 b(status)j Fp(:)k(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(20)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 4 3 3 bop 143 45 a Fx(3.3)46 b(Data)14 b(t)o(yp)q(e)h(matc)o(hing)g(and)h (data)e(con)o(v)o(ersion)j Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(21)248 102 y(3.3.1)50 b(T)o(yp)q(e)16 b(matc)o(hing)f(rules)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(21)248 158 y(3.3.2)50 b(Data)14 b(con)o(v)o(ersion)46 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(24)143 214 y(3.4)46 b(Comm)o(unication)15 b(Mo)q(des)i Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(25)143 271 y(3.5)46 b(Seman)o(tics)15 b(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(29)143 327 y(3.6)46 b(Bu\013er)15 b(allo)q(cation)h(and)g(usage)38 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(33)248 384 y(3.6.1)50 b(Mo)q(del)16 b(implemen)o(tation)h(of)d(bu\013ered)i(mo)q (de)27 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)69 b Fx(34)143 440 y(3.7)46 b(Non)o(blo)q(c)o(king)16 b(comm)o(unication)j Fp(:)j(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(34)248 497 y(3.7.1)50 b(Comm)o(unication)16 b(Ob)s(jects)23 b Fp(:)g(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(36)248 553 y(3.7.2)50 b(Comm)o(unication)16 b(initiation)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fx(36)248 610 y(3.7.3)50 b(Comm)o(unication)16 b(Completion)i Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(39)248 666 y(3.7.4)50 b(Seman)o(tics)16 b(of)f(Non)o(blo)q(c)o(king)h(Comm)o(unications)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(42)248 723 y(3.7.5)50 b(Multiple)17 b(Completions)41 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(43)143 779 y(3.8)46 b(Prob)q(e)15 b(and)g(Cancel)g Fp(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(48)143 835 y(3.9)46 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(requests)33 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(52)143 892 y(3.10)23 b(Send-receiv)o(e)d Fp(:)i(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(56)143 948 y(3.11)23 b(Null)16 b(pro)q(cesses)i Fp(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(58)143 1005 y(3.12)23 b(Deriv)o(ed)15 b(datat)o(yp)q(es)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(59)248 1061 y(3.12.1)27 b(Datat)o(yp)q(e)14 b(constructors)27 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fx(60)248 1118 y(3.12.2)27 b(Address)16 b(and)f(exten)o(t)g(functions)32 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(67)248 1174 y(3.12.3)27 b(Lo)o(w)o(er-b)q(ound)16 b(and)f(upp)q(er-b)q(ound)j(mark)o(ers)33 b Fp(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(69)248 1231 y(3.12.4)27 b(Commit)15 b(and)g(free)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(70)248 1287 y(3.12.5)27 b(Use)16 b(of)e(general)i(datat)o(yp)q(es)f(in)h(comm) o(unication)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)69 b Fx(71)248 1344 y(3.12.6)27 b(Correct)15 b(use)g(of)g (addresses)26 b Fp(:)d(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(74)248 1400 y(3.12.7)27 b(Examples)33 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(75)143 1456 y(3.13)23 b(P)o(ac)o(k)14 b(and)i(unpac)o(k)29 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(83)75 1558 y Fq(4)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(90)143 1615 y Fx(4.1)46 b(In)o(tro)q(duction)16 b(and)f(Ov)o(erview) 44 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(90)143 1671 y(4.2)46 b(Comm)o(unicator)14 b(argumen)o(t)e Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(93)143 1728 y(4.3)46 b(Barrier)15 b(sync)o(hronization)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(93)143 1784 y(4.4)46 b(Broadcast)27 b Fp(:)c(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(93)248 1841 y(4.4.1)50 b(Example)16 b(using)g Fv(MPI)p 786 1841 14 2 v 16 w(BCAST)26 b Fp(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(94)143 1897 y(4.5)46 b(Gather)17 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(94)248 1954 y(4.5.1)50 b(Examples)16 b(using)g Fv(MPI)p 804 1954 V 16 w(GA)l(THER)p Fx(,)g Fv(MPI)p 1106 1954 V 16 w(GA)l(THERV)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fx(96)143 2010 y(4.6)46 b(Scatter)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fx(103)248 2066 y(4.6.1)k(Examples)16 b(using)g Fv(MPI)p 804 2066 V 16 w(SCA)l(TTER)p Fx(,)g Fv(MPI)p 1129 2066 V 15 w(SCA)l(TTERV)25 b Fp(:)e(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b Fx(105)143 2123 y(4.7)g(Gather-to-all)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b Fx(107)248 2179 y(4.7.1)k(Examples)16 b(using)g Fv(MPI)p 804 2179 V 16 w(ALLGA)l(THER)p Fx(,)f Fv(MPI)p 1185 2179 V 16 w(ALLGA)l(THERV)i Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(109)143 2236 y(4.8)g(All-to-All)17 b(Scatter/Gather)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(109)143 2292 y(4.9)g(Global)16 b(Reduction)g(Op)q(erations)34 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(111)248 2349 y(4.9.1)k(Reduce)45 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(111)248 2405 y(4.9.2)k(Prede\014ned) 17 b(reduce)f(op)q(erations)37 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(112)248 2462 y(4.9.3)k(MINLOC)16 b(and)f(MAXLOC)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(114)248 2518 y(4.9.4)k (User-De\014ned)16 b(Op)q(erations)h Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(118)248 2575 y(4.9.5)k(All-Reduce)43 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(122)143 2631 y(4.10)23 b(Reduce-Scatter)33 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(123)143 2687 y(4.11)23 b(Scan)28 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(124)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 5 4 4 bop 248 45 a Fx(4.11.1)27 b(Example)16 b(using)g Fv(MPI)p 786 45 14 2 v 16 w(SCAN)k Fp(:)i(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(124)143 102 y(4.12)23 b(Correctness)32 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(126)75 203 y Fq(5)c(Groups,)17 b(Con)o(texts,)f(Comm)o (unicators,)g(and)i(Cac)o(hing)543 b(130)143 260 y Fx(5.1)46 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(130)248 316 y(5.1.1)k(F)l (eatures)15 b(Needed)h(to)f(Supp)q(ort)h(Libraries)40 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)46 b Fx(130)248 373 y(5.1.2)k Fv(MPI)p Fx('s)14 b(Supp)q(ort)i(for)f (Libraries)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(131)143 429 y(5.2)g(Basic)16 b(Concepts)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(133)248 486 y(5.2.1)k(Groups)43 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)46 b Fx(133)248 542 y(5.2.2)k(Con)o(texts)c Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(133)248 599 y(5.2.3)k(In)o(tra-Comm)o(unicators)44 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(134)248 655 y(5.2.4)k(Prede\014ned)17 b(In)o(tra-Comm)o(unicators)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fx(134)143 712 y(5.3)g(Group)15 b(Managemen)o(t)41 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(135)248 768 y(5.3.1)k(Group)15 b(Accessors)34 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(135)248 824 y(5.3.2)k(Group)15 b(Constructors)37 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(136)248 881 y(5.3.3)k(Group)15 b(Destructors)27 b Fp(:)c(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fx(140)143 937 y(5.4)g(Comm)o(unicator)14 b(Managemen)o(t)i Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(141)248 994 y(5.4.1)k(Comm)o(unicator)15 b(Accessors)43 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(141)248 1050 y(5.4.2)k(Comm)o (unicator)15 b(Constructors)10 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(142)248 1107 y(5.4.3)k(Comm)o(unicator)15 b(Destructors)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(145)143 1163 y(5.5)g(Motiv)m(ating) 15 b(Examples)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b Fx(146)248 1220 y(5.5.1)k(Curren)o(t)15 b(Practice)g(#1)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(146)248 1276 y(5.5.2)k(Curren)o(t)15 b(Practice)g(#2)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(147)248 1333 y(5.5.3)k(\(Appro)o(ximate\))15 b(Curren)o(t)f(Practice)i(#3)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fx(147)248 1389 y(5.5.4)k(Example)16 b(#4)44 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(148)248 1445 y(5.5.5)k(Library)16 b(Example)g(#1)22 b Fp(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(149)248 1502 y(5.5.6)k(Library)16 b(Example)g(#2)22 b Fp(:)g(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fx(151)143 1558 y(5.6)g(In)o(ter-Comm)o(unication) 14 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(153)248 1615 y(5.6.1)k(In)o(ter-comm)o(unicator)15 b(Accessors)h Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(154)248 1671 y(5.6.2)k(In)o(ter-comm)o(unicator)15 b(Op)q(erations)22 b Fp(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fx(156)248 1728 y(5.6.3)k(In)o(ter-Comm)o (unication)16 b(Examples)g Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(158)143 1784 y(5.7)g(Cac)o(hing)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(165)248 1841 y(5.7.1)k(F)l(unctionalit)o(y)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(165)248 1897 y(5.7.2)k(A)o (ttributes)15 b(Example)43 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(169)143 1954 y(5.8)g(F)l(ormalizing)16 b(the)f(Lo)q(osely)h (Sync)o(hronous)g(Mo)q(del)g(\(Usage,)e(Safet)o(y\))j Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(171)248 2010 y(5.8.1)k(Basic)16 b(Statemen)o(ts)22 b Fp(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)46 b Fx(171)248 2066 y(5.8.2)k(Mo)q(dels)16 b(of)f(Execution)29 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(172)75 2168 y Fq(6)c(Pro)q(cess)17 b(T)l(op)q(ologies)1224 b(174)143 2225 y Fx(6.1)46 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b Fx(174)143 2281 y(6.2)g(Virtual)16 b(T)l(op)q(ologies)37 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(175)143 2338 y(6.3)g(Em)o(b)q(edding)16 b(in)g Fv(MPI)29 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(175)143 2394 y(6.4)g(Ov)o(erview)16 b(of)e(the)i(F)l(unctions)22 b Fp(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(176)143 2451 y(6.5)g(T)l(op)q(ology)15 b(Constructors)20 b Fp(:)i(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(177)248 2507 y(6.5.1)k(Cartesian)15 b(Constructor)27 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b Fx(177)248 2564 y(6.5.2)k(Cartesian)15 b(Con)o(v)o (enience)h(F)l(unction:)21 b Fv(MPI)p 1152 2564 V 16 w(DIMS)p 1279 2564 V 16 w(CREA)l(TE)39 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fx(177)248 2620 y(6.5.3)k(General)16 b(\(Graph\))e (Constructor)27 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(178)248 2676 y(6.5.4)k(T)l(op)q(ology)15 b(inquiry)i(functions)42 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(180)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 6 5 5 bop 248 45 a Fx(6.5.5)50 b(Cartesian)15 b(Shift)h(Co)q(ordinates)23 b Fp(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(184)248 102 y(6.5.6)k(P)o(artitioning) 16 b(of)e(Cartesian)h(structures)45 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(185)248 158 y(6.5.7)k(Lo)o(w-lev)o(el)17 b(top)q(ology)d(functions)45 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b Fx(185)143 214 y(6.6)g(An)15 b(Application)i(Example)g Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b Fx(187)75 316 y Fq(7)c(MPI)17 b(En)o(vironmen)o(tal)g (Managemen)o(t)883 b(189)143 373 y Fx(7.1)46 b(Implemen)o(tation)16 b(information)22 b Fp(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(189)248 429 y(7.1.1)k(En)o(vironmen)o(tal)16 b(Inquiries)28 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(189)143 486 y(7.2)g(Error)14 b(handling)41 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(191)143 542 y(7.3)g(Error)14 b(co)q(des)i(and)f(classes)e Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fx(194)143 599 y(7.4)g(Timers)18 b Fp(:)k(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(195)143 655 y(7.5)g(Startup)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(195)75 757 y Fq(8)c(Pro\014ling)18 b(In)o(terface)1238 b(198)143 813 y Fx(8.1)46 b(Requiremen)o(ts)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(198)143 870 y(8.2)g(Discussion)21 b Fp(:)i(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(198)143 926 y(8.3)g(Logic)16 b(of)e(the)i(design)29 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(199)248 983 y(8.3.1)k(Miscellaneous)17 b(con)o(trol)e(of)g (pro\014ling)28 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(199)143 1039 y(8.4)g(Examples)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fx(200)248 1096 y(8.4.1)k(Pro\014ler)16 b(implemen)o(tation)35 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(200)248 1152 y(8.4.2)k(MPI)15 b(library)h(implemen)o(tation)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(201)248 1209 y(8.4.3)k (Complications)e Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(202)143 1265 y(8.5)g(Multiple)17 b(lev)o(els)f(of)f(in)o (terception)25 b Fp(:)d(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(203)75 1367 y Fq(Bibliograph)o(y)1430 b(204)75 1469 y(A)28 b(Language)19 b(Binding)1242 b(207)143 1525 y Fx(A.1)35 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(207)143 1582 y(A.2)35 b(De\014ned)16 b(Constan)o(ts)e(for)g(C)h(and)h(F)l (ortran)21 b Fp(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(207)143 1638 y(A.3)35 b(C)15 b(bindings)i(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o (unication)21 b Fp(:)i(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fx(210)143 1695 y(A.4)35 b(C)15 b(Bindings)i(for)d (Collectiv)o(e)j(Comm)o(unication)34 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(214)143 1751 y(A.5)35 b(C)15 b(Bindings)i(for)d(Groups,)h(Con)o (texts,)f(and)h(Comm)o(unicators)26 b Fp(:)c(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fx(215)143 1808 y(A.6)35 b(C)15 b(Bindings)i(for)d(Pro)q(cess)h(T)l(op)q(ologies)43 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(216)143 1864 y(A.7)35 b(C)15 b(bindings)i(for)d(En)o(vironmen)o(tal)i(Inquiry)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b Fx(217)143 1921 y(A.8)35 b(C)15 b(Bindings)i(for)d(Pro\014ling)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fx(217)143 1977 y(A.9)35 b(F)l(ortran)14 b(Bindings)j(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o(unication)39 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(217)143 2033 y(A.10)12 b(F)l(ortran)i(Bindings)j(for)d(Collectiv)o (e)j(Comm)o(unication)23 b Fp(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b Fx(221)143 2090 y(A.11)12 b(F)l(ortran)i(Bindings)j(for)d(Groups,)h(Con)o(texts,)f(and)h(Comm)o (unicators)g Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(223)143 2146 y(A.12)12 b(F)l(ortran)i(Bindings)j(for)d(Pro)q(cess) h(T)l(op)q(ologies)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(224)143 2203 y(A.13)12 b(F)l(ortran)i(Bindings)j(for)d(En)o(vironmen)o(tal)i (Inquiry)24 b Fp(:)e(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fx(225)143 2259 y(A.14)12 b(F)l(ortran)i(Bindings) j(for)d(Pro\014ling)24 b Fp(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(226)75 2361 y Fq(MPI)17 b(F)l(unction)h(Index)1252 b(227)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 7 6 6 bop 75 75 a Fr(Ac)m(kno)m(wledgmen)m(ts)166 282 y Fx(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f(w)o(as)e(carried)i(out)f (b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o(as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d) h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P)o(assing)g(In)o (terface)75 395 y(\()p Fv(MPI)p Fx(\),)f(man)o(y)g(p)q(eople)j(serv)o (ed)e(in)h(p)q(ositions)g(of)f(resp)q(onsibilit)o(y)j(and)d(are)g (listed)i(b)q(elo)o(w.)143 497 y Ft(\017)23 b Fx(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o(er,)g(Con)o(v)o(eners)g(and)g (Meeting)h(Chairs)143 589 y Ft(\017)23 b Fx(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes)143 682 y Ft(\017)23 b Fx(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i(Lusk,)f(P)o(oin)o (t-to-P)o(oin)o(t)f(Comm)o(unications)143 774 y Ft(\017)23 b Fx(Al)16 b(Geist,)e(Marc)h(Snir,)h(Stev)o(e)f(Otto,)f(Collectiv)o(e)j (Comm)o(unications)143 867 y Ft(\017)23 b Fx(Stev)o(e)15 b(Otto,)f(Editor)143 959 y Ft(\017)23 b Fx(Rolf)15 b(Hemp)q(el,)i(Pro)q (cess)e(T)l(op)q(ologies)143 1052 y Ft(\017)23 b Fx(Ewing)15 b(Lusk,)h(Language)f(Binding)143 1144 y Ft(\017)23 b Fx(William)17 b(Gropp,)d(En)o(vironmen)o(tal)i(Managemen)o(t)143 1237 y Ft(\017)23 b Fx(James)15 b(Co)o(wnie,)g(Pro\014ling)143 1330 y Ft(\017)23 b Fx(T)l(on)o(y)16 b(Skjellum,)j(Lyndon)f(Clark)o(e,) f(Marc)g(Snir,)h(Ric)o(hard)g(Little\014eld,)h(Mark)e(Sears,)g(Groups,) 189 1386 y(Con)o(texts,)c(and)j(Comm)o(unicators)143 1479 y Ft(\017)23 b Fx(Stev)o(en)15 b(Huss-Lederman,)h(Initial)h (Implemen)o(tation)f(Subset)166 1580 y(The)23 b(follo)o(wing)g(list)g (includes)i(some)d(of)g(the)g(activ)o(e)h(participan)o(ts)g(in)g(the)g Fv(MPI)f Fx(pro)q(cess)g(not)75 1637 y(men)o(tioned)16 b(ab)q(o)o(v)o(e.)100 1771 y(Ed)f(Anderson)114 b(Rob)q(ert)15 b(Babb)135 b(Jo)q(e)16 b(Baron)186 b(Eric)16 b(Barszcz)100 1827 y(Scott)f(Berryman)56 b(Rob)15 b(Bjornson)119 b(Nathan)15 b(Doss)133 b(Anne)16 b(Elster)100 1883 y(Jim)g(F)l(eeney)149 b(Vince)16 b(F)l(ernando)81 b(Sam)15 b(Fineb)q(erg)115 b(Jon)16 b(Flo)o(w)o(er)100 1940 y(Daniel)g(F)l(ry)o(e)143 b(Ian)15 b(Glendinning)74 b(Adam)15 b(Green)o(b)q(erg)50 b(Rob)q(ert)16 b(Harrison)100 1996 y(Leslie)h(Hart)153 b(T)l(om)15 b(Haupt)163 b(Don)15 b(Heller)176 b(T)l(om)15 b(Henderson)100 2053 y(Alex)h(Ho)210 b(C.T.)14 b(Ho)o(w)o(ard)g(Ho)62 b(Gary)14 b(Ho)o(w)o(ell)143 b(John)16 b(Kap)q(enga)100 2109 y(James)f(Kohl)141 b(Susan)15 b(Krauss)125 b(Bob)15 b(Leary)182 b(Arth)o(ur)15 b(Maccab)q(e)100 2166 y(P)o(eter)g(Madams)83 b(Alan)16 b(Main)o(w)o(aring)49 b(Oliv)o(er)17 b(McBry)o(an)70 b(Phil)17 b(McKinley)100 2222 y(Charles)e(Mosher)63 b(Dan)15 b(Nessett)149 b(P)o(eter)15 b(P)o(ac)o(heco)109 b(Ho)o(w)o(ard)14 b(P)o(almer)100 2279 y(P)o(aul)h(Pierce)147 b(Sanja)o(y)14 b(Rank)m(a)121 b(P)o(eter)15 b(Rigsb)q(ee)117 b(Arc)o(h)15 b(Robison)100 2335 y(Eric)o(h)h(Sc)o(hikuta)81 b(Am)o(buj)15 b(Singh)133 b(Alan)16 b(Sussman)108 b(Rob)q(ert)16 b(T)l(omlinson)100 2392 y(Rob)q(ert)f(G.)g(V)l(oigt)50 b(Dennis)16 b(W)l(eeks)117 b(Stephen)16 b(Wheat)83 b(Stev)o(e)15 b(Zenith)166 2534 y(The)e(Univ)o(ersit)o(y)h(of)f(T)l(ennessee)i(and)e(Oak)g(Ridge)i (National)e(Lab)q(oratory)g(made)g(the)g(draft)g(a)o(v)m(ail-)75 2591 y(able)j(b)o(y)f(anon)o(ymous)f(FTP)h(mail)g(serv)o(ers)g(and)g(w) o(ere)g(instrumen)o(tal)g(in)h(distributing)h(the)e(do)q(cumen)o(t.)166 2647 y Fv(MPI)c Fx(op)q(erated)h(on)f(a)h(v)o(ery)f(tigh)o(t)h(budget)g (\(in)g(realit)o(y)l(,)g(it)g(had)g(no)g(budget)g(when)g(the)g(\014rst) f(meeting)75 2704 y(w)o(as)i(announced\).)20 b(ARP)l(A)14 b(and)f(NSF)h(ha)o(v)o(e)e(supp)q(orted)i(researc)o(h)g(at)e(v)m (arious)i(institutions)h(that)d(ha)o(v)o(e)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 8 7 7 bop 75 45 a Fx(made)17 b(a)f(con)o(tribution)h(to)o(w)o(ards)e(tra) o(v)o(el)h(for)g(the)g(U.S.)g(academics.)25 b(Supp)q(ort)17 b(for)f(sev)o(eral)h(Europ)q(ean)75 102 y(participan)o(ts)f(w)o(as)e (pro)o(vided)i(b)o(y)f(ESPRIT.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 1 9 1 8 bop 75 356 a Fo(Chapter)34 b(1)75 564 y Fr(In)m(tro)s(duction)41 b(to)g(MPI)75 805 y Fn(1.1)59 b(Overview)19 b(and)g(Goals)75 906 y Fx(Message)11 b(passing)i(is)f(a)g(paradigm)g(used)g(widely)i(on) d(certain)i(classes)f(of)g(parallel)h(mac)o(hines,)g(esp)q(ecially)75 963 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1019 y(pro)q(cesses)j(comm)o(unicating)h(through)e(messages)h(is)g (w)o(ell)h(understo)q(o)q(d.)34 b(Ov)o(er)20 b(the)g(last)g(ten)g(y)o (ears,)75 1076 y(substan)o(tial)11 b(progress)f(has)g(b)q(een)h(made)g (in)g(casting)f(signi\014can)o(t)i(applications)g(in)f(this)g (paradigm.)18 b(Eac)o(h)75 1132 y(v)o(endor)13 b(has)g(implemen)o(ted)i (its)e(o)o(wn)g(v)m(arian)o(t.)19 b(More)13 b(recen)o(tly)l(,)h(sev)o (eral)f(systems)g(ha)o(v)o(e)f(demonstrated)75 1188 y(that)k(a)h (message)f(passing)h(system)g(can)g(b)q(e)g(e\016cien)o(tly)h(and)f(p)q (ortably)h(implemen)o(ted.)26 b(It)17 b(is)g(th)o(us)g(an)75 1245 y(appropriate)c(time)g(to)f(try)h(to)f(de\014ne)i(b)q(oth)f(the)g (syn)o(tax)f(and)h(seman)o(tics)g(of)g(a)g(core)f(of)h(library)h (routines)75 1301 y(that)i(will)i(b)q(e)f(useful)h(to)e(a)g(wide)h (range)f(of)h(users)f(and)h(e\016cien)o(tly)h(implemen)o(table)g(on)f (a)f(wide)h(range)75 1358 y(of)e(computers.)166 1414 y(In)c(designing)h Fv(MPI)e Fx(w)o(e)h(ha)o(v)o(e)f(sough)o(t)g(to)g (mak)o(e)g(use)h(of)f(the)h(most)e(attractiv)o(e)h(features)g(of)g(a)h (n)o(um)o(b)q(er)75 1471 y(of)16 b(existing)i(message)e(passing)h (systems,)g(rather)f(than)g(selecting)j(one)d(of)h(them)f(and)h (adopting)g(it)g(as)75 1527 y(the)g(standard.)26 b(Th)o(us,)18 b Fv(MPI)e Fx(has)h(b)q(een)i(strongly)e(in\015uenced)j(b)o(y)d(w)o (ork)f(at)h(the)g(IBM)h(T.)f(J.)g(W)l(atson)75 1584 y(Researc)o(h)e (Cen)o(ter)g([1)o(,)g(2)o(],)f(In)o(tel's)i(NX/2)e([23)o(],)g(Express)h ([22)o(],)f(nCUBE's)h(V)l(ertex)g([21)o(],)f(p4)h([7)o(,)g(6],)f(and)75 1640 y(P)l(ARMA)o(CS)h([5,)f(8].)19 b(Other)c(imp)q(ortan)o(t)g(con)o (tributions)h(ha)o(v)o(e)e(come)h(from)f(Zip)q(co)q(de)j([24)o(,)d (25],)g(Chimp)75 1697 y([14)o(,)h(15)o(],)g(PVM)g([4)o(,)g(11)o(],)g (Chameleon)g([19)o(],)g(and)g(PICL)h([18)o(].)166 1753 y(The)e Fv(MPI)f Fx(standardization)h(e\013ort)e(in)o(v)o(olv)o(ed)j (ab)q(out)e(60)g(p)q(eople)i(from)e(40)g(organizations)h(mainly)75 1810 y(from)j(the)g(United)i(States)e(and)h(Europ)q(e.)27 b(Most)16 b(of)h(the)h(ma)s(jor)e(v)o(endors)h(of)g(concurren)o(t)h (computers)75 1866 y(w)o(ere)11 b(in)o(v)o(olv)o(ed)h(in)h Fv(MPI)p Fx(,)d(along)h(with)h(researc)o(hers)f(from)g(univ)o (ersities,)j(go)o(v)o(ernmen)o(t)c(lab)q(oratories,)i(and)75 1922 y(industry)l(.)21 b(The)14 b(standardization)h(pro)q(cess)f(b)q (egan)h(with)f(the)h(W)l(orkshop)f(on)g(Standards)g(for)g(Message)75 1979 y(P)o(assing)j(in)i(a)e(Distributed)h(Memory)f(En)o(vironmen)o(t,) h(sp)q(onsored)g(b)o(y)f(the)h(Cen)o(ter)f(for)g(Researc)o(h)h(on)75 2035 y(P)o(arallel)11 b(Computing,)g(held)h(April)f(29-30,)f(1992,)g (in)h(Williamsburg,)h(Virginia)g([29)o(].)18 b(A)o(t)9 b(this)i(w)o(orkshop)75 2092 y(the)17 b(basic)h(features)f(essen)o (tial)h(to)f(a)g(standard)g(message)g(passing)g(in)o(terface)h(w)o(ere) f(discussed,)i(and)e(a)75 2148 y(w)o(orking)e(group)g(established)i(to) d(con)o(tin)o(ue)i(the)f(standardization)h(pro)q(cess.)166 2205 y(A)e(preliminary)i(draft)e(prop)q(osal,)g(kno)o(wn)g(as)g(MPI1,)g (w)o(as)f(put)h(forw)o(ard)f(b)o(y)h(Dongarra,)f(Hemp)q(el,)75 2261 y(Hey)l(,)24 b(and)f(W)l(alk)o(er)g(in)g(No)o(v)o(em)o(b)q(er)f (1992,)h(and)f(a)h(revised)g(v)o(ersion)g(w)o(as)e(completed)j(in)f(F)l (ebruary)75 2318 y(1993)e([12)o(].)39 b(MPI1)22 b(em)o(b)q(o)q(died)h (the)f(main)g(features)g(that)f(w)o(ere)g(iden)o(ti\014ed)j(at)d(the)h (Williamsburg)75 2374 y(w)o(orkshop)d(as)h(b)q(eing)h(necessary)g(in)g (a)f(message)f(passing)i(standard.)34 b(Since)22 b(MPI1)d(w)o(as)h (primarily)75 2431 y(in)o(tended)12 b(to)d(promote)h(discussion)i(and)e (\\get)g(the)h(ball)g(rolling,")h(it)f(fo)q(cused)g(mainly)g(on)f(p)q (oin)o(t-to-p)q(oin)o(t)75 2487 y(comm)o(unications.)33 b(MPI1)20 b(brough)o(t)f(to)g(the)g(forefron)o(t)f(a)h(n)o(um)o(b)q(er) h(of)f(imp)q(ortan)o(t)h(standardization)75 2543 y(issues,)c(but)f(did) h(not)f(include)i(an)o(y)e(collectiv)o(e)i(comm)o(unication)f(routines) f(and)h(w)o(as)e(not)h(thread-safe.)166 2600 y(In)j(No)o(v)o(em)o(b)q (er)f(1992,)g(a)g(meeting)h(of)f(the)h Fv(MPI)f Fx(w)o(orking)g(group)h (w)o(as)e(held)j(in)g(Minneap)q(olis,)h(at)75 2656 y(whic)o(h)15 b(it)g(w)o(as)e(decided)k(to)c(place)j(the)e(standardization)h(pro)q (cess)g(on)f(a)g(more)g(formal)g(fo)q(oting,)g(and)h(to)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 10 2 9 bop 75 -100 a Fx(2)903 b Fm(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)75 45 y Fx(generally)i(adopt)f(the)g(pro)q(cedures)i(and)e (organization)g(of)g(the)g(High)h(P)o(erformance)f(F)l(ortran)f(F)l (orum.)75 102 y(Sub)q(committees)i(w)o(ere)e(formed)g(for)g(the)h(ma)s (jor)e(comp)q(onen)o(t)i(areas)f(of)g(the)h(standard,)f(and)g(an)h (email)75 158 y(discussion)23 b(service)g(established)g(for)e(eac)o(h.) 39 b(In)22 b(addition,)i(the)d(goal)h(of)f(pro)q(ducing)i(a)e(draft)g Fv(MPI)75 214 y Fx(standard)13 b(b)o(y)g(the)g(F)l(all)h(of)e(1993)g(w) o(as)g(set.)19 b(T)l(o)13 b(ac)o(hiev)o(e)h(this)f(goal)g(the)g Fv(MPI)g Fx(w)o(orking)g(group)f(met)h(ev)o(ery)75 271 y(6)k(w)o(eeks)g(for)f(t)o(w)o(o)g(da)o(ys)h(throughout)f(the)h (\014rst)g(9)g(mon)o(ths)f(of)h(1993,)f(and)h(presen)o(ted)h(the)f (draft)g Fv(MPI)75 327 y Fx(standard)d(at)g(the)g(Sup)q(ercomputing)i (93)e(conference)h(in)g(No)o(v)o(em)o(b)q(er)f(1993.)19 b(These)14 b(meetings)h(and)g(the)75 384 y(email)h(discussion)g (together)f(constituted)g(the)g Fv(MPI)f Fx(F)l(orum,)g(mem)o(b)q (ership)i(of)f(whic)o(h)h(has)e(b)q(een)i(op)q(en)75 440 y(to)f(all)h(mem)o(b)q(ers)f(of)g(the)g(high)h(p)q(erformance)f (computing)h(comm)o(unit)o(y)l(.)166 497 y(The)j(main)f(adv)m(an)o (tages)g(of)g(establishing)j(a)d(message-passing)g(standard)g(are)g(p)q (ortabilit)o(y)i(and)75 553 y(ease-of-use.)g(In)14 b(a)g(distributed)h (memory)e(comm)o(unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f (higher)h(lev)o(el)75 610 y(routines)c(and/or)g(abstractions)f(are)h (build)i(up)q(on)e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i (routines)f(the)g(b)q(ene\014ts)75 666 y(of)22 b(standardization)h(are) f(particularly)h(apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g(message)75 723 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop)q(osed)g(here,)h(pro)o(vides)f (v)o(endors)h(with)f(a)g(clearly)h(de\014ned)h(base)75 779 y(set)10 b(of)g(routines)g(that)g(they)g(can)h(implemen)o(t)g (e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o(vide)f(hardw)o (are)g(supp)q(ort)75 835 y(for,)k(thereb)o(y)h(enhancing)i(scalabilit)o (y)l(.)166 892 y(The)h(goal)g(of)g(the)g(Message)g(P)o(assing)g(In)o (terface)g(simply)i(stated)d(is)i(to)f(dev)o(elop)h(a)f(widely)h(used) 75 948 y(standard)f(for)g(writing)h(message-passing)g(programs.)29 b(As)19 b(suc)o(h)g(the)g(in)o(terface)g(should)g(establish)h(a)75 1005 y(practical,)c(p)q(ortable,)f(e\016cien)o(t,)g(and)h(\015exible)h (standard)e(for)g(message)f(passing.)166 1061 y(A)h(complete)h(list)g (of)f(goals)g(follo)o(ws.)143 1144 y Ft(\017)23 b Fx(Design)11 b(an)f(application)i(programming)e(in)o(terface)h(\(not)f(necessarily)i (for)d(compilers)j(or)e(a)g(system)189 1201 y(implemen)o(tation)16 b(library\).)143 1288 y Ft(\017)23 b Fx(Allo)o(w)15 b(e\016cien)o(t)h (comm)o(unication:)21 b(Av)o(oid)16 b(memory-to-memory)e(cop)o(ying)h (and)h(allo)o(w)f(o)o(v)o(erlap)189 1344 y(of)c(computation)h(and)g (comm)o(unication)h(and)f(o\017oad)f(to)g(comm)o(unication)i(co-pro)q (cessor,)f(where)189 1401 y(a)o(v)m(ailable.)143 1488 y Ft(\017)23 b Fx(Allo)o(w)15 b(for)g(implemen)o(tations)h(that)f(can)g (b)q(e)h(used)g(in)g(a)f(heterogeneous)g(en)o(vironmen)o(t.)143 1575 y Ft(\017)23 b Fx(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C)f(and)h(F) l(ortran)e(77)g(bindings)j(for)e(the)g(in)o(terface.)143 1662 y Ft(\017)23 b Fx(Assume)14 b(a)h(reliable)h(comm)o(unication)g (in)o(terface:)j(the)c(user)g(need)g(not)f(cop)q(e)h(with)g(comm)o (unica-)189 1718 y(tion)c(failures.)19 b(Suc)o(h)12 b(failures)f(are)g (dealt)g(with)g(b)o(y)g(the)g(underlying)i(comm)o(unication)e (subsystem.)143 1806 y Ft(\017)23 b Fx(De\014ne)15 b(an)f(in)o(terface) h(that)f(is)h(not)f(to)q(o)g(di\013eren)o(t)g(from)g(curren)o(t)h (practice,)f(suc)o(h)h(as)f(PVM,)g(NX,)189 1862 y(Express,)g(p4,)h (etc.,)g(and)g(pro)o(vides)h(extensions)g(that)e(allo)o(w)i(greater)e (\015exibilit)o(y)l(.)143 1949 y Ft(\017)23 b Fx(De\014ne)d(an)f(in)o (terface)h(that)e(can)i(b)q(e)g(implemen)o(ted)h(on)e(man)o(y)g(v)o (endor's)g(platforms,)g(with)h(no)189 2006 y(signi\014can)o(t)c(c)o (hanges)f(in)h(the)f(underlying)i(comm)o(unication)f(and)g(system)e (soft)o(w)o(are.)143 2093 y Ft(\017)23 b Fx(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h(should)g(b)q(e)g(language)f(indep)q(enden)o (t.)143 2180 y Ft(\017)23 b Fx(The)15 b(in)o(terface)h(should)g(b)q(e)g (designed)g(to)f(allo)o(w)g(for)g(thread-safet)o(y)l(.)75 2320 y Fn(1.2)59 b(Who)20 b(Should)g(Use)g(This)f(Standa)n(rd?)75 2421 y Fx(This)d(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h(all) g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 2478 y(programs)9 b(in)j(F)l(ortran)d(77)i(and)f (C.)h(This)g(includes)i(individual)h(application)e(programmers,)e(dev)o (elop)q(ers)75 2534 y(of)17 b(soft)o(w)o(are)e(designed)j(to)f(run)g (on)g(parallel)i(mac)o(hines,)f(and)f(creators)f(of)h(en)o(vironmen)o (ts)g(and)g(to)q(ols.)75 2591 y(In)g(order)f(to)f(b)q(e)i(attractiv)o (e)e(to)h(this)g(wide)i(audience,)f(the)g(standard)e(m)o(ust)h(pro)o (vide)h(a)e(simple,)j(easy-)75 2647 y(to-use)f(in)o(terface)g(for)f (the)g(basic)i(user)f(while)h(not)e(seman)o(tically)i(precluding)h(the) e(high-p)q(erformance)75 2704 y(message-passing)e(op)q(erations)h(a)o (v)m(ailable)h(on)e(adv)m(anced)h(mac)o(hines.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 11 3 10 bop 75 -100 a Fm(1.3.)34 b(WHA)l(T)15 b(PLA)l(TF)o(ORMS)h(ARE)g(T) l(AR)o(GETS)f(F)o(OR)g(IMPLEMENT)l(A)l(TION?)296 b Fx(3)75 45 y Fn(1.3)59 b(What)20 b(Platfo)n(rms)g(Are)g(T)-5 b(a)n(rgets)19 b(F)n(o)n(r)i(Implementat)o(ion?)75 147 y Fx(The)c(attractiv)o(eness)f(of)g(the)h(message-passing)g(paradigm)f (at)g(least)h(partially)h(stems)e(from)g(its)h(wide)75 203 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o (y)f(ma)o(y)g(run)h(on)g(distributed-memory)h(m)o(ultipro)q(cessors,)75 259 y(net)o(w)o(orks)i(of)i(w)o(orkstations,)e(and)i(com)o(binations)g (of)g(all)g(of)f(these.)34 b(In)20 b(addition,)h(shared-memory)75 316 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 372 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 429 y(th)o(us)d(should)i(b)q(e)f(b)q(oth)g (p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f(on) h(a)f(great)g(v)m(ariet)o(y)g(of)75 485 y(mac)o(hines,)i(including)h (those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h(of)e(other)h (mac)o(hines,)g(parallel)75 542 y(or)d(not,)f(connected)i(b)o(y)f(a)g (comm)o(unication)h(net)o(w)o(ork.)166 598 y(The)21 b(in)o(terface)f (is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i(general)f(MIMD)f (programs,)g(as)g(w)o(ell)h(as)f(those)75 655 y(written)d(in)g(the)g (more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g(Although)h(no)f(explicit)j (supp)q(ort)e(for)f(threads)g(is)75 711 y(pro)o(vided,)e(the)f(in)o (terface)h(has)f(b)q(een)h(designed)g(so)f(as)g(not)f(to)h(prejudice)i (their)e(use.)20 b(With)13 b(this)h(v)o(ersion)75 768 y(of)h Fv(MPI)f Fx(no)h(supp)q(ort)h(is)g(pro)o(vided)g(for)e(dynamic)i (spa)o(wning)g(of)e(tasks.)166 824 y Fv(MPI)e Fx(pro)o(vides)h(man)o(y) g(features)f(in)o(tended)i(to)e(impro)o(v)o(e)h(p)q(erformance)g(on)g (scalable)h(parallel)g(com-)75 880 y(puters)23 b(with)g(sp)q(ecialized) j(in)o(terpro)q(cessor)c(comm)o(unication)i(hardw)o(are.)42 b(Th)o(us,)24 b(w)o(e)e(exp)q(ect)i(that)75 937 y(nativ)o(e,)18 b(high-p)q(erformance)h(implemen)o(tations)g(of)f Fv(MPI)f Fx(will)i(b)q(e)g(pro)o(vided)f(on)g(suc)o(h)g(mac)o(hines.)28 b(A)o(t)75 993 y(the)17 b(same)f(time,)h(implemen)o(tations)h(of)e Fv(MPI)g Fx(on)g(top)h(of)f(standard)g(Unix)i(in)o(terpro)q(cessor)e (comm)o(uni-)75 1050 y(cation)e(proto)q(cols)g(will)i(pro)o(vide)e(p)q (ortabilit)o(y)h(to)f(w)o(orkstation)e(clusters)j(and)f(heterogenous)g (net)o(w)o(orks)75 1106 y(of)h(w)o(orkstations.)j(Sev)o(eral)e (proprietary)l(,)e(nativ)o(e)i(implemen)o(tations)g(of)f Fv(MPI)p Fx(,)f(and)h(a)g(public)i(domain,)75 1163 y(p)q(ortable)f (implemen)o(tation)g(of)f Fv(MPI)g Fx(are)f(in)i(progress)f(at)g(the)g (time)g(of)g(this)h(writing)f([17,)f(13].)75 1306 y Fn(1.4)59 b(What)20 b(Is)f(Included)g(In)g(The)g(Standa)n(rd?)75 1407 y Fx(The)c(standard)g(includes:)143 1514 y Ft(\017)23 b Fx(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 1608 y Ft(\017)23 b Fx(Collectiv)o(e)16 b(op)q(erations)143 1701 y Ft(\017)23 b Fx(Pro)q(cess)15 b(groups)143 1795 y Ft(\017)23 b Fx(Comm)o(unication)15 b(con)o(texts)143 1889 y Ft(\017)23 b Fx(Pro)q(cess)15 b(top)q(ologies)143 1983 y Ft(\017)23 b Fx(Bindings)17 b(for)d(F)l(ortran)g(77)h(and)g(C) 143 2077 y Ft(\017)23 b Fx(En)o(vironmen)o(tal)15 b(Managemen)o(t)f (and)i(inquiry)143 2170 y Ft(\017)23 b Fx(Pro\014ling)16 b(in)o(terface)75 2314 y Fn(1.5)59 b(What)20 b(Is)f(Not)h(Included)e (In)i(The)f(Standa)n(rd?)75 2415 y Fx(The)c(standard)g(do)q(es)h(not)f (sp)q(ecify:)143 2521 y Ft(\017)23 b Fx(Explicit)17 b(shared-memory)e (op)q(erations)143 2615 y Ft(\017)23 b Fx(Op)q(erations)c(that)e (require)i(more)f(op)q(erating)h(system)f(supp)q(ort)g(than)g(is)h (curren)o(tly)g(standard;)189 2672 y(for)14 b(example,)i(in)o (terrupt-driv)o(en)g(receiv)o(es,)g(remote)f(execution,)h(or)e(activ)o (e)i(messages)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 12 4 11 bop 75 -100 a Fx(4)903 b Fm(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION) 17 b(TO)e(MPI)143 45 y Ft(\017)23 b Fx(Program)13 b(construction)j(to)q (ols)143 139 y Ft(\017)23 b Fx(Debugging)15 b(facilities)143 233 y Ft(\017)23 b Fx(Explicit)17 b(supp)q(ort)e(for)g(threads)143 327 y Ft(\017)23 b Fx(Supp)q(ort)15 b(for)g(task)f(managemen)o(t)143 420 y Ft(\017)23 b Fx(I/O)15 b(functions)166 527 y(There)g(are)f(man)o (y)h(features)f(that)g(ha)o(v)o(e)g(b)q(een)i(considered)g(and)f(not)g (included)i(in)f(this)f(standard.)75 583 y(This)i(happ)q(ened)h(for)e (a)g(n)o(um)o(b)q(er)h(of)f(reasons,)g(one)h(of)f(whic)o(h)h(is)g(the)g (time)g(constrain)o(t)f(that)g(w)o(as)f(self-)75 640 y(imp)q(osed)h(in)h(\014nishing)g(the)e(standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g(included)j(can)e(alw)o(a)o(ys)e (b)q(e)i(o\013ered)f(as)75 696 y(extensions)j(b)o(y)f(sp)q(eci\014c)h (implemen)o(tations.)27 b(P)o(erhaps)17 b(future)g(v)o(ersions)g(of)f Fv(MPI)h Fx(will)h(address)f(some)75 752 y(of)e(these)g(issues.)75 896 y Fn(1.6)59 b(Organization)20 b(of)g(this)f(Do)r(cument)75 997 y Fx(The)k(follo)o(wing)h(is)f(a)f(list)i(of)e(the)h(remaining)h(c) o(hapters)e(in)i(this)f(do)q(cumen)o(t,)i(along)e(with)g(a)f(brief)75 1054 y(description)17 b(of)d(eac)o(h.)143 1160 y Ft(\017)23 b Fx(Chapter)c(2,)i Fv(MPI)e(T)l(erms)f(and)i(Conventions)p Fx(,)j(explains)e(notational)f(terms)g(and)g(con)o(v)o(en)o(tions)189 1216 y(used)15 b(throughout)g(the)g Fv(MPI)g Fx(do)q(cumen)o(t.)143 1310 y Ft(\017)23 b Fx(Chapter)16 b(3,)g Fv(P)o(oint)h(to)f(P)o(oint)h (Comm)m(unication)p Fx(,)e(de\014nes)i(the)g(basic,)g(pairwise)g(comm)o (unication)189 1367 y(subset)d(of)g Fv(MPI)p Fx(.)f Fl(send)h Fx(and)g Fl(r)n(e)n(c)n(eive)f Fx(are)h(found)h(here,)f(along)g(with)h (man)o(y)f(asso)q(ciated)g(functions)189 1423 y(designed)i(to)f(mak)o (e)f(basic)i(comm)o(unication)g(p)q(o)o(w)o(erful)g(and)f(e\016cien)o (t.)143 1517 y Ft(\017)23 b Fx(Chapter)12 b(4,)h Fv(Collective)g(Com)o (m)n(unications)p Fx(,)e(de\014nes)j(pro)q(cess-group)f(collectiv)o(e)h (comm)o(unication)189 1573 y(op)q(erations.)28 b(W)l(ell)19 b(kno)o(wn)f(examples)h(of)e(this)i(are)e(barrier)i(and)f(broadcast)f (o)o(v)o(er)g(a)h(group)f(of)189 1630 y(pro)q(cesses)e(\(not)g (necessarily)h(all)g(the)g(pro)q(cesses\).)143 1724 y Ft(\017)23 b Fx(Chapter)16 b(5,)g Fv(Groups,)h(Contexts,)i(and)e(Comm)m (unicato)o(rs)p Fx(,)d(sho)o(ws)i(ho)o(w)g(groups)g(of)g(pro)q(cesses)h (are)189 1780 y(formed)c(and)h(manipulated,)i(ho)o(w)d(unique)i(comm)o (unication)g(con)o(texts)e(are)h(obtained,)g(and)g(ho)o(w)189 1836 y(the)h(t)o(w)o(o)f(are)h(b)q(ound)h(together)e(in)o(to)h(a)g Fl(c)n(ommunic)n(ator)p Fx(.)143 1930 y Ft(\017)23 b Fx(Chapter)18 b(6,)h Fv(Pro)q(cess)h(T)l(op)q(ologies)p Fx(,)f(explains)h(a)f(set)f(of)h(utilit)o(y)h(functions)f(mean)o(t)f (to)h(assist)f(in)189 1987 y(the)13 b(mapping)i(of)e(pro)q(cess)g (groups)h(\(a)f(linearly)i(ordered)f(set\))f(to)g(ric)o(her)h(top)q (ological)g(structures)189 2043 y(suc)o(h)h(as)g(m)o(ulti-dimensional)j (grids.)143 2137 y Ft(\017)23 b Fx(Chapter)10 b(7,)h Fv(MPI)g(Environmental)e(Management)p Fx(,)g(explains)j(ho)o(w)f(the)g (programmer)e(can)i(manage)189 2193 y(and)g(mak)o(e)f(inquiries)j(of)d (the)h(curren)o(t)f Fv(MPI)g Fx(en)o(vironmen)o(t.)19 b(These)11 b(functions)g(are)g(needed)h(for)e(the)189 2250 y(writing)k(of)f(correct,)g(robust)g(programs,)f(and)i(are)f(esp)q (ecially)j(imp)q(ortan)o(t)d(for)g(the)h(construction)189 2306 y(of)g(highly-p)q(ortable)k(message-passing)d(programs.)143 2400 y Ft(\017)23 b Fx(Chapter)d(8,)h Fv(Pro\014ling)f(Interface)p Fx(,)i(explains)g(a)e(simple)i(name-shifting)g(con)o(v)o(en)o(tion)e (that)g(an)o(y)189 2457 y Fv(MPI)h Fx(implemen)o(tation)i(m)o(ust)f (supp)q(ort.)40 b(One)23 b(motiv)m(ation)f(for)f(this)i(is)f(the)g (abilit)o(y)h(to)f(put)189 2513 y(p)q(erformance)c(pro\014ling)i(calls) g(in)o(to)f Fv(MPI)f Fx(without)g(the)h(need)g(for)f(access)h(to)f(the) h Fv(MPI)f Fx(source)189 2570 y(co)q(de.)31 b(The)19 b(name)g(shift)g(is)h(merely)f(an)g(in)o(terface,)h(it)f(sa)o(ys)f (nothing)i(ab)q(out)e(ho)o(w)h(the)g(actual)189 2626 y(pro\014ling)d(should)g(b)q(e)f(done)h(and)f(in)h(fact,)e(the)h(name)g (shift)g(can)g(b)q(e)h(useful)g(for)e(other)h(purp)q(oses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 13 5 12 bop 75 -100 a Fm(1.6.)34 b(OR)o(GANIZA)l(TION)17 b(OF)e(THIS)h(DOCUMENT)810 b Fx(5)143 45 y Ft(\017)23 b Fx(Annex)18 b(A,)g Fv(Language)g(Bindings)p Fx(,)i(giv)o(es)d(sp)q (eci\014c)j(syn)o(tax)d(in)i(F)l(ortran)d(77)i(and)g(C,)f(for)g(all)i Fv(MPI)189 102 y Fx(functions,)c(constan)o(ts,)f(and)h(t)o(yp)q(es.)143 195 y Ft(\017)23 b Fx(The)12 b Fv(MPI)g(F)o(unction)i(Index)f Fx(is)g(a)f(simple)i(index)g(sho)o(wing)e(the)h(lo)q(cation)g(of)f(the) h(precise)g(de\014nition)189 252 y(of)h(eac)o(h)i Fv(MPI)e Fx(function,)i(together)e(with)i(b)q(oth)f(C)g(and)h(F)l(ortran)e (bindings.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 14 6 13 bop 75 356 a Fo(Chapter)34 b(2)75 564 y Fr(MPI)40 b(T)-10 b(erms)39 b(and)g(Con)m(v)m(en)m(tions)75 786 y Fx(This)12 b(c)o(hapter)f(explains)i(notational)e(terms)g(and)g(con)o (v)o(en)o(tions)g(used)h(throughout)f(the)g Fv(MPI)g Fx(do)q(cumen)o(t,)75 843 y(some)k(of)g(the)g(c)o(hoices)h(that)e(ha)o (v)o(e)h(b)q(een)i(made,)d(and)i(the)f(rationale)h(b)q(ehind)h(those)e (c)o(hoices.)75 986 y Fn(2.1)59 b(Do)r(cument)18 b(Notation)189 1088 y Fl(R)n(ationale.)45 b Fx(Throughout)15 b(this)i(do)q(cumen)o(t,) f(the)g(rationale)h(for)e(the)h(design)h(c)o(hoices)g(made)f(in)189 1144 y(the)j(in)o(terface)g(sp)q(eci\014cation)i(is)e(set)g(o\013)f(in) i(this)f(format.)30 b(Some)19 b(readers)g(ma)o(y)f(wish)i(to)e(skip)189 1201 y(these)i(sections,)h(while)h(readers)e(in)o(terested)g(in)h(in)o (terface)f(design)h(ma)o(y)f(w)o(an)o(t)f(to)g(read)h(them)189 1257 y(carefully)l(.)h(\()p Fl(End)16 b(of)g(r)n(ationale.)p Fx(\))189 1363 y Fl(A)n(dvic)n(e)21 b(to)h(users.)78 b Fx(Throughout)21 b(this)h(do)q(cumen)o(t,)h(material)f(that)f(sp)q (eaks)g(to)g(users)h(and)189 1420 y(illustrates)12 b(usage)g(is)g(set)f (o\013)g(in)i(this)f(format.)17 b(Some)12 b(readers)f(ma)o(y)g(wish)i (to)e(skip)h(these)g(sections,)189 1476 y(while)g(readers)e(in)o (terested)h(in)g(programming)f(in)h Fv(MPI)f Fx(ma)o(y)f(w)o(an)o(t)h (to)f(read)i(them)f(carefully)l(.)20 b(\()p Fl(End)189 1533 y(of)c(advic)n(e)g(to)h(users.)p Fx(\))189 1639 y Fl(A)n(dvic)n(e)k(to)i(implementors.)80 b Fx(Throughout)22 b(this)g(do)q(cumen)o(t,)i(material)e(that)f(is)i(primarily)189 1695 y(commen)o(tary)13 b(to)g(implemen)o(tors)i(is)f(set)g(o\013)f(in) i(this)g(format.)j(Some)c(readers)g(ma)o(y)f(wish)i(to)e(skip)189 1752 y(these)20 b(sections,)i(while)g(readers)f(in)o(terested)g(in)g Fv(MPI)f Fx(implemen)o(tations)i(ma)o(y)e(w)o(an)o(t)f(to)h(read)189 1808 y(them)15 b(carefully)l(.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g (to)h(implementors.)p Fx(\))75 1951 y Fn(2.2)59 b(Pro)r(cedure)19 b(Sp)r(eci\014cation)75 2053 y Fv(MPI)g Fx(pro)q(cedures)i(are)e(sp)q (eci\014ed)j(using)f(a)f(language)g(indep)q(enden)o(t)i(notation.)33 b(The)20 b(argumen)o(ts)f(of)75 2109 y(pro)q(cedure)d(calls)g(are)f (mark)o(ed)g(as)g Fk(IN)p Fx(,)g Fk(OUT)g Fx(or)g Fk(INOUT)p Fx(.)g(The)h(meanings)f(of)g(these)h(are:)143 2203 y Ft(\017)23 b Fx(the)15 b(call)h(uses)g(but)f(do)q(es)h(not)e(up)q(date) i(an)f(argumen)o(t)g(mark)o(ed)f Fk(IN)p Fx(,)143 2297 y Ft(\017)23 b Fx(the)15 b(call)h(ma)o(y)f(up)q(date)h(an)f(argumen)o (t)f(mark)o(ed)h Fk(OUT)p Fx(,)143 2391 y Ft(\017)23 b Fx(the)15 b(call)h(b)q(oth)g(uses)f(and)g(up)q(dates)h(an)f(argumen)o (t)g(mark)o(ed)f Fk(INOUT)p Fx(.)166 2485 y(There)20 b(is)g(one)g(sp)q(ecial)i(case)e(|)g(if)h(an)e(argumen)o(t)h(is)g(a)g (handle)h(to)e(an)h(opaque)g(ob)s(ject)f(\(these)75 2541 y(terms)d(are)h(de\014ned)h(in)g(Section)f(2.4.1\),)e(and)i(the)g(ob)s (ject)f(is)i(up)q(dated)f(b)o(y)g(the)g(pro)q(cedure)h(call,)g(then)75 2598 y(the)g(argumen)o(t)g(is)g(mark)o(ed)g Fk(OUT)p Fx(.)g(It)h(is)f(mark)o(ed)g(this)h(w)o(a)o(y)e(ev)o(en)i(though)f(the) g(handle)h(itself)h(is)e(not)75 2654 y(mo)q(di\014ed)c(|)f(w)o(e)f(use) h(the)g Fk(OUT)g Fx(attribute)f(to)g(denote)h(that)f(what)g(the)g (handle)i Fl(r)n(efer)n(enc)n(es)c Fx(is)j(up)q(dated.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 15 7 14 bop 75 -100 a Fm(2.3.)34 b(SEMANTIC)15 b(TERMS)1244 b Fx(7)166 45 y(The)17 b(de\014nition)i(of)d Fv(MPI)g Fx(tries)h(to)f(a)o(v)o(oid,)h(to)f(the)h(largest)f(p)q(ossible)j (exten)o(t,)d(the)h(use)g(of)f Fk(INOUT)75 102 y Fx(argumen)o(ts,)e(b)q (ecause)i(suc)o(h)g(use)f(is)h(error-prone,)e(esp)q(ecially)k(for)d (scalar)g(argumen)o(ts.)166 158 y(A)k(common)f(o)q(ccurrence)h(for)f Fv(MPI)g Fx(functions)i(is)f(an)f(argumen)o(t)g(that)g(is)h(used)g(as)f Fk(IN)h Fx(b)o(y)f(some)75 214 y(pro)q(cesses)12 b(and)g Fk(OUT)f Fx(b)o(y)h(other)f(pro)q(cesses.)19 b(Suc)o(h)12 b(argumen)o(t)f(is,)h(syn)o(tactically)l(,)h(an)f Fk(INOUT)g Fx(argumen)o(t)75 271 y(and)j(is)h(mark)o(ed)e(as)g(suc)o(h,)h (although,)g(seman)o(tically)l(,)h(it)f(is)h(not)e(used)i(in)f(one)g (call)i(b)q(oth)e(for)f(input)i(and)75 327 y(for)f(output.)166 384 y(Another)h(frequen)o(t)g(situation)h(arises)g(when)f(an)g(argumen) o(t)g(v)m(alue)h(is)g(needed)h(only)f(b)o(y)f(a)g(subset)75 440 y(of)f(the)g(pro)q(cesses.)20 b(When)c(an)f(argumen)o(t)f(is)i(not) f(signi\014can)o(t)h(at)e(a)h(pro)q(cess)h(then)f(an)g(arbitrary)g(v)m (alue)75 497 y(can)g(b)q(e)h(passed)g(as)e(argumen)o(t.)166 553 y(Unless)h(sp)q(eci\014ed)g(otherwise,)f(an)g(argumen)o(t)f(of)g(t) o(yp)q(e)h Fk(OUT)g Fx(or)f(t)o(yp)q(e)h Fk(INOUT)g Fx(cannot)g(b)q(e)g (aliased)75 610 y(with)h(an)o(y)f(other)g(argumen)o(t)g(passed)g(to)g (an)g Fv(MPI)g Fx(pro)q(cedure.)21 b(An)15 b(example)g(of)f(argumen)o (t)f(aliasing)j(in)75 666 y(C)f(app)q(ears)g(b)q(elo)o(w.)21 b(If)15 b(w)o(e)g(de\014ne)h(a)f(C)g(pro)q(cedure)h(lik)o(e)h(this,)75 760 y Fu(void)23 b(copyIntBuffer\()f(int)i(*pin,)f(int)g(*pout,)g(int)h (len)f(\))75 816 y({)71 b(int)24 b(i;)170 873 y(for)g(\(i=0;)f(i)g(BUF\(*\))170 2356 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 2443 y Fx(The)15 b(blo)q(c)o(king)i(seman)o(tics)e(of)g(this)h(call)g (are)f(describ)q(ed)i(in)f(Sec.)k(3.4.)75 2561 y Fj(3.2.2)49 b(Message)16 b(data)75 2647 y Fx(The)e(send)g(bu\013er)g(sp)q (eci\014ed)i(b)o(y)e(the)f Fv(MPI)p 803 2647 14 2 v 16 w(SEND)h Fx(op)q(eration)g(consists)g(of)f Fv(count)i Fx(successiv)o(e)g(en)o(tries)f(of)75 2704 y(the)f(t)o(yp)q(e)h (indicated)h(b)o(y)e Fv(datat)o(yp)q(e)p Fx(,)i(starting)e(with)h(the)f (en)o(try)g(at)g(address)h Fv(buf)p Fx(.)20 b(Note)13 b(that)g(w)o(e)g(sp)q(ecify)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 25 17 24 bop 75 -100 a Fm(3.2.)34 b(BLOCKING)17 b(SEND)e(AND)g(RECEIVE)g (OPERA)l(TIONS)598 b Fx(17)75 45 y(the)19 b(message)f(length)i(in)g (terms)e(of)g(n)o(um)o(b)q(er)h(of)g Fl(elements)p Fx(,)f(not)g(n)o(um) o(b)q(er)i(of)e Fl(bytes)p Fx(.)30 b(The)19 b(former)f(is)75 102 y(mac)o(hine)e(indep)q(enden)o(t)i(and)d(closer)h(to)e(the)h (application)i(lev)o(el.)166 158 y(The)g(data)f(part)g(of)g(the)g (message)h(consists)f(of)h(a)f(sequence)i(of)e Fv(count)i Fx(v)m(alues,)f(eac)o(h)g(of)f(the)h(t)o(yp)q(e)75 214 y(indicated)j(b)o(y)e Fv(datat)o(yp)q(e)p Fx(.)31 b Fv(count)20 b Fx(ma)o(y)d(b)q(e)i(zero,)g(in)g(whic)o(h)g(case)g(the)f(data)g(part) f(of)h(the)h(message)e(is)75 271 y(empt)o(y)l(.)i(The)13 b(basic)h(datat)o(yp)q(es)e(that)h(can)g(b)q(e)h(sp)q(eci\014ed)h(for)d (message)h(data)f(v)m(alues)i(corresp)q(ond)g(to)e(the)75 327 y(basic)k(datat)o(yp)q(es)f(of)g(the)g(host)g(language.)20 b(P)o(ossible)d(v)m(alues)f(of)f(this)h(argumen)o(t)e(for)h(F)l(ortran) f(and)i(the)75 384 y(corresp)q(onding)g(F)l(ortran)e(t)o(yp)q(es)h(are) g(listed)i(b)q(elo)o(w.)p 472 532 1006 2 v 471 589 2 57 v 497 572 a(MPI)e(datat)o(yp)q(e)p 1046 589 V 294 w(F)l(ortran)f(datat)o(yp)q(e)p 1477 589 V 472 591 1006 2 v 471 647 2 57 v 497 630 a Fv(MPI)p 582 630 14 2 v 16 w(INTEGER)p 1046 647 2 57 v 287 w Fu(INTEGER)p 1477 647 V 471 703 V 497 687 a Fv(MPI)p 582 687 14 2 v 16 w(REAL)p 1046 703 2 57 v 365 w Fu(REAL)p 1477 703 V 471 760 V 497 743 a Fv(MPI)p 582 743 14 2 v 16 w(DOUBLE)p 777 743 V 17 w(PRECISION)p 1046 760 2 57 v 50 w Fu(DOUBLE)23 b(PRECISION)p 1477 760 V 471 816 V 497 799 a Fv(MPI)p 582 799 14 2 v 16 w(COMPLEX)p 1046 816 2 57 v 263 w Fu(COMPLEX)p 1477 816 V 471 873 V 497 856 a Fv(MPI)p 582 856 14 2 v 16 w(LOGICAL)p 1046 873 2 57 v 291 w Fu(LOGICAL)p 1477 873 V 471 929 V 497 912 a Fv(MPI)p 582 912 14 2 v 16 w(CHARA)o(CTER)p 1046 929 2 57 v 211 w Fu(CHARACTER\(1\))p 1477 929 V 471 986 V 497 969 a Fv(MPI)p 582 969 14 2 v 16 w(BYTE)p 1046 986 2 57 v 1477 986 V 471 1042 V 497 1025 a(MPI)p 582 1025 14 2 v 16 w(P)l(A)o(CKED)p 1046 1042 2 57 v 1477 1042 V 472 1044 1006 2 v 75 1214 a Fx(P)o(ossible)16 b(v)m(alues)g(for)f(this)h(argumen)o(t)e(for)h(C)g(and)g(the)g(corresp) q(onding)h(C)f(t)o(yp)q(es)h(are)f(listed)h(b)q(elo)o(w.)p 466 1354 1019 2 v 465 1410 2 57 v 491 1393 a(MPI)f(datat)o(yp)q(e)p 1004 1410 V 259 w(C)g(datat)o(yp)q(e)p 1483 1410 V 466 1412 1019 2 v 465 1468 2 57 v 491 1451 a Fv(MPI)p 576 1451 14 2 v 16 w(CHAR)p 1004 1468 2 57 v 321 w Fu(signed)23 b(char)p 1483 1468 V 465 1525 V 491 1508 a Fv(MPI)p 576 1508 14 2 v 16 w(SHORT)p 1004 1525 2 57 v 291 w Fu(signed)g(short)p 1483 1525 V 465 1581 V 491 1564 a Fv(MPI)p 576 1564 14 2 v 16 w(INT)p 1004 1581 2 57 v 365 w Fu(signed)g(int)p 1483 1581 V 465 1638 V 491 1621 a Fv(MPI)p 576 1621 14 2 v 16 w(LONG)p 1004 1638 2 57 v 321 w Fu(signed)g(long)g(int)p 1483 1638 V 465 1694 V 491 1677 a Fv(MPI)p 576 1677 14 2 v 16 w(UNSIGNED)p 815 1677 V 17 w(CHAR)p 1004 1694 2 57 v 81 w Fu(unsigned)f(char)p 1483 1694 V 465 1750 V 491 1733 a Fv(MPI)p 576 1733 14 2 v 16 w(UNSIGNED)p 815 1733 V 17 w(SHORT)p 1004 1750 2 57 v 51 w Fu(unsigned)g(short)i (int)p 1483 1750 V 465 1807 V 491 1790 a Fv(MPI)p 576 1790 14 2 v 16 w(UNSIGNED)p 1004 1807 2 57 v 218 w Fu(unsigned)e(int)p 1483 1807 V 465 1863 V 491 1846 a Fv(MPI)p 576 1846 14 2 v 16 w(UNSIGNED)p 815 1846 V 17 w(LONG)p 1004 1863 2 57 v 81 w Fu(unsigned)g(long)i(int)p 1483 1863 V 465 1920 V 491 1903 a Fv(MPI)p 576 1903 14 2 v 16 w(FLO)o(A)l(T)p 1004 1920 2 57 v 301 w Fu(float)p 1483 1920 V 465 1976 V 491 1959 a Fv(MPI)p 576 1959 14 2 v 16 w(DOUBLE)p 1004 1976 2 57 v 262 w Fu(double)p 1483 1976 V 465 2033 V 491 2016 a Fv(MPI)p 576 2016 14 2 v 16 w(LONG)p 712 2016 V 16 w(DOUBLE)p 1004 2033 2 57 v 126 w Fu(long)f(double)p 1483 2033 V 465 2089 V 491 2072 a Fv(MPI)p 576 2072 14 2 v 16 w(BYTE)p 1004 2089 2 57 v 1483 2089 V 465 2146 V 491 2129 a(MPI)p 576 2129 14 2 v 16 w(P)l(A)o(CKED)p 1004 2146 2 57 v 1483 2146 V 466 2147 1019 2 v 166 2308 a Fx(The)f(datat)o(yp)q(es)f Fv(MPI)p 566 2308 14 2 v 16 w(BYTE)h Fx(and)g Fv(MPI)p 899 2308 V 15 w(P)l(A)o(CKED)g Fx(do)f(not)g(corresp)q(ond)h(to)f(a)h(F)l(ortran)e(or)h(C)75 2365 y(datat)o(yp)q(e.)42 b(A)23 b(v)m(alue)h(of)e(t)o(yp)q(e)h Fv(MPI)p 742 2365 V 15 w(BYTE)h Fx(consists)f(of)f(a)g(b)o(yte)h(\(8)f (binary)h(digits\).)43 b(A)23 b(b)o(yte)f(is)75 2421 y(unin)o(terpreted)h(and)f(is)h(di\013eren)o(t)f(from)f(a)h(c)o (haracter.)39 b(Di\013eren)o(t)22 b(mac)o(hines)h(ma)o(y)e(ha)o(v)o(e)h (di\013eren)o(t)75 2478 y(represen)o(tations)15 b(for)g(c)o(haracters,) f(or)h(ma)o(y)g(use)h(more)f(than)g(one)g(b)o(yte)h(to)e(represen)o(t)i (c)o(haracters.)j(On)75 2534 y(the)g(other)g(hand,)h(a)f(b)o(yte)g(has) g(the)g(same)g(binary)g(v)m(alue)i(on)e(all)h(mac)o(hines.)32 b(The)20 b(use)f(of)g(the)g(t)o(yp)q(e)75 2591 y Fv(MPI)p 160 2591 V 16 w(P)l(A)o(CKED)c Fx(is)h(explained)h(in)f(Section)g (3.13.)166 2647 y Fv(MPI)c Fx(requires)h(supp)q(ort)g(of)f(the)h(datat) o(yp)q(es)f(listed)i(ab)q(o)o(v)o(e,)e(whic)o(h)h(matc)o(h)f(the)h (basic)g(datat)o(yp)q(es)f(of)75 2704 y(F)l(ortran)h(77)g(and)h(ANSI)h (C.)f(Additional)h Fv(MPI)f Fx(datat)o(yp)q(es)f(should)i(b)q(e)g(pro)o (vided)g(if)f(the)g(host)g(language)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 18 26 18 25 bop 75 -100 a Fx(18)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(has)e(additional)i (data)e(t)o(yp)q(es:)20 b Fv(MPI)p 698 45 14 2 v 16 w(LONG)p 834 45 V 16 w(LONG)p 970 45 V 17 w(INT)p Fx(,)15 b(for)g(\(64)f(bit\))i (C)f(in)o(tegers)g(declared)i(to)d(b)q(e)i(of)75 102 y(t)o(yp)q(e)11 b Fv(longlong)f(int)p Fx(;)i Fv(MPI)p 503 102 V 16 w(DOUBLE)p 698 102 V 17 w(COMPLEX)f Fx(for)f(double)i (precision)g(complex)g(in)f(F)l(ortran)f(declared)75 158 y(to)16 b(b)q(e)h(of)f(t)o(yp)q(e)g Fv(DOUBLE)h(PRECISION)p Fx(;)g Fv(MPI)p 892 158 V 16 w(REAL2)p Fx(,)f Fv(MPI)p 1153 158 V 16 w(REAL4)g Fx(and)h Fv(MPI)p 1491 158 V 16 w(REAL8)f Fx(for)g(F)l(ortran)75 214 y(reals,)i(declared)h(to)e(b)q (e)h(of)f(t)o(yp)q(e)h Fv(REAL*2)p Fx(,)g Fv(REAL*4)f Fx(and)h Fv(REAL*8)p Fx(,)g(resp)q(ectiv)o(ely;)i Fv(MPI)p 1650 214 V 16 w(INTEGER1)75 271 y(MPI)p 160 271 V 16 w(INTEGER2)10 b Fx(and)h Fv(MPI)p 564 271 V 16 w(INTEGER4)f Fx(for)g(F)l(ortran)f(in)o(tegers,)i(declared)h(to)d(b)q(e)i(of)f(t)o (yp)q(e)g Fv(INTEGER*1)p Fx(,)75 327 y Fv(INTEGER*2)15 b Fx(and)h Fv(INTEGER*4)p Fx(,)f(resp)q(ectiv)o(ely;)h(etc.)189 407 y Fl(R)n(ationale.)80 b Fx(One)22 b(goal)g(of)f(the)h(design)h(is)g (to)e(allo)o(w)h(for)f Fv(MPI)g Fx(to)h(b)q(e)g(implemen)o(ted)i(as)d (a)189 464 y(library)l(,)d(with)f(no)g(need)h(for)e(additional)j (prepro)q(cessing)f(or)e(compilation.)27 b(Th)o(us,)17 b(one)g(cannot)189 520 y(assume)c(that)g(a)g(comm)o(unication)h(call)g (has)g(information)f(on)h(the)f(datat)o(yp)q(e)g(of)g(v)m(ariables)i (in)f(the)189 576 y(comm)o(unication)20 b(bu\013er;)i(this)f (information)f(m)o(ust)f(b)q(e)i(supplied)h(b)o(y)e(an)g(explicit)i (argumen)o(t.)189 633 y(The)17 b(need)h(for)f(suc)o(h)h(datat)o(yp)q(e) e(information)i(will)h(b)q(ecome)f(clear)g(in)g(Section)g(3.3.2.)24 b(\()p Fl(End)18 b(of)189 689 y(r)n(ationale.)p Fx(\))75 808 y Fj(3.2.3)49 b(Message)16 b(envelop)q(e)75 893 y Fx(In)j(addition)g(to)e(the)h(data)g(part,)f(messages)h(carry)f (information)i(that)e(can)h(b)q(e)h(used)f(to)g(distinguish)75 950 y(messages)12 b(and)h(selectiv)o(ely)i(receiv)o(e)f(them.)19 b(This)14 b(information)f(consists)g(of)f(a)h(\014xed)g(n)o(um)o(b)q (er)h(of)e(\014elds,)75 1006 y(whic)o(h)k(w)o(e)f(collectiv)o(ely)j (call)e(the)f Fq(message)i(en)o(v)o(elop)q(e)p Fx(.)i(These)d(\014elds) g(are)913 1081 y(source)865 1138 y(destination)943 1194 y(tag)834 1251 y(comm)o(unicator)166 1325 y(The)c(message)g(source)g (is)h(implicitly)i(determined)f(b)o(y)e(the)g(iden)o(tit)o(y)h(of)f (the)g(message)g(sender.)20 b(The)75 1382 y(other)15 b(\014elds)h(are)f(sp)q(eci\014ed)i(b)o(y)f(argumen)o(ts)e(in)i(the)f (send)h(op)q(eration.)166 1438 y(The)f(message)g(destination)h(is)g(sp) q(eci\014ed)h(b)o(y)e(the)h Fv(dest)g Fx(argumen)o(t.)166 1495 y(The)h(in)o(teger-v)m(alued)h(message)e(tag)f(is)i(sp)q (eci\014ed)i(b)o(y)d(the)g Fv(tag)h Fx(argumen)o(t.)22 b(This)17 b(in)o(teger)g(can)f(b)q(e)75 1551 y(used)21 b(b)o(y)f(the)h(program)e(to)h(distinguish)i(di\013eren)o(t)f(t)o(yp)q (es)f(of)g(messages.)35 b(The)21 b(range)f(of)g(v)m(alid)i(tag)75 1608 y(v)m(alues)17 b(is)f Fv(0,...,UB)p Fx(,)e(where)i(the)g(v)m(alue) h(of)f Fv(UB)g Fx(is)g(implemen)o(tation)h(dep)q(enden)o(t.)24 b(It)15 b(can)h(b)q(e)h(found)f(b)o(y)75 1664 y(querying)i(the)g(v)m (alue)g(of)f(the)h(attribute)f Fi(MPI)p 870 1664 13 2 v 14 w(T)m(A)o(G)p 964 1664 V 14 w(UB)p Fx(,)g(as)g(describ)q(ed)i(in)f (Chapter)f(7.)26 b Fv(MPI)17 b Fx(requires)75 1721 y(that)d Fv(UB)i Fx(b)q(e)g(no)f(less)h(than)f(32767.)166 1777 y(The)f Fv(comm)9 b Fx(argumen)o(t)k(sp)q(eci\014es)j(the)e Fq(comm)o(unicator)h Fx(that)e(is)i(used)g(for)e(the)h(send)h(op)q (eration.)75 1833 y(Comm)o(unicators)f(are)h(explained)j(in)e(Chapter)e (5;)h(b)q(elo)o(w)h(is)f(a)g(brief)h(summary)f(of)g(their)g(usage.)166 1890 y(A)h(comm)o(unicator)f(sp)q(eci\014es)j(the)d(comm)o(unication)i (con)o(text)e(for)g(a)h(comm)o(unication)g(op)q(eration.)75 1946 y(Eac)o(h)f(comm)o(unication)h(con)o(text)f(pro)o(vides)g(a)g (separate)g(\\comm)o(unication)h(univ)o(erse:")21 b(messages)14 b(are)75 2003 y(alw)o(a)o(ys)h(receiv)o(ed)j(within)f(the)f(con)o(text) f(they)h(w)o(ere)g(sen)o(t,)g(and)g(messages)g(sen)o(t)f(in)i (di\013eren)o(t)g(con)o(texts)75 2059 y(do)e(not)g(in)o(terfere.)166 2116 y(The)20 b(comm)o(unicator)f(also)g(sp)q(eci\014es)i(the)f(set)f (of)g(pro)q(cesses)g(that)g(share)g(this)h(comm)o(unication)75 2172 y(con)o(text.)i(This)17 b Fq(pro)q(cess)h(group)e Fx(is)g(ordered)h(and)f(pro)q(cesses)g(are)g(iden)o(ti\014ed)i(b)o(y)e (their)h(rank)f(within)75 2229 y(this)g(group.)21 b(Th)o(us,)15 b(the)g(range)h(of)f(v)m(alid)i(v)m(alues)f(for)f Fv(dest)i Fx(is)f Fv(0,)f(...)20 b(,)c(n-1)p Fx(,)f(where)h Fv(n)g Fx(is)g(the)g(n)o(um)o(b)q(er)f(of)75 2285 y(pro)q(cesses)i(in)h(the)e (group.)25 b(\(If)16 b(the)h(comm)o(unicator)f(is)i(an)e(in)o(ter-comm) o(unicator,)h(then)g(destinations)75 2342 y(are)e(iden)o(ti\014ed)i(b)o (y)e(their)h(rank)f(in)h(the)f(remote)g(group.)k(See)d(Chapter)f(5.\)) 166 2398 y(A)g(prede\014ned)i(comm)o(unicator)d Fi(MPI)p 808 2398 V 15 w(COMM)p 953 2398 V 14 w(W)o(ORLD)h Fx(is)h(pro)o(vided)g (b)o(y)f Fv(MPI)p Fx(.)f(It)h(allo)o(ws)g(comm)o(u-)75 2454 y(nication)22 b(with)f(all)h(pro)q(cesses)f(that)f(are)h (accessible)i(after)d Fv(MPI)g Fx(initialization)k(and)d(pro)q(cesses)g (are)75 2511 y(iden)o(ti\014ed)c(b)o(y)e(their)h(rank)f(in)h(the)f (group)g(of)g Fi(MPI)p 939 2511 V 14 w(COMM)p 1083 2511 V 15 w(W)o(ORLD)p Fx(.)189 2591 y Fl(A)n(dvic)n(e)j(to)i(users.)60 b Fx(Users)18 b(that)g(are)g(comfortable)g(with)h(the)g(notion)g(of)f (a)g(\015at)g(name)h(space)189 2647 y(for)d(pro)q(cesses,)h(and)g(a)g (single)h(comm)o(unication)g(con)o(text,)e(as)h(o\013ered)f(b)o(y)h (most)f(existing)i(com-)189 2704 y(m)o(unication)d(libraries,)h(need)f (only)h(use)e(the)h(prede\014ned)h(v)m(ariable)g Fi(MPI)p 1437 2704 V 15 w(COMM)p 1582 2704 V 14 w(W)o(ORLD)f Fx(as)f(the)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 19 27 19 26 bop 75 -100 a Fm(3.2.)34 b(BLOCKING)17 b(SEND)e(AND)g(RECEIVE)g (OPERA)l(TIONS)598 b Fx(19)189 45 y Fv(comm)12 b Fx(argumen)o(t.)29 b(This)19 b(will)h(allo)o(w)f(comm)o(unication)g(with)g(all)g(the)g (pro)q(cesses)f(a)o(v)m(ailable)j(at)189 102 y(initialization)d(time.) 189 177 y(Users)h(ma)o(y)g(de\014ne)i(new)f(comm)o(unicators,)g(as)g (explained)h(in)g(Chapter)f(5.)33 b(Comm)o(unicators)189 233 y(pro)o(vide)13 b(an)g(imp)q(ortan)o(t)g(encapsulation)h(mec)o (hanism)f(for)g(libraries)h(and)f(mo)q(dules.)21 b(They)13 b(allo)o(w)189 290 y(mo)q(dules)21 b(to)e(ha)o(v)o(e)h(their)g(o)o(wn)g (disjoin)o(t)h(comm)o(unication)f(univ)o(erse)h(and)g(their)f(o)o(wn)g (pro)q(cess)189 346 y(n)o(um)o(b)q(ering)c(sc)o(heme.)k(\()p Fl(End)c(of)g(advic)n(e)g(to)h(users.)p Fx(\))189 452 y Fl(A)n(dvic)n(e)i(to)h(implementors.)65 b Fx(The)20 b(message)f(en)o(v)o(elop)q(e)h(w)o(ould)g(normally)g(b)q(e)g(enco)q (ded)h(b)o(y)e(a)189 509 y(\014xed-length)d(message)e(header.)20 b(Ho)o(w)o(ev)o(er,)13 b(the)i(actual)g(enco)q(ding)h(is)f(implemen)o (tation)g(dep)q(en-)189 565 y(den)o(t.)k(Some)c(of)e(the)i(information) f(\(e.g.,)f(source)h(or)g(destination\))h(ma)o(y)e(b)q(e)i(implicit,)i (and)d(need)189 622 y(not)j(b)q(e)i(explicitly)i(carried)e(b)o(y)f (messages.)28 b(Also,)19 b(pro)q(cesses)f(ma)o(y)f(b)q(e)i(iden)o (ti\014ed)h(b)o(y)e(relativ)o(e)189 678 y(ranks,)c(or)h(absolute)g (ids,)h(etc..)j(\()p Fl(End)d(of)g(advic)n(e)h(to)f(implementors.)p Fx(\))75 800 y Fj(3.2.4)49 b(Blo)q(cking)18 b(receive)75 886 y Fx(The)d(syn)o(tax)g(of)g(the)g(blo)q(c)o(king)h(receiv)o(e)h(op) q(eration)e(is)h(giv)o(en)f(b)q(elo)o(w.)75 1037 y Fv(MPI)p 160 1037 14 2 v 16 w(RECV)h(\(buf,)f(count,)h(datat)o(yp)q(e,)h (source,)e(tag,)g(comm)m(,)d(status\))117 1114 y Fk(OUT)108 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 1189 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1264 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1339 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o (teger\))117 1414 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1490 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1565 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1689 y Fu(int)23 b(MPI)p 245 1689 15 2 v 17 w(Recv\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 1689 V 16 w(Datatype)f(datatype,)g(int)g(source,) 393 1746 y(int)h(tag,)f(MPI)p 683 1746 V 17 w(Comm)g(comm,)g(MPI)p 1034 1746 V 17 w(Status)g(*status\))75 1832 y(MPI)p 150 1832 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS,)g(IERROR\))170 1888 y()g(BUF\(*\))170 1945 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 1945 V 16 w(STATUS)p 1623 1945 V 17 w(SIZE\),)170 2001 y(IERROR)166 2088 y Fx(The)15 b(blo)q(c)o(king)i (seman)o(tics)e(of)g(this)h(call)g(are)f(describ)q(ed)i(in)f(Sec.)k (3.4.)166 2144 y(The)15 b(receiv)o(e)h(bu\013er)f(consists)h(of)e(the)h (storage)f(con)o(taining)i Fv(count)g Fx(consecutiv)o(e)g(elemen)o(ts)g (of)f(the)75 2201 y(t)o(yp)q(e)e(sp)q(eci\014ed)i(b)o(y)e Fv(datat)o(yp)q(e)p Fx(,)h(starting)e(at)h(address)f Fv(buf)p Fx(.)20 b(The)13 b(length)h(of)e(the)h(receiv)o(ed)h(message)f (m)o(ust)75 2257 y(b)q(e)20 b(less)g(than)f(or)g(equal)h(to)f(the)h (length)g(of)f(the)g(receiv)o(e)h(bu\013er.)33 b(An)20 b(o)o(v)o(er\015o)o(w)e(error)g(o)q(ccurs)i(if)g(all)75 2314 y(incoming)c(data)f(do)q(es)h(not)e(\014t,)h(without)g (truncation,)g(in)o(to)g(the)h(receiv)o(e)g(bu\013er.)189 2420 y Fl(A)n(dvic)n(e)g(to)i(users.)47 b Fx(The)16 b Fv(MPI)p 733 2420 14 2 v 16 w(PROBE)i Fx(function)f(describ)q(ed)h(in)f (Section)h(3.8)d(can)i(b)q(e)g(used)g(to)189 2476 y(receiv)o(e)f (messages)e(of)h(unkno)o(wn)h(length.)k(\()p Fl(End)c(of)g(advic)n(e)g (to)h(users.)p Fx(\))189 2583 y Fl(A)n(dvic)n(e)f(to)i(implementors.)46 b Fx(Ev)o(en)16 b(though)g(no)h(sp)q(eci\014c)h(b)q(eha)o(vior)f(is)f (mandated)h(b)o(y)f Fv(MPI)g Fx(for)189 2639 y(erroneous)f(programs,)f (the)h(recommended)h(handling)h(of)e(o)o(v)o(er\015o)o(w)f(situations)h (is)h(to)f(return)g(in)189 2696 y Fv(status)21 b Fx(information)e(ab)q (out)g(the)g(source)g(and)g(tag)f(of)h(the)g(incoming)h(message.)31 b(The)19 b(receiv)o(e)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 28 20 27 bop 75 -100 a Fx(20)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fx(op)q(eration)g(will) j(return)d(an)g(error)g(co)q(de.)28 b(A)17 b(qualit)o(y)h(implemen)o (tation)h(will)g(also)f(ensure)g(that)189 102 y(no)d(memory)f(that)g (is)i(outside)g(the)f(receiv)o(e)h(bu\013er)f(will)i(ev)o(er)e(b)q(e)g (o)o(v)o(erwritten.)k(\()p Fl(End)d(of)g(advic)n(e)189 158 y(to)g(implementors.)p Fx(\))166 255 y(The)22 b(selection)i(of)d(a) h(message)g(b)o(y)g(a)f(receiv)o(e)i(op)q(eration)g(is)f(go)o(v)o (erned)g(b)o(y)g(the)g(v)m(alue)h(of)f(the)75 311 y(message)13 b(en)o(v)o(elop)q(e.)20 b(A)14 b(message)f(can)g(b)q(e)h(receiv)o(ed)h (b)o(y)e(a)g(receiv)o(e)i(op)q(eration)e(if)h(its)g(en)o(v)o(elop)q(e)g (matc)o(hes)75 368 y(the)e Fv(source)p Fx(,)g Fv(tag)g Fx(and)g Fv(comm)6 b Fx(v)m(alues)13 b(sp)q(eci\014ed)h(b)o(y)e(the)f (receiv)o(e)i(op)q(eration.)19 b(The)12 b(receiv)o(er)h(ma)o(y)e(sp)q (ecify)75 424 y(a)18 b(wildcard)h Fi(MPI)p 380 424 13 2 v 15 w(ANY)p 480 424 V 14 w(SOURCE)e Fx(v)m(alue)i(for)f Fv(source)p Fx(,)h(and/or)e(a)h(wildcard)36 b Fi(MPI)p 1491 424 V 14 w(ANY)p 1590 424 V 15 w(T)m(A)o(G)17 b Fx(v)m(alue)j(for)75 481 y Fv(tag)p Fx(,)12 b(indicating)i(that)d(an)o (y)h(source)g(and/or)f(tag)g(are)g(acceptable.)20 b(It)12 b(cannot)g(sp)q(ecify)h(a)f(wildcard)h(v)m(alue)75 537 y(for)h Fv(comm)m Fx(.)j(Th)o(us,)d(a)g(message)g(can)h(b)q(e)g(receiv) o(ed)g(b)o(y)g(a)f(receiv)o(e)h(op)q(eration)g(only)g(if)g(it)g(is)g (addressed)g(to)75 594 y(the)i(receiving)i(pro)q(cess,)e(has)g(a)g (matc)o(hing)g(comm)o(unicator,)f(has)h(matc)o(hing)g(source)g(unless)i (source=)75 650 y Fi(MPI)p 152 650 V 14 w(ANY)p 251 650 V 15 w(SOURCE)13 b Fx(in)j(the)g(pattern,)e(and)i(has)f(a)f(matc)o (hing)i(tag)e(unless)i(tag=)e Fi(MPI)p 1553 650 V 14 w(ANY)p 1652 650 V 15 w(T)m(A)o(G)g Fx(in)i(the)75 707 y(pattern.)166 763 y(The)24 b(message)g(tag)f(is)i(sp)q(eci\014ed)h(b)o (y)e(the)g Fv(tag)g Fx(argumen)o(t)g(of)g(the)g(receiv)o(e)h(op)q (eration.)47 b(The)75 820 y(argumen)o(t)24 b Fv(source)p Fx(,)i(if)f(di\013eren)o(t)g(from)e Fi(MPI)p 879 820 V 14 w(ANY)p 978 820 V 15 w(SOURCE)p Fx(,)f(is)j(sp)q(eci\014ed)h(as)e (a)g(rank)g(within)i(the)75 876 y(pro)q(cess)e(group)f(asso)q(ciated)g (with)h(that)e(same)i(comm)o(unicator)e(\(remote)h(pro)q(cess)h(group,) g(for)f(in-)75 932 y(tercomm)o(unicators\).)37 b(Th)o(us,)23 b(the)e(range)g(of)g(v)m(alid)j(v)m(alues)e(for)f(the)g Fv(source)h Fx(argumen)o(t)f(is)h Ft(f)p Fv(0,...,n-)75 989 y(1)p Ft(g[f)p Fv(MPI)p 259 989 14 2 v 15 w(ANY)p 366 989 V 17 w(SOURCE)p Ft(g)p Fx(,)16 b(where)g Fv(n)g Fx(is)f(the)h(n)o(um)o(b)q(er)f(of)g(pro)q(cesses)g(in)h(this)g(group.) 166 1045 y(Note)i(the)i(asymmetry)d(b)q(et)o(w)o(een)j(send)f(and)g (receiv)o(e)h(op)q(erations:)28 b(A)19 b(receiv)o(e)h(op)q(eration)f (ma)o(y)75 1102 y(accept)13 b(messages)f(from)g(an)h(arbitrary)f (sender,)i(on)f(the)g(other)f(hand,)h(a)g(send)g(op)q(eration)g(m)o (ust)g(sp)q(ecify)75 1158 y(a)i(unique)j(receiv)o(er.)k(This)16 b(matc)o(hes)f(a)h(\\push")f(comm)o(unication)i(mec)o(hanism,)f(where)g (data)f(transfer)75 1215 y(is)i(e\013ected)h(b)o(y)f(the)g(sender)g (\(rather)f(than)h(a)g(\\pull")h(mec)o(hanism,)g(where)f(data)f (transfer)g(is)i(e\013ected)75 1271 y(b)o(y)d(the)g(receiv)o(er\).)166 1328 y(Source)g(=)h(destination)g(is)g(allo)o(w)o(ed,)f(that)f(is,)i(a) e(pro)q(cess)i(can)f(send)h(a)f(message)g(to)f(itself.)21 b(\(Ho)o(w-)75 1384 y(ev)o(er,)c(it)h(is)g(unsafe)f(to)g(do)g(so)g (with)h(the)g(blo)q(c)o(king)g(send)g(and)g(receiv)o(e)g(op)q(erations) f(describ)q(ed)j(ab)q(o)o(v)o(e,)75 1441 y(since)c(this)g(ma)o(y)e (lead)i(to)f(deadlo)q(c)o(k.)21 b(See)16 b(Sec.)k(3.5.\))189 1538 y Fl(A)n(dvic)n(e)15 b(to)i(implementors.)43 b Fx(Message)15 b(con)o(text)g(and)h(other)g(comm)o(unicator)f(information)h(can)189 1594 y(b)q(e)k(implemen)o(ted)h(as)e(an)h(additional)h(tag)d(\014eld.) 34 b(It)20 b(di\013ers)g(from)f(the)g(regular)h(message)f(tag)189 1650 y(in)i(that)e(wild)j(card)e(matc)o(hing)h(is)f(not)g(allo)o(w)o (ed)h(on)f(this)h(\014eld,)i(and)d(that)g(v)m(alue)h(setting)g(for)189 1707 y(this)e(\014eld)h(is)g(con)o(trolled)g(b)o(y)f(comm)o(unicator)f (manipulation)j(functions.)32 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to) 189 1763 y(implementors.)p Fx(\))75 1884 y Fj(3.2.5)49 b(Return)16 b(status)75 1970 y Fx(The)g(source)f(or)g(tag)f(of)h(a)g (receiv)o(ed)h(message)f(ma)o(y)g(not)g(b)q(e)g(kno)o(wn)h(if)f (wildcard)i(v)m(alues)f(w)o(ere)f(used)h(in)75 2026 y(the)h(receiv)o(e) i(op)q(eration.)26 b(The)18 b(information)f(is)h(returned)g(b)o(y)f (the)h Fv(status)h Fx(argumen)o(t)e(of)g Fv(MPI)p 1734 2026 V 16 w(RECV)p Fx(.)75 2083 y(The)h(t)o(yp)q(e)f(of)h Fv(status)h Fx(is)f Fv(MPI)p Fx(-de\014ned.)28 b(Status)17 b(v)m(ariables)i(need)f(to)f(b)q(e)i(explicitly)h(allo)q(cated)e(b)o(y) g(the)75 2139 y(user,)d(that)f(is,)i(they)f(are)g(not)g(system)g(ob)s (jects.)166 2195 y(In)c(C,)f Fv(status)j Fx(is)e(a)g(structure)f(that)g (con)o(tains)h(t)o(w)o(o)e(\014elds)j(named)f Fi(MPI)p 1343 2195 13 2 v 14 w(SOURCE)e Fx(and)i Fi(MPI)p 1687 2195 V 15 w(T)m(A)o(G)p Fx(,)e(and)75 2252 y(the)k(structure)f(ma)o(y)g (con)o(tain)h(additional)h(\014elds.)20 b(Th)o(us,)13 b Fv(status.MPI)p 1277 2252 14 2 v 17 w(SOURCE)h Fx(and)f Fv(status.MPI)p 1774 2252 V 18 w(T)l(A)o(G)75 2308 y Fx(con)o(tain)i(the)h(source)f(and)g(tag,)f(resp)q(ectiv)o(ely)l(,)j (of)e(the)g(receiv)o(ed)h(message.)166 2365 y(In)f(F)l(ortran,)f Fv(status)i Fx(is)g(an)e(arra)o(y)g(of)g Fu(INTEGER)p Fx(s)g(of)g(size)i Fi(MPI)p 1219 2365 13 2 v 14 w(ST)m(A)m(TUS)p 1386 2365 V 14 w(SIZE)p Fx(.)e(The)h(t)o(w)o(o)f(constan)o(ts)75 2421 y Fi(MPI)p 152 2421 V 14 w(SOURCE)e Fx(and)i Fi(MPI)p 502 2421 V 14 w(T)m(A)o(G)f Fx(are)g(the)h(indices)i(of)d(the)g(en)o (tries)h(that)f(store)g(the)h(source)g(and)f(tag)g(\014elds.)75 2478 y(Th)o(us)k Fv(status\(MPI)p 406 2478 14 2 v 17 w(SOURCE\))h Fx(and)e Fv(status\(MPI)p 933 2478 V 18 w(T)l(A)o(G\))h Fx(con)o(tain,)f(resp)q(ectiv)o(ely)l(,)j(the)d(source) h(and)g(the)75 2534 y(tag)d(of)h(the)g(receiv)o(ed)i(message.)166 2591 y(The)h(status)e(argumen)o(t)h(also)g(returns)g(information)h(on)f (the)h(length)g(of)f(the)g(message)g(receiv)o(ed.)75 2647 y(Ho)o(w)o(ev)o(er,)12 b(this)i(information)f(is)h(not)f(directly) h(a)o(v)m(ailable)h(as)e(a)g(\014eld)h(of)f(the)h(status)e(v)m(ariable) j(and)e(a)g(call)75 2704 y(to)i Fv(MPI)p 216 2704 V 15 w(GET)p 319 2704 V 17 w(COUNT)h Fx(is)g(required)g(to)f(\\deco)q(de")g (this)h(information.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 29 21 28 bop 75 -100 a Fm(3.3.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) g(AND)g(D)o(A)l(T)l(A)g(CONVERSION)520 b Fx(21)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GET)p 264 45 V 17 w(COUNT\(status,)17 b(datat)o(yp)q(e,)f(count\))117 122 y Fk(IN)155 b Fv(status)476 b Fk(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\)) 117 197 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)e(\(handle\))117 273 y(OUT)108 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(elemen)o(ts)f(\(in)o(teger\))75 397 y Fu(int)23 b(MPI)p 245 397 15 2 v 17 w(Get)p 334 397 V 17 w(count\(MPI)p 567 397 V 16 w(Status)g(status,)g(MPI)p 1013 397 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 483 y(MPI)p 150 483 V 17 w(GET)p 239 483 V 17 w(COUNT\(STATUS,)e (DATATYPE,)g(COUNT,)h(IERROR\))170 540 y(INTEGER)g(STATUS\(MPI)p 604 540 V 16 w(STATUS)p 764 540 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 626 y Fx(Returns)16 b(the)f(n)o(um)o(b)q(er)h(of)f(elemen)o (ts)h(receiv)o(ed.)21 b(\(Again,)15 b(w)o(e)g(coun)o(t)h Fl(elements)p Fx(,)d(not)i Fl(bytes)p Fx(.\))20 b(The)75 683 y Fv(datat)o(yp)q(e)h Fx(argumen)o(t)e(should)i(matc)o(h)e(the)h (argumen)o(t)f(pro)o(vided)i(b)o(y)f(the)g(receiv)o(e)g(call)h(that)e (set)h(the)75 739 y Fv(status)f Fx(v)m(ariable.)26 b(\(W)l(e)17 b(shall)h(later)f(see,)g(in)h(Section)g(3.12.5,)d(that)h Fv(MPI)p 1347 739 14 2 v 16 w(GET)p 1451 739 V 16 w(COUNT)i Fx(ma)o(y)e(return,)75 796 y(in)g(certain)g(situations,)f(the)g(v)m (alue)i Fi(MPI)p 773 796 13 2 v 14 w(UNDEFINED)p Fx(.\))189 902 y Fl(R)n(ationale.)56 b Fx(Some)18 b(message)f(passing)h(libraries) i(use)e Fk(INOUT)h Fv(count)p Fx(,)g Fv(tag)f Fx(and)g Fv(source)g Fx(argu-)189 958 y(men)o(ts,)f(th)o(us)h(using)g(them)g(b)q (oth)g(to)f(sp)q(ecify)i(the)f(selection)h(criteria)f(for)f(incoming)i (messages)189 1015 y(and)c(return)g(the)h(actual)f(en)o(v)o(elop)q(e)i (v)m(alues)f(of)f(the)g(receiv)o(ed)i(message.)i(The)d(use)g(of)f(a)f (separate)189 1071 y(status)g(argumen)o(t)h(prev)o(en)o(ts)g(errors)g (that)g(are)g(often)g(attac)o(hed)g(with)h Fk(INOUT)g Fx(argumen)o(t)f(\(e.g.,)189 1128 y(using)i(the)g Fi(MPI)p 466 1128 V 14 w(ANY)p 565 1128 V 15 w(T)m(A)o(G)e Fx(constan)o(t)h(as)g (the)g(tag)g(in)h(a)g(send\).)24 b(Some)16 b(libraries)i(use)f(calls)h (that)189 1184 y(refer)d(implicitly)j(to)d(the)g(\\last)g(message)f (receiv)o(ed.")21 b(This)16 b(is)g(not)e(thread)i(safe.)189 1259 y(The)e Fv(datat)o(yp)q(e)i Fx(argumen)o(t)e(is)h(passed)f(to)g Fv(MPI)p 991 1259 14 2 v 16 w(GET)p 1095 1259 V 17 w(COUNT)h Fx(so)f(as)g(to)g(impro)o(v)o(e)g(p)q(erformance.)189 1316 y(A)k(message)g(migh)o(t)g(b)q(e)h(receiv)o(ed)h(without)e(coun)o (ting)h(the)g(n)o(um)o(b)q(er)g(of)e(elemen)o(ts)j(it)e(con)o(tains,) 189 1372 y(and)g(the)h(coun)o(t)f(v)m(alue)h(is)g(often)f(not)g (needed.)31 b(Also,)19 b(this)g(allo)o(ws)g(the)f(same)g(function)i(to) d(b)q(e)189 1429 y(used)e(after)g(a)g(call)h(to)f Fv(MPI)p 662 1429 V 15 w(PROBE)p Fx(.)i(\()p Fl(End)e(of)i(r)n(ationale.)p Fx(\))166 1535 y(All)i(send)g(and)g(receiv)o(e)g(op)q(erations)f(use)h (the)f Fv(buf)p Fx(,)h Fv(count)p Fx(,)h Fv(datat)o(yp)q(e)p Fx(,)g Fv(source)p Fx(,)f Fv(dest)p Fx(,)h Fv(tag)p Fx(,)f Fv(comm)75 1592 y Fx(and)12 b Fv(status)i Fx(argumen)o(ts)d(in)h(the)g (same)g(w)o(a)o(y)f(as)g(the)h(blo)q(c)o(king)h Fv(MPI)p 1216 1592 V 16 w(SEND)f Fx(and)g Fv(MPI)p 1528 1592 V 16 w(RECV)h Fx(op)q(erations)75 1648 y(describ)q(ed)k(in)f(this)g (section.)75 1791 y Fn(3.3)59 b(Data)19 b(t)n(yp)r(e)g(matching)f(and)h (data)g(conversion)75 1894 y Fj(3.3.1)49 b(T)l(yp)q(e)17 b(matching)d(rules)75 1980 y Fx(One)i(can)f(think)h(of)f(message)g (transfer)f(as)h(consisting)h(of)f(the)g(follo)o(wing)h(three)f (phases.)131 2074 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i (send)f(bu\013er)h(and)f(a)g(message)g(is)g(assem)o(bled.)131 2168 y(2.)22 b(A)15 b(message)g(is)g(transferred)g(from)f(sender)i(to)f (receiv)o(er.)131 2262 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h (incoming)i(message)e(and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o (e)h(bu\013er.)166 2355 y(T)o(yp)q(e)k(matc)o(hing)f(has)h(to)e(b)q(e)j (observ)o(ed)e(at)g(eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f(eac)o(h)75 2412 y(v)m(ariable)i(in)f(the)g (sender)g(bu\013er)g(has)g(to)f(matc)o(h)g(the)g(t)o(yp)q(e)h(sp)q (eci\014ed)i(for)d(that)g(en)o(try)g(b)o(y)h(the)f(send)75 2468 y(op)q(eration;)d(the)f(t)o(yp)q(e)h(sp)q(eci\014ed)i(b)o(y)e(the) f(send)i(op)q(eration)e(has)h(to)f(matc)o(h)g(the)h(t)o(yp)q(e)f(sp)q (eci\014ed)j(b)o(y)e(the)75 2525 y(receiv)o(e)g(op)q(eration;)e(and)h (the)g(t)o(yp)q(e)f(of)h(eac)o(h)f(v)m(ariable)i(in)g(the)e(receiv)o(e) i(bu\013er)f(has)f(to)g(matc)o(h)g(the)h(t)o(yp)q(e)75 2581 y(sp)q(eci\014ed)f(for)e(that)g(en)o(try)g(b)o(y)g(the)g(receiv)o (e)h(op)q(eration.)20 b(A)12 b(program)f(that)h(fails)h(to)e(observ)o (e)h(these)h(three)75 2638 y(rules)j(is)g(erroneous.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 30 22 29 bop 75 -100 a Fx(22)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fx(T)l(o)e(de\014ne)i (t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l(,)j(w)o(e)d(need)i(to)e (deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 102 y(t)o(yp)q(es)e(of)g(the)h(host)f(language)g(with)h(t)o(yp) q(es)g(sp)q(eci\014ed)h(in)f(comm)o(unication)h(op)q(erations;)e(and)h (matc)o(hing)75 158 y(of)h(t)o(yp)q(es)g(at)g(sender)g(and)h(receiv)o (er.)166 214 y(The)k(t)o(yp)q(es)f(of)h(a)f(send)h(and)g(receiv)o(e)h (matc)o(h)e(\(phase)g(t)o(w)o(o\))f(if)j(b)q(oth)e(op)q(erations)h(use) g(iden)o(tical)75 271 y(names.)36 b(That)20 b(is,)i Fv(MPI)p 517 271 14 2 v 16 w(INTEGER)f Fx(matc)o(hes)f Fv(MPI)p 1006 271 V 16 w(INTEGER)p Fx(,)h Fv(MPI)p 1327 271 V 16 w(REAL)g Fx(matc)o(hes)f Fv(MPI)p 1738 271 V 16 w(REAL)p Fx(,)75 327 y(and)12 b(so)g(on.)18 b(There)12 b(is)h(one)f(exception)h (to)e(this)h(rule,)h(discussed)g(in)g(Sec.)20 b(3.13,)10 b(the)i(t)o(yp)q(e)g Fv(MPI)p 1687 327 V 16 w(P)l(A)o(CKED)75 384 y Fx(can)j(matc)o(h)g(an)o(y)g(other)g(t)o(yp)q(e.)166 440 y(The)k(t)o(yp)q(e)g(of)f(a)g(v)m(ariable)i(in)g(a)e(host)h (program)e(matc)o(hes)h(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)i(in)e(the)g (comm)o(u-)75 497 y(nication)g(op)q(eration)g(if)f(the)h(datat)o(yp)q (e)f(name)g(used)h(b)o(y)f(that)f(op)q(eration)i(corresp)q(onds)f(to)g (the)g(basic)75 553 y(t)o(yp)q(e)f(of)g(the)g(host)g(program)f(v)m (ariable.)27 b(F)l(or)17 b(example,)h(an)f(en)o(try)g(with)g(t)o(yp)q (e)g(name)g Fv(MPI)p 1672 553 V 16 w(INTEGER)75 610 y Fx(matc)o(hes)c(a)g(F)l(ortran)g(v)m(ariable)i(of)e(t)o(yp)q(e)g Fu(INTEGER)p Fx(.)f(A)i(table)g(giving)g(this)g(corresp)q(ondence)h (for)e(F)l(ortran)75 666 y(and)20 b(C)f(app)q(ears)h(in)h(Sec.)33 b(3.2.2.)f(There)20 b(are)f(t)o(w)o(o)g(exceptions)h(to)f(this)i(last)e (rule:)30 b(an)19 b(en)o(try)h(with)75 723 y(t)o(yp)q(e)f(name)g Fv(MPI)p 391 723 V 16 w(BYTE)h Fx(or)f Fv(MPI)p 687 723 V 16 w(P)l(A)o(CKED)g Fx(can)g(b)q(e)h(used)g(to)e(matc)o(h)h(an)o(y)g (b)o(yte)g(of)g(storage)e(\(on)i(a)75 779 y(b)o(yte-addressable)g(mac)o (hine\),)g(irresp)q(ectiv)o(e)h(of)e(the)g(datat)o(yp)q(e)g(of)g(the)g (v)m(ariable)i(that)d(con)o(tains)i(this)75 835 y(b)o(yte.)f(The)12 b(t)o(yp)q(e)g Fv(MPI)p 466 835 V 16 w(P)l(A)o(CKED)f Fx(is)h(used)g(to)f(send)i(data)e(that)f(has)i(b)q(een)h(explicitly)h (pac)o(k)o(ed,)e(or)f(receiv)o(e)75 892 y(data)k(that)g(will)i(b)q(e)g (explicitly)h(unpac)o(k)o(ed,)e(see)g(Section)g(3.13.)k(The)c(t)o(yp)q (e)g Fv(MPI)p 1468 892 V 16 w(BYTE)g Fx(allo)o(ws)g(one)g(to)75 948 y(transfer)e(the)i(binary)g(v)m(alue)g(of)f(a)g(b)o(yte)g(in)h (memory)e(unc)o(hanged.)166 1005 y(T)l(o)h(summarize,)g(the)g(t)o(yp)q (e)h(matc)o(hing)f(rules)h(fall)g(in)o(to)f(the)g(three)h(categories)f (b)q(elo)o(w.)143 1095 y Ft(\017)23 b Fx(Comm)o(unication)11 b(of)f(t)o(yp)q(ed)i(v)m(alues)g(\(e.g.,)e(with)h(datat)o(yp)q(e)g (di\013eren)o(t)g(from)21 b Fv(MPI)p 1586 1095 V 16 w(BYTE)p Fx(\),)11 b(where)189 1152 y(the)k(datat)o(yp)q(es)f(of)g(the)h (corresp)q(onding)h(en)o(tries)f(in)h(the)f(sender)g(program,)e(in)j (the)f(send)g(call,)h(in)189 1208 y(the)f(receiv)o(e)h(call)g(and)g(in) g(the)f(receiv)o(er)h(program)e(m)o(ust)h(all)h(matc)o(h.)143 1301 y Ft(\017)23 b Fx(Comm)o(unication)16 b(of)g(un)o(t)o(yp)q(ed)h(v) m(alues)g(\(e.g.,)e(of)g(datat)o(yp)q(e)h Fv(MPI)p 1330 1301 V 16 w(BYTE)p Fx(\),)g(where)g(b)q(oth)g(sender)189 1357 y(and)g(receiv)o(er)h(use)f(the)g(datat)o(yp)q(e)g Fv(MPI)p 881 1357 V 15 w(BYTE)p Fx(.)h(In)f(this)h(case,)f(there)g(are) g(no)f(requiremen)o(ts)i(on)189 1414 y(the)d(t)o(yp)q(es)h(of)f(the)g (corresp)q(onding)h(en)o(tries)g(in)h(the)e(sender)h(and)g(the)f (receiv)o(er)h(programs,)e(nor)h(is)189 1470 y(it)h(required)h(that)f (they)g(b)q(e)h(the)f(same.)143 1563 y Ft(\017)23 b Fx(Comm)o (unication)15 b(in)o(v)o(olving)i(pac)o(k)o(ed)e(data,)f(where)h Fv(MPI)p 1195 1563 V 16 w(P)l(A)o(CKED)g Fx(is)h(used.)166 1653 y(The)f(follo)o(wing)h(examples)g(illustrate)h(the)e(\014rst)g(t)o (w)o(o)e(cases.)75 1755 y Fq(Example)18 b(3.1)k Fx(Sender)16 b(and)g(receiv)o(er)g(sp)q(ecify)g(matc)o(hing)g(t)o(yp)q(es.)75 1845 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1902 y(IF\(rank.EQ.0\))123 1958 y(THEN)170 2015 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 2071 y(ELSE)170 2128 y(CALL)h(MPI_RECV\(b\(1\),)e(15,)h(MPI_REAL,)g(0,) g(tag,)h(comm,)f(status,)g(ierr\))75 2184 y(END)g(IF)166 2274 y Fx(This)16 b(co)q(de)h(is)f(correct)f(if)h(b)q(oth)g Fv(a)g Fx(and)g Fv(b)g Fx(are)f(real)h(arra)o(ys)f(of)g(size)i Ft(\025)d Fx(10.)21 b(\(In)16 b(F)l(ortran,)e(it)i(migh)o(t)75 2331 y(b)q(e)e(correct)e(to)h(use)g(this)g(co)q(de)h(ev)o(en)f(if)h Fv(a)f Fx(or)f Fv(b)i Fx(ha)o(v)o(e)e(size)i Fp(<)f Fx(10:)19 b(e.g.,)12 b(when)h Fv(a\(1\))f Fx(can)h(b)q(e)h(equiv)m(alenced)75 2387 y(to)h(an)g(arra)o(y)f(with)h(ten)g(reals.\))75 2489 y Fq(Example)j(3.2)k Fx(Sender)16 b(and)g(receiv)o(er)g(do)f(not)g (sp)q(ecify)h(matc)o(hing)f(t)o(yp)q(es.)75 2591 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2647 y(IF\(rank.EQ.0\))123 2704 y(THEN)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 31 23 30 bop 75 -100 a Fm(3.3.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) g(AND)g(D)o(A)l(T)l(A)g(CONVERSION)520 b Fx(23)170 45 y Fu(CALL)24 b(MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,) f(ierr\))123 102 y(ELSE)170 158 y(CALL)h(MPI_RECV\(b\(1\),)e(40,)h (MPI_BYTE,)g(0,)g(tag,)h(comm,)f(status,)g(ierr\))75 214 y(END)g(IF)166 318 y Fx(This)d(co)q(de)g(is)f(erroneous,)h(since)g (sender)g(and)g(receiv)o(er)g(do)f(not)f(pro)o(vide)i(matc)o(hing)f (datat)o(yp)q(e)75 375 y(argumen)o(ts.)75 478 y Fq(Example)f(3.3)k Fx(Sender)16 b(and)g(receiv)o(er)g(sp)q(ecify)g(comm)o(unication)g(of)f (un)o(t)o(yp)q(ed)h(v)m(alues.)75 570 y Fu(CALL)23 b (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 627 y(IF\(rank.EQ.0\))123 683 y(THEN)170 740 y(CALL)h(MPI_SEND\(a\(1\),)e(40,)h(MPI_BYTE,)g(1,)g (tag,)h(comm,)f(ierr\))123 796 y(ELSE)170 853 y(CALL)h (MPI_RECV\(b\(1\),)e(60,)h(MPI_BYTE,)g(0,)g(tag,)h(comm,)f(status,)g (ierr\))75 909 y(END)g(IF)166 1001 y Fx(This)16 b(co)q(de)g(is)g (correct,)e(irresp)q(ectiv)o(e)j(of)e(the)h(t)o(yp)q(e)f(and)h(size)g (of)f Fv(a)g Fx(and)h Fv(b)f Fx(\(unless)i(this)e(results)h(in)75 1058 y(an)f(out)g(of)g(b)q(ound)h(memory)e(access\).)189 1161 y Fl(A)n(dvic)n(e)e(to)i(users.)38 b Fx(If)12 b(a)g(bu\013er)g(of) f(t)o(yp)q(e)h Fi(MPI)p 965 1161 13 2 v 15 w(BYTE)f Fx(is)i(passed)f (as)g(an)g(argumen)o(t)f(to)g Fv(MPI)p 1732 1161 14 2 v 16 w(SEND)p Fx(,)189 1218 y(then)j Fv(MPI)g Fx(will)i(send)f(the)g (data)f(stored)g(at)f(con)o(tiguous)i(lo)q(cations,)g(starting)f(from)f (the)i(address)189 1274 y(indicated)22 b(b)o(y)e(the)h Fv(buf)g Fx(argumen)o(t.)35 b(This)21 b(ma)o(y)f(ha)o(v)o(e)g(unexp)q (ected)i(results)f(when)g(the)g(data)189 1331 y(la)o(y)o(out)f(is)i (not)f(as)f(a)h(casual)h(user)f(w)o(ould)h(exp)q(ect)f(it)h(to)e(b)q (e.)39 b(F)l(or)20 b(example,)j(some)e(F)l(ortran)189 1387 y(compilers)e(implemen)o(t)g(v)m(ariables)g(of)f(t)o(yp)q(e)g Fi(CHARA)o(CTER)e Fx(as)i(a)g(structure)f(that)h(con)o(tains)g(the)189 1444 y(c)o(haracter)c(length)i(and)g(a)f(p)q(oin)o(ter)h(to)f(the)g (actual)h(string.)k(In)c(suc)o(h)g(an)f(en)o(vironmen)o(t,)h(sending) 189 1500 y(and)g(receiving)h(a)e(F)l(ortran)g Fi(CHARA)o(CTER)f Fx(v)m(ariable)j(using)g(the)e Fi(MPI)p 1378 1500 13 2 v 15 w(BYTE)g Fx(t)o(yp)q(e)h(will)h(not)e(ha)o(v)o(e)189 1557 y(the)f(an)o(ticipated)i(result)f(of)f(transferring)h(the)f(c)o (haracter)g(string.)20 b(F)l(or)14 b(this)h(reason,)f(the)h(user)f(is) 189 1613 y(advised)i(to)e(use)i(t)o(yp)q(ed)f(comm)o(unications)h (whenev)o(er)g(p)q(ossible.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)h(to)f (users.)p Fx(\))75 1733 y Fv(T)l(yp)q(e)g Fi(MPI)p 260 1733 V 15 w(CHARA)o(CTER)75 1819 y Fx(The)c(t)o(yp)q(e)g Fv(MPI)p 349 1819 14 2 v 16 w(CHARA)o(CTER)h Fx(matc)o(hes)f(one)g(c)o (haracter)f(of)g(a)h(F)l(ortran)f(v)m(ariable)i(of)f(t)o(yp)q(e)g Fu(CHARACTER)p Fx(,)75 1875 y(rather)20 b(then)g(the)h(en)o(tire)g(c)o (haracter)e(string)i(stored)f(in)h(the)f(v)m(ariable.)37 b(F)l(ortran)19 b(v)m(ariables)j(of)e(t)o(yp)q(e)75 1932 y Fi(CHARA)o(CTER)g Fx(or)i(substrings)g(are)f(transferred)h(as)f(if)i (they)f(w)o(ere)g(arra)o(ys)e(of)i(c)o(haracters.)39 b(This)22 b(is)75 1988 y(illustrated)17 b(in)f(the)f(example)h(b)q(elo) o(w.)75 2092 y Fq(Example)i(3.4)k Fx(T)l(ransfer)15 b(of)g(F)l(ortran)f Fi(CHARA)o(CTER)p Fx(s.)75 2195 y Fu(CHARACTER*10)22 b(a)75 2252 y(CHARACTER*10)g(b)75 2365 y(CALL)h(MPI_COMM_RANK\(comm,)f (rank,)h(ierr\))75 2421 y(IF\(rank.EQ.0\))123 2478 y(THEN)170 2534 y(CALL)h(MPI_SEND\(a,)e(5,)i(MPI_CHARACTER,)e(1,)h(tag,)h(comm,)f (ierr\))123 2591 y(ELSE)170 2647 y(CALL)h(MPI_RECV\(b\(6:10\),)d(5,)j (MPI_CHARACTER,)e(0,)h(tag,)h(comm,)f(status,)g(ierr\))75 2704 y(END)g(IF)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 32 24 31 bop 75 -100 a Fx(24)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fx(The)d(last)h(\014v)o (e)f(c)o(haracters)g(of)f(string)i Fi(b)f Fx(at)g(pro)q(cess)g(1)g(are) g(replaced)i(b)o(y)e(the)g(\014rst)g(\014v)o(e)h(c)o(haracters)75 102 y(of)g(string)g Fi(a)g Fx(at)g(pro)q(cess)g(0.)189 208 y Fl(R)n(ationale.)45 b Fx(The)16 b(alternativ)o(e)h(c)o(hoice)g(w) o(ould)f(b)q(e)h(for)e Fv(MPI)p 1246 208 14 2 v 16 w(CHARA)o(CTER)i Fx(to)f(matc)o(h)f(a)h(c)o(har-)189 264 y(acter)e(of)h(arbitrary)g (length.)20 b(This)c(runs)f(in)o(to)h(problems.)189 339 y(A)g(F)l(ortran)e(c)o(haracter)h(v)m(ariable)j(is)e(a)g(constan)o(t)f (length)h(string,)g(with)g(no)g(sp)q(ecial)i(termination)189 396 y(sym)o(b)q(ol.)h(There)12 b(is)h(no)g(\014xed)f(con)o(v)o(en)o (tion)h(on)f(ho)o(w)g(to)g(represen)o(t)g(c)o(haracters,)g(and)g(ho)o (w)g(to)g(store)189 452 y(their)h(length.)20 b(Some)13 b(compilers)h(pass)e(a)h(c)o(haracter)f(argumen)o(t)g(to)g(a)h(routine) g(as)f(a)h(pair)g(of)f(argu-)189 509 y(men)o(ts,)g(one)g(holding)h(the) f(address)h(of)e(the)h(string)g(and)h(the)f(other)f(holding)j(the)e (length)h(of)e(string.)189 565 y(Consider)h(the)f(case)h(of)f(an)g Fv(MPI)g Fx(comm)o(unication)h(call)h(that)e(is)h(passed)g(a)f(comm)o (unication)h(bu\013er)189 622 y(with)17 b(t)o(yp)q(e)f(de\014ned)i(b)o (y)f(a)f(deriv)o(ed)h(datat)o(yp)q(e)f(\(Section)i(3.12\).)k(If)17 b(this)g(comm)o(unicator)f(bu\013er)189 678 y(con)o(tains)e(v)m (ariables)h(of)e(t)o(yp)q(e)h Fu(CHARACTER)f Fx(then)h(the)g (information)g(on)g(their)h(length)f(will)i(not)d(b)q(e)189 735 y(passed)i(to)g(the)g Fv(MPI)g Fx(routine.)189 810 y(This)j(problem)g(forces)f(us)h(to)f(pro)o(vide)h(explicit)i (information)d(on)h(c)o(haracter)f(length)h(with)g(the)189 866 y Fv(MPI)d Fx(call.)21 b(One)c(could)f(add)g(a)f(length)h (parameter)f(to)g(the)g(t)o(yp)q(e)h Fv(MPI)p 1411 866 V 15 w(CHARA)o(CTER)p Fx(,)h(but)f(this)189 923 y(do)q(es)11 b(not)g(add)h(m)o(uc)o(h)f(con)o(v)o(enience)i(and)e(the)h(same)f (functionalit)o(y)h(can)g(b)q(e)g(ac)o(hiev)o(ed)g(b)o(y)f(de\014ning) 189 979 y(a)k(suitable)h(deriv)o(ed)g(datat)o(yp)q(e.)k(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 1085 y Fl(A)n(dvic)n(e)f(to) i(implementors.)48 b Fx(Some)16 b(compilers)i(pass)e(F)l(ortran)g Fi(CHARA)o(CTER)e Fx(argumen)o(ts)i(as)g(a)189 1142 y(structure)g(with) i(a)f(length)g(and)h(a)e(p)q(oin)o(ter)i(to)e(the)h(actual)h(string.)25 b(In)18 b(suc)o(h)f(an)g(en)o(vironmen)o(t,)189 1198 y(the)h Fv(MPI)g Fx(call)h(needs)h(to)d(dereference)j(the)e(p)q(oin)o (ter)h(in)g(order)g(to)e(reac)o(h)h(the)h(string.)29 b(\()p Fl(End)19 b(of)189 1255 y(advic)n(e)d(to)g(implementors.)p Fx(\))75 1376 y Fj(3.3.2)49 b(Data)17 b(conversion)75 1462 y Fx(One)g(of)e(the)h(goals)f(of)h Fv(MPI)f Fx(is)h(to)f(supp)q (ort)h(parallel)i(computations)d(across)g(heterogeneous)h(en)o(viron-) 75 1519 y(men)o(ts.)j(Comm)o(unication)d(in)g(a)f(heterogeneous)g(en)o (vironmen)o(t)g(ma)o(y)g(require)h(data)e(con)o(v)o(ersions.)20 b(W)l(e)75 1575 y(use)c(the)f(follo)o(wing)h(terminology)l(.)75 1669 y Fq(t)o(yp)q(e)h(con)o(v)o(ersion)22 b Fx(c)o(hanges)12 b(the)h(datat)o(yp)q(e)f(of)g(a)g(v)m(alue,)i(e.g.,)e(b)o(y)g(rounding) h(a)f Fu(REAL)g Fx(to)g(an)g Fu(INTEGER)p Fx(.)75 1763 y Fq(represen)o(tation)17 b(con)o(v)o(ersion)22 b Fx(c)o(hanges)16 b(the)g(binary)g(represen)o(tation)g(of)f(a)g(v)m(alue,)i(e.g.,)e(from) g(Hex)189 1819 y(\015oating)g(p)q(oin)o(t)g(to)g(IEEE)g(\015oating)h(p) q(oin)o(t.)166 1913 y(The)e(t)o(yp)q(e)f(matc)o(hing)h(rules)g(imply)h (that)e Fv(MPI)g Fx(comm)o(unication)h(nev)o(er)f(en)o(tails)i(t)o(yp)q (e)e(con)o(v)o(ersion.)75 1970 y(On)20 b(the)g(other)f(hand,)i Fv(MPI)e Fx(requires)h(that)f(a)g(represen)o(tation)h(con)o(v)o(ersion) f(b)q(e)h(p)q(erformed)g(when)g(a)75 2026 y(t)o(yp)q(ed)h(v)m(alue)h (is)f(transferred)f(across)g(en)o(vironmen)o(ts)h(that)f(use)h (di\013eren)o(t)f(represen)o(tations)h(for)f(the)75 2082 y(datat)o(yp)q(e)h(of)f(this)i(v)m(alue.)38 b Fv(MPI)21 b Fx(do)q(es)g(not)g(sp)q(ecify)h(rules)g(for)f(represen)o(tation)g (con)o(v)o(ersion.)37 b(Suc)o(h)75 2139 y(con)o(v)o(ersion)21 b(is)g(exp)q(ected)i(to)d(preserv)o(e)h(in)o(teger,)h(logical)g(or)e(c) o(haracter)h(v)m(alues,)i(and)e(to)f(con)o(v)o(ert)g(a)75 2195 y(\015oating)15 b(p)q(oin)o(t)h(v)m(alue)g(to)f(the)g(nearest)g(v) m(alue)h(that)f(can)g(b)q(e)h(represen)o(ted)g(on)f(the)g(target)f (system.)166 2252 y(Ov)o(er\015o)o(w)h(and)g(under\015o)o(w)g (exceptions)h(ma)o(y)f(o)q(ccur)g(during)h(\015oating)f(p)q(oin)o(t)h (con)o(v)o(ersions.)k(Con-)75 2308 y(v)o(ersion)h(of)g(in)o(tegers)g (or)g(c)o(haracters)f(ma)o(y)g(also)i(lead)f(to)g(exceptions)h(when)g (a)e(v)m(alue)j(that)d(can)h(b)q(e)75 2365 y(represen)o(ted)16 b(in)g(one)f(system)g(cannot)g(b)q(e)h(represen)o(ted)f(in)h(the)f (other)g(system.)20 b(An)15 b(exception)h(o)q(ccur-)75 2421 y(ring)i(during)g(represen)o(tation)g(con)o(v)o(ersion)f(results)h (in)h(a)e(failure)i(of)e(the)g(comm)o(unication.)28 b(An)18 b(error)75 2478 y(o)q(ccurs)e(either)f(in)h(the)g(send)g(op)q(eration,) f(or)f(the)i(receiv)o(e)g(op)q(eration,)f(or)f(b)q(oth.)166 2534 y(If)22 b(a)f(v)m(alue)i(sen)o(t)e(in)h(a)f(message)g(is)h(un)o(t) o(yp)q(ed)h(\(i.e.,)f(of)f(t)o(yp)q(e)h Fv(MPI)p 1370 2534 V 15 w(BYTE)p Fx(\),)g(then)g(the)f(binary)75 2591 y(represen)o(tation)e(of)f(the)h(b)o(yte)g(stored)f(at)g(the)h(receiv)o (er)h(is)f(iden)o(tical)i(to)d(the)h(binary)g(represen)o(tation)75 2647 y(of)d(the)h(b)o(yte)g(loaded)g(at)g(the)f(sender.)26 b(This)17 b(holds)h(true,)e(whether)h(sender)h(and)f(receiv)o(er)g(run) g(in)h(the)75 2703 y(same)i(or)f(in)i(distinct)g(en)o(vironmen)o(ts.)35 b(No)20 b(represen)o(tation)g(con)o(v)o(ersion)g(is)h(required.)35 b(\(Note)20 b(that)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 33 25 32 bop 75 -100 a Fm(3.4.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b Fx(25)75 45 y(represen)o(tation)15 b(con)o(v)o(ersion)g (ma)o(y)g(o)q(ccur)g(when)h(v)m(alues)g(of)e(t)o(yp)q(e)i Fv(MPI)p 1307 45 14 2 v 15 w(CHARA)o(CTER)h Fx(or)d Fv(MPI)p 1741 45 V 16 w(CHAR)75 102 y Fx(are)h(transferred,)f(for)h(example,)g (from)g(an)g(EBCDIC)g(enco)q(ding)i(to)d(an)h(ASCI)q(I)i(enco)q (ding.\))166 158 y(No)d(con)o(v)o(ersion)g(o)q(ccurs)g(when)g(an)g Fv(MPI)g Fx(program)f(executes)h(in)h(a)f(homogeneous)f(system,)h (where)75 214 y(all)i(pro)q(cesses)g(run)f(in)h(the)f(same)g(en)o (vironmen)o(t.)166 271 y(Consider)d(the)g(three)g(examples,)h(3.1{3.3.) j(The)c(\014rst)g(program)e(is)j(correct,)e(assuming)h(that)f Fv(a)h Fx(and)75 327 y Fv(b)i Fx(are)g Fu(REAL)f Fx(arra)o(ys)g(of)g (size)i Ft(\025)e Fx(10.)19 b(If)14 b(the)g(sender)g(and)g(receiv)o(er) h(execute)f(in)h(di\013eren)o(t)f(en)o(vironmen)o(ts,)75 384 y(then)21 b(the)g(ten)h(real)f(v)m(alues)h(that)e(are)h(fetc)o(hed) g(from)g(the)g(send)g(bu\013er)g(will)i(b)q(e)f(con)o(v)o(erted)e(to)h (the)75 440 y(represen)o(tation)g(for)f(reals)g(on)h(the)g(receiv)o(er) g(site)g(b)q(efore)g(they)f(are)h(stored)f(in)h(the)g(receiv)o(e)g (bu\013er.)75 497 y(While)e(the)e(n)o(um)o(b)q(er)h(of)f(real)g(elemen) o(ts)h(fetc)o(hed)g(from)f(the)g(send)h(bu\013er)f(equal)i(the)e(n)o (um)o(b)q(er)h(of)f(real)75 553 y(elemen)o(ts)d(stored)e(in)i(the)f (receiv)o(e)h(bu\013er,)f(the)g(n)o(um)o(b)q(er)g(of)g(b)o(ytes)g (stored)f(need)i(not)f(equal)h(the)f(n)o(um)o(b)q(er)75 610 y(of)e(b)o(ytes)g(loaded.)20 b(F)l(or)11 b(example,)i(the)e(sender) h(ma)o(y)f(use)h(a)g(four)f(b)o(yte)g(represen)o(tation)h(and)g(the)f (receiv)o(er)75 666 y(an)k(eigh)o(t)h(b)o(yte)f(represen)o(tation)g (for)f(reals.)166 723 y(The)h(second)h(program)e(is)i(erroneous,)f(and) g(its)g(b)q(eha)o(vior)h(is)g(unde\014ned.)166 779 y(The)e(third)g (program)e(is)i(correct.)19 b(The)13 b(exact)g(same)g(sequence)i(of)e (fort)o(y)f(b)o(ytes)h(that)g(w)o(ere)g(loaded)75 835 y(from)i(the)h(send)g(bu\013er)g(will)h(b)q(e)g(stored)e(in)h(the)g (receiv)o(e)h(bu\013er,)e(ev)o(en)h(if)g(sender)h(and)f(receiv)o(er)g (run)g(in)75 892 y(a)g(di\013eren)o(t)g(en)o(vironmen)o(t.)22 b(The)17 b(message)e(sen)o(t)h(has)g(exactly)g(the)g(same)g(length)h (\(in)f(b)o(ytes\))g(and)g(the)75 948 y(same)g(binary)g(represen)o (tation)g(as)g(the)g(message)f(receiv)o(ed.)24 b(If)16 b Fv(a)f Fx(and)i Fv(b)f Fx(are)g(of)f(di\013eren)o(t)h(t)o(yp)q(es,)g (or)f(if)75 1005 y(they)h(are)f(of)h(the)g(same)f(t)o(yp)q(e)h(but)g (di\013eren)o(t)g(data)f(represen)o(tations)h(are)g(used,)g(then)g(the) g(bits)g(stored)75 1061 y(in)h(the)f(receiv)o(e)h(bu\013er)g(ma)o(y)e (enco)q(de)i(v)m(alues)h(that)d(are)h(di\013eren)o(t)g(from)g(the)g(v)m (alues)h(they)g(enco)q(ded)g(in)75 1118 y(the)e(send)h(bu\013er.)166 1174 y(Data)e(represen)o(tation)i(con)o(v)o(ersion)f(also)h(applies)h (to)e(the)g(en)o(v)o(elop)q(e)i(of)e(a)g(message:)20 b(source,)15 b(des-)75 1231 y(tination)h(and)f(tag)f(are)h(all)h(in)o (tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i(con)o(v)o(erted.)189 1322 y Fl(A)n(dvic)n(e)g(to)i(implementors.)50 b Fx(The)17 b(curren)o(t)f(de\014nition)j(do)q(es)e(not)g(require)g(messages)f(to)g (carry)189 1379 y(data)h(t)o(yp)q(e)i(information.)29 b(Both)18 b(sender)h(and)f(receiv)o(er)h(pro)o(vide)g(complete)g(data)e (t)o(yp)q(e)i(infor-)189 1435 y(mation.)24 b(In)17 b(a)g(heterogeneous) f(en)o(vironmen)o(t,)h(one)g(can)g(either)h(use)f(a)f(mac)o(hine)i (indep)q(enden)o(t)189 1492 y(enco)q(ding)h(suc)o(h)f(as)g(XDR,)f(or)h (ha)o(v)o(e)f(the)h(receiv)o(er)g(con)o(v)o(ert)g(from)f(the)h(sender)g (represen)o(tation)189 1548 y(to)c(its)i(o)o(wn,)e(or)h(ev)o(en)g(ha)o (v)o(e)g(the)g(sender)h(do)f(the)g(con)o(v)o(ersion.)189 1621 y(Additional)20 b(t)o(yp)q(e)e(information)g(migh)o(t)g(b)q(e)h (added)g(to)f(messages)g(in)h(order)f(to)f(allo)o(w)i(the)f(sys-)189 1678 y(tem)g(to)f(detect)i(mismatc)o(hes)f(b)q(et)o(w)o(een)g(datat)o (yp)q(e)g(at)g(sender)g(and)h(receiv)o(er.)29 b(This)19 b(migh)o(t)f(b)q(e)189 1734 y(particularly)13 b(useful)g(in)f(a)g(slo)o (w)o(er)f(but)h(safer)g(debug)g(mo)q(de.)19 b(\()p Fl(End)13 b(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))166 1826 y Fv(MPI)19 b Fx(do)q(es)g(not)g(require)h(supp)q(ort)g(for)e(in)o (ter-language)i(comm)o(unication.)33 b(The)20 b(b)q(eha)o(vior)f(of)g (a)75 1882 y(program)12 b(is)i(unde\014ned)h(if)e(messages)g(are)g(sen) o(t)g(b)o(y)g(a)g(C)g(pro)q(cess)g(and)h(receiv)o(ed)g(b)o(y)f(a)g(F)l (ortran)f(pro)q(cess,)75 1939 y(or)j(vice-v)o(ersa.)189 2030 y Fl(R)n(ationale.)39 b Fv(MPI)15 b Fx(do)q(es)g(not)f(handle)i (in)o(ter-language)f(comm)o(unication)h(b)q(ecause)g(there)f(are)f(no) 189 2087 y(agreed)h(standards)g(for)g(the)g(corresp)q(ondence)i(b)q(et) o(w)o(een)f(C)f(t)o(yp)q(es)g(and)h(F)l(ortran)e(t)o(yp)q(es.)21 b(There-)189 2143 y(fore,)14 b Fv(MPI)h Fx(programs)f(that)g(mix)i (languages)f(w)o(ould)h(not)e(p)q(ort.)20 b(\()p Fl(End)c(of)g(r)n (ationale.)p Fx(\))189 2235 y Fl(A)n(dvic)n(e)k(to)h(implementors.)71 b Fv(MPI)20 b Fx(implemen)o(tors)h(ma)o(y)f(w)o(an)o(t)f(to)h(supp)q (ort)g(in)o(ter-language)189 2292 y(comm)o(unication)13 b(b)o(y)g(allo)o(wing)g(F)l(ortran)f(programs)f(to)h(use)h(\\C)f(MPI)h (t)o(yp)q(es,")g(suc)o(h)g(as)f Fi(MPI)p 1782 2292 13 2 v 14 w(INT,)189 2348 y(MPI)p 266 2348 V 14 w(CHAR)p Fx(,)19 b(etc.,)h(and)f(allo)o(wing)i(C)e(programs)g(to)g(use)h(F)l (ortran)e(t)o(yp)q(es.)33 b(\()p Fl(End)20 b(of)h(advic)n(e)f(to)189 2404 y(implementors.)p Fx(\))75 2546 y Fn(3.4)59 b(Communication)18 b(Mo)r(des)75 2647 y Fx(The)e(send)h(call)h(describ)q(ed)g(in)f (Section)g(3.2.1)d(is)j Fq(blo)q(c)o(king)p Fx(:)23 b(it)16 b(do)q(es)h(not)f(return)g(un)o(til)h(the)f(message)75 2704 y(data)j(and)g(en)o(v)o(elop)q(e)h(ha)o(v)o(e)f(b)q(een)h(safely)g (stored)f(a)o(w)o(a)o(y)e(so)i(that)g(the)g(sender)h(is)g(free)f(to)f (access)i(and)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 34 26 33 bop 75 -100 a Fx(26)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(o)o(v)o(erwrite)e (the)g(send)h(bu\013er.)k(The)c(message)f(migh)o(t)g(b)q(e)h(copied)g (directly)h(in)o(to)e(the)h(matc)o(hing)f(receiv)o(e)75 102 y(bu\013er,)g(or)f(it)i(migh)o(t)f(b)q(e)h(copied)g(in)o(to)f(a)g (temp)q(orary)g(system)g(bu\013er.)166 158 y(Message)c(bu\013ering)i (decouples)g(the)f(send)h(and)f(receiv)o(e)h(op)q(erations.)19 b(A)12 b(blo)q(c)o(king)h(send)g(can)f(com-)75 214 y(plete)i(as)e(so)q (on)h(as)g(the)f(message)h(w)o(as)f(bu\013ered,)h(ev)o(en)h(if)f(no)g (matc)o(hing)g(receiv)o(e)g(has)g(b)q(een)h(executed)g(b)o(y)75 271 y(the)f(receiv)o(er.)19 b(On)14 b(the)e(other)g(hand,)i(message)e (bu\013ering)h(can)g(b)q(e)g(exp)q(ensiv)o(e,)h(as)e(it)h(en)o(tails)h (additional)75 327 y(memory-to-memory)j(cop)o(ying,)h(and)g(it)g (requires)h(the)f(allo)q(cation)h(of)e(memory)h(for)f(bu\013ering.)29 b Fv(MPI)75 384 y Fx(o\013ers)12 b(the)i(c)o(hoice)g(of)f(sev)o(eral)g (comm)o(unication)i(mo)q(des)e(that)g(allo)o(w)g(one)h(to)e(con)o(trol) h(the)h(c)o(hoice)g(of)f(the)75 440 y(comm)o(unication)j(proto)q(col.) 166 497 y(The)g(send)h(call)g(describ)q(ed)h(in)e(Section)h(3.2.1)d (used)j(the)f Fq(standard)g Fx(comm)o(unication)h(mo)q(de.)22 b(In)75 553 y(this)16 b(mo)q(de,)e(it)i(is)f(up)h(to)e Fv(MPI)h Fx(to)f(decide)j(whether)e(outgoing)g(messages)f(will)j(b)q(e) f(bu\013ered.)k Fv(MPI)15 b Fx(ma)o(y)75 610 y(bu\013er)j(outgoing)h (messages.)29 b(In)19 b(suc)o(h)g(a)f(case,)h(the)f(send)h(call)h(ma)o (y)e(complete)h(b)q(efore)g(a)f(matc)o(hing)75 666 y(receiv)o(e)d(is)f (in)o(v)o(ok)o(ed.)20 b(On)14 b(the)g(other)g(hand,)g(bu\013er)g(space) g(ma)o(y)f(b)q(e)h(una)o(v)m(ailable,)i(or)e Fv(MPI)f Fx(ma)o(y)g(c)o(ho)q(ose)75 723 y(not)k(to)h(bu\013er)f(outgoing)h (messages,)g(for)f(p)q(erformance)h(reasons.)27 b(In)19 b(this)f(case,)g(the)g(send)g(call)h(will)75 779 y(not)13 b(complete)h(un)o(til)g(a)f(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)i (p)q(osted,)e(and)g(the)h(data)e(has)h(b)q(een)i(mo)o(v)o(ed)d(to)h (the)75 835 y(receiv)o(er.)166 892 y(Th)o(us,)j(a)f(send)i(in)g (standard)f(mo)q(de)g(can)g(b)q(e)h(started)e(whether)h(or)g(not)f(a)h (matc)o(hing)g(receiv)o(e)h(has)75 948 y(b)q(een)d(p)q(osted.)19 b(It)12 b(ma)o(y)g(complete)h(b)q(efore)g(a)f(matc)o(hing)g(receiv)o(e) i(is)f(p)q(osted.)19 b(The)13 b(standard)f(mo)q(de)g(send)75 1005 y(is)18 b Fq(non-lo)q(cal)p Fx(:)27 b(successful)19 b(completion)g(of)e(the)g(send)h(op)q(eration)g(ma)o(y)f(dep)q(end)i (on)e(the)h(o)q(ccurrence)75 1061 y(of)d(a)g(matc)o(hing)g(receiv)o(e.) 189 1168 y Fl(R)n(ationale.)42 b Fx(The)15 b(reluctance)i(of)e Fv(MPI)g Fx(to)g(mandate)g(whether)h(standard)f(sends)h(are)f (bu\013ering)189 1224 y(or)e(not)h(stems)g(from)f(the)i(desire)g(to)f (ac)o(hiev)o(e)g(p)q(ortable)h(programs.)j(Since)e(an)o(y)e(system)g (will)i(run)189 1280 y(out)d(of)g(bu\013er)g(resources)h(as)f(message)g (sizes)h(are)f(increased,)i(and)f(some)f(implemen)o(tations)i(ma)o(y) 189 1337 y(w)o(an)o(t)i(to)g(pro)o(vide)i(little)h(bu\013ering,)f Fv(MPI)e Fx(tak)o(es)h(the)g(p)q(osition)h(that)f(correct)g(\(and)g (therefore,)189 1393 y(p)q(ortable\))c(programs)e(do)i(not)g(rely)g(on) g(system)g(bu\013ering)g(in)h(standard)e(mo)q(de.)20 b(Bu\013ering)15 b(ma)o(y)189 1450 y(impro)o(v)o(e)g(the)h(p)q (erformance)g(of)g(a)f(correct)h(program,)e(but)i(it)g(do)q(esn't)g (a\013ect)f(the)h(result)g(of)g(the)189 1506 y(program.)33 b(If)21 b(the)f(user)g(wishes)h(to)f(guaran)o(tee)f(a)h(certain)h (amoun)o(t)e(of)h(bu\013ering,)h(the)g(user-)189 1563 y(pro)o(vided)13 b(bu\013er)g(system)f(of)g(Sec.)20 b(3.6)12 b(should)i(b)q(e)f(used,)g(along)g(with)g(the)g(bu\013ered-mo)q(de)h (send.)189 1619 y(\()p Fl(End)h(of)i(r)n(ationale.)p Fx(\))166 1725 y(There)e(are)g(three)h(additional)g(comm)o(unication)g (mo)q(des.)166 1782 y(A)j Fq(bu\013ered)f Fx(mo)q(de)h(send)g(op)q (eration)g(can)g(b)q(e)h(started)e(whether)h(or)f(not)h(a)f(matc)o (hing)h(receiv)o(e)75 1838 y(has)g(b)q(een)h(p)q(osted.)31 b(It)19 b(ma)o(y)f(complete)i(b)q(efore)f(a)f(matc)o(hing)h(receiv)o(e) h(is)g(p)q(osted.)31 b(Ho)o(w)o(ev)o(er,)18 b(unlik)o(e)75 1895 y(the)k(standard)f(send,)i(this)f(op)q(eration)g(is)g Fq(lo)q(cal)p Fx(,)j(and)d(its)g(completion)h(do)q(es)f(not)f(dep)q (end)i(on)f(the)75 1951 y(o)q(ccurrence)d(of)e(a)h(matc)o(hing)g (receiv)o(e.)28 b(Th)o(us,)18 b(if)g(a)g(send)g(is)g(executed)h(and)f (no)g(matc)o(hing)g(receiv)o(e)g(is)75 2008 y(p)q(osted,)13 b(then)f Fv(MPI)g Fx(m)o(ust)f(bu\013er)h(the)g(outgoing)g(message,)g (so)g(as)f(to)h(allo)o(w)g(the)g(send)h(call)g(to)f(complete.)75 2064 y(An)h(error)g(will)i(o)q(ccur)e(if)h(there)f(is)g(insu\016cien)o (t)i(bu\013er)e(space.)20 b(The)13 b(amoun)o(t)f(of)h(a)o(v)m(ailable)i (bu\013er)e(space)75 2121 y(is)k(con)o(trolled)f(b)o(y)g(the)g(user)h (|)f(see)g(Section)h(3.6.)22 b(Bu\013er)16 b(allo)q(cation)h(b)o(y)f (the)g(user)g(ma)o(y)f(b)q(e)i(required)75 2177 y(for)e(the)g (bu\013ered)g(mo)q(de)h(to)f(b)q(e)g(e\013ectiv)o(e.)166 2234 y(A)j(send)g(that)f(uses)g(the)h Fq(sync)o(hronous)e Fx(mo)q(de)i(can)g(b)q(e)g(started)f(whether)g(or)g(not)h(a)f(matc)o (hing)75 2290 y(receiv)o(e)k(w)o(as)e(p)q(osted.)34 b(Ho)o(w)o(ev)o (er,)19 b(the)h(send)h(will)g(complete)g(successfully)h(only)e(if)g(a)g (matc)o(hing)g(re-)75 2346 y(ceiv)o(e)f(is)h(p)q(osted,)f(and)g(the)f (receiv)o(e)i(op)q(eration)f(has)f(started)g(to)g(receiv)o(e)h(the)g (message)f(sen)o(t)g(b)o(y)h(the)75 2403 y(sync)o(hronous)c(send.)21 b(Th)o(us,)14 b(the)h(completion)i(of)d(a)h(sync)o(hronous)g(send)h (not)e(only)i(indicates)h(that)d(the)75 2459 y(send)g(bu\013er)f(can)g (b)q(e)h(reused,)f(but)g(also)g(indicates)i(that)d(the)h(receiv)o(er)h (has)f(reac)o(hed)g(a)g(certain)h(p)q(oin)o(t)f(in)75 2516 y(its)i(execution,)g(namely)g(that)e(it)i(has)f(started)g (executing)h(the)g(matc)o(hing)f(receiv)o(e.)21 b(If)14 b(b)q(oth)h(sends)g(and)75 2572 y(receiv)o(es)e(are)e(blo)q(c)o(king)j (op)q(erations)e(then)g(the)g(use)h(of)e(the)h(sync)o(hronous)g(mo)q (de)g(pro)o(vides)h(sync)o(hronous)75 2629 y(comm)o(unication)18 b(seman)o(tics:)25 b(a)18 b(comm)o(unication)g(do)q(es)g(not)g (complete)g(at)f(either)h(end)h(b)q(efore)f(b)q(oth)75 2685 y(pro)q(cesses)e(rendezv)o(ous)f(at)g(the)g(comm)o(unication.)21 b(A)15 b(send)h(executed)g(in)g(this)g(mo)q(de)f(is)h Fq(non-lo)q(cal)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 35 27 34 bop 75 -100 a Fm(3.4.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b Fx(27)166 45 y(A)14 b(send)h(that)e(uses)i(the)f Fq(ready)g Fx(comm)o(unication)g(mo)q(de)h(ma)o(y)e(b)q(e)i(started)e Fl(only)h Fx(if)h(the)f(matc)o(hing)75 102 y(receiv)o(e)k(is)g(already) f(p)q(osted.)27 b(Otherwise,)18 b(the)f(op)q(eration)h(is)f(erroneous)g (and)h(its)f(outcome)g(is)h(unde-)75 158 y(\014ned.)i(On)13 b(some)f(systems,)h(this)g(allo)o(ws)g(the)f(remo)o(v)m(al)h(of)f(a)h (hand-shak)o(e)g(op)q(eration)g(that)f(is)h(otherwise)75 214 y(required)k(and)f(results)g(in)h(impro)o(v)o(ed)f(p)q(erformance.) 22 b(The)16 b(completion)h(of)e(the)h(send)h(op)q(eration)f(do)q(es)75 271 y(not)g(dep)q(end)i(on)e(the)g(status)f(of)h(a)g(matc)o(hing)g (receiv)o(e,)h(and)f(merely)h(indicates)h(that)d(the)h(send)h(bu\013er) 75 327 y(can)i(b)q(e)g(reused.)30 b(A)19 b(send)g(op)q(eration)g(that)f (uses)g(the)h(ready)f(mo)q(de)h(has)g(the)f(same)g(seman)o(tics)h(as)f (a)75 384 y(standard)i(send)i(op)q(eration,)g(or)e(a)g(sync)o(hronous)h (send)g(op)q(eration;)j(it)d(is)g(merely)g(that)g(the)f(sender)75 440 y(pro)o(vides)e(additional)g(information)g(to)e(the)i(system)f (\(namely)g(that)g(a)f(matc)o(hing)i(receiv)o(e)g(is)g(already)75 497 y(p)q(osted\),)d(that)g(can)h(sa)o(v)o(e)e(some)i(o)o(v)o(erhead.)k (In)c(a)f(correct)g(program,)f(therefore,)h(a)g(ready)h(send)g(could)75 553 y(b)q(e)i(replaced)g(b)o(y)f(a)f(standard)h(send)h(with)f(no)g (e\013ect)g(on)f(the)i(b)q(eha)o(vior)f(of)g(the)g(program)e(other)i (than)75 610 y(p)q(erformance.)166 666 y(Three)g(additional)i(send)f (functions)g(are)f(pro)o(vided)h(for)e(the)h(three)h(additional)g(comm) o(unication)75 723 y(mo)q(des.)23 b(The)16 b(comm)o(unication)h(mo)q (de)g(is)f(indicated)i(b)o(y)e(a)g(one)g(letter)h(pre\014x:)22 b Fv(B)16 b Fx(for)g(bu\013ered,)g Fv(S)h Fx(for)75 779 y(sync)o(hronous,)e(and)g Fv(R)h Fx(for)e(ready)l(.)75 930 y Fv(MPI)p 160 930 14 2 v 16 w(BSEND)i(\(buf,)f(count,)h(datat)o (yp)q(e,)h(dest,)f(tag,)f(comm)m(\))117 1007 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 1081 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1154 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1227 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1301 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1374 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 1499 y Fu(int)23 b(MPI)p 245 1499 15 2 v 17 w(Bsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1499 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1555 y(int)h(tag,)f(MPI)p 683 1555 V 17 w(Comm)g(comm\))75 1642 y(MPI)p 150 1642 V 17 w(BSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 1698 y()f(BUF\(*\))170 1755 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 1841 y Fx(Send)16 b(in)g(bu\013ered)g(mo)q(de.)75 1992 y Fv(MPI)p 160 1992 14 2 v 16 w(SSEND)g(\(buf,)f(count,)h(datat)o(yp)q (e,)g(dest,)h(tag,)e(comm)m(\))117 2069 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 2143 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 2216 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 2289 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2363 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 2436 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 2561 y Fu(int)23 b(MPI)p 245 2561 15 2 v 17 w(Ssend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2561 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2617 y(int)h(tag,)f(MPI)p 683 2617 V 17 w(Comm)g(comm\))75 2704 y(MPI)p 150 2704 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 36 28 35 bop 75 -100 a Fx(28)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Fu()23 b(BUF\(*\))170 102 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h (COMM,)f(IERROR)166 188 y Fx(Send)16 b(in)g(sync)o(hronous)f(mo)q(de.) 75 339 y Fv(MPI)p 160 339 14 2 v 16 w(RSEND)h(\(buf,)f(count,)h(datat)o (yp)q(e,)h(dest,)f(tag,)f(comm)m(\))117 416 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 491 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 566 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 640 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 715 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 790 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 914 y Fu(int)23 b(MPI)p 245 914 15 2 v 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 914 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 971 y(int)h(tag,)f(MPI)p 683 971 V 17 w(Comm)g(comm\))75 1057 y(MPI)p 150 1057 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 1114 y()f(BUF\(*\))170 1170 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 1257 y Fx(Send)16 b(in)g(ready)f(mo)q(de.)166 1313 y(There)20 b(is)g(only)h(one)f(receiv)o(e)h(op)q(eration,)g(whic)o(h)g(can)f(matc) o(h)f(an)o(y)g(of)h(the)g(send)g(mo)q(des.)35 b(The)75 1369 y(receiv)o(e)15 b(op)q(eration)g(describ)q(ed)h(in)f(the)f(last)g (section)h(is)g Fq(blo)q(c)o(king)p Fx(:)20 b(it)15 b(returns)f(only)g (after)g(the)g(receiv)o(e)75 1426 y(bu\013er)k(con)o(tains)h(the)f (newly)i(receiv)o(ed)f(message.)29 b(A)19 b(receiv)o(e)g(can)g (complete)g(b)q(efore)f(the)h(matc)o(hing)75 1482 y(send)d(has)f (completed)h(\(of)e(course,)h(it)h(can)f(complete)h(only)g(after)e(the) i(matc)o(hing)f(send)h(has)f(started\).)166 1539 y(In)f(a)g(m)o (ulti-threaded)h(implemen)o(tation)h(of)d Fv(MPI)p Fx(,)g(the)h(system) g(ma)o(y)f(de-sc)o(hedule)j(a)e(thread)g(that)75 1595 y(is)h(blo)q(c)o(k)o(ed)h(on)f(a)g(send)g(or)f(receiv)o(e)i(op)q (eration,)f(and)g(sc)o(hedule)i(another)d(thread)h(for)f(execution)i (in)g(the)75 1652 y(same)g(address)g(space.)22 b(In)16 b(suc)o(h)h(a)e(case)h(it)g(is)h(the)f(user's)f(resp)q(onsibilit)o(y)k (not)c(to)g(access)h(or)g(mo)q(dify)g(a)75 1708 y(comm)o(unication)i (bu\013er)e(un)o(til)i(the)f(comm)o(unication)h(completes.)25 b(Otherwise,)18 b(the)f(outcome)f(of)h(the)75 1765 y(computation)e(is)h (unde\014ned.)189 1868 y Fl(R)n(ationale.)56 b Fx(W)l(e)18 b(prohibit)i(read)d(accesses)i(to)e(a)h(send)g(bu\013er)g(while)i(it)e (is)g(b)q(eing)i(used,)f(ev)o(en)189 1924 y(though)f(the)g(send)g(op)q (eration)h(is)f(not)g(supp)q(osed)h(to)e(alter)h(the)g(con)o(ten)o(t)g (of)f(this)i(bu\013er.)28 b(This)189 1981 y(ma)o(y)17 b(seem)h(more)f(stringen)o(t)g(than)h(necessary)l(,)g(but)g(the)g (additional)h(restriction)g(causes)f(little)189 2037 y(loss)e(of)g(functionalit)o(y)i(and)f(allo)o(ws)g(b)q(etter)f(p)q (erformance)h(on)f(some)g(systems)g(|)h(consider)h(the)189 2094 y(case)13 b(where)g(data)f(transfer)g(is)i(done)f(b)o(y)g(a)g(DMA) f(engine)j(that)d(is)h(not)g(cac)o(he-coheren)o(t)h(with)f(the)189 2150 y(main)i(pro)q(cessor.)20 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 2254 y Fl(A)n(dvic)n(e)10 b(to)j(implementors.)36 b Fx(Since)12 b(a)e(sync)o(hronous)g(send)h(cannot)f(complete)h(b)q (efore)g(a)f(matc)o(hing)189 2310 y(receiv)o(e)16 b(is)f(p)q(osted,)h (one)f(will)i(not)e(normally)g(bu\013er)h(messages)e(sen)o(t)h(b)o(y)g (suc)o(h)h(an)f(op)q(eration.)189 2385 y(It)j(is)h(recommended)g(to)e (c)o(ho)q(ose)i(bu\013ering)f(o)o(v)o(er)g(blo)q(c)o(king)h(the)g (sender,)g(whenev)o(er)g(p)q(ossible,)189 2441 y(for)13 b(standard)h(sends.)20 b(The)15 b(programmer)e(can)i(signal)g(his)g(or) f(her)g(preference)h(for)f(blo)q(c)o(king)i(the)189 2498 y(sender)f(un)o(til)i(a)e(matc)o(hing)g(receiv)o(e)h(o)q(ccurs)g(b)o(y) f(using)h(the)f(sync)o(hronous)g(send)h(mo)q(de.)189 2572 y(A)f(p)q(ossible)i(comm)o(unication)g(proto)q(col)e(for)g(the)h (v)m(arious)f(comm)o(unication)i(mo)q(des)e(is)h(outlined)189 2629 y(b)q(elo)o(w.)189 2704 y Fv(ready)f(send)p Fx(:)21 b(The)16 b(message)e(is)i(sen)o(t)f(as)g(so)q(on)g(as)g(p)q(ossible.) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 37 29 36 bop 75 -100 a Fm(3.5.)29 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)455 b Fx(29)189 45 y Fv(synchronous)20 b(send:)28 b Fx(The)19 b(sender)g(sends)g(a)f(request-to-send)h (message.)30 b(The)18 b(receiv)o(er)i(stores)189 102 y(this)12 b(request.)19 b(When)13 b(a)e(matc)o(hing)i(receiv)o(e)g(is)f (p)q(osted,)h(the)f(receiv)o(er)h(sends)g(bac)o(k)f(a)f(p)q(ermission-) 189 158 y(to-send)k(message,)f(and)i(the)f(sender)h(no)o(w)f(sends)g (the)h(message.)189 233 y Fv(standa)o(rd)e(send:)21 b Fx(First)13 b(proto)q(col)h(ma)o(y)e(b)q(e)j(used)f(for)f(short)f (messages,)h(and)h(second)g(proto)q(col)g(for)189 289 y(long)h(messages.)189 364 y Fv(bu\013ered)j(send:)25 b Fx(The)18 b(sender)g(copies)g(the)f(message)g(in)o(to)g(a)g(bu\013er) g(and)h(then)f(sends)h(it)g(with)f(a)189 421 y(non)o(blo)q(c)o(king)f (send)g(\(using)g(the)f(same)g(proto)q(col)g(as)g(for)f(standard)h (send\).)189 495 y(Additional)i(con)o(trol)f(messages)g(migh)o(t)g(b)q (e)g(needed)i(for)d(\015o)o(w)h(con)o(trol)f(and)i(error)e(reco)o(v)o (ery)l(.)22 b(Of)189 552 y(course,)15 b(there)g(are)g(man)o(y)f(other)h (p)q(ossible)i(proto)q(cols.)189 627 y(Ready)i(send)h(can)g(b)q(e)g (implemen)o(ted)h(as)e(a)f(standard)h(send.)33 b(In)20 b(this)g(case)f(there)g(will)i(b)q(e)f(no)189 683 y(p)q(erformance)15 b(adv)m(an)o(tage)g(\(or)f(disadv)m(an)o(tage\))h(for)g(the)g(use)h(of) e(ready)h(send.)189 758 y(A)g(standard)f(send)i(can)f(b)q(e)h(implemen) o(ted)h(as)e(a)f(sync)o(hronous)i(send.)k(In)c(suc)o(h)f(a)g(case,)g (no)g(data)189 814 y(bu\013ering)g(is)h(needed.)21 b(Ho)o(w)o(ev)o(er,) 14 b(man)o(y)h(\(most?\))k(users)c(exp)q(ect)h(some)f(bu\013ering.)189 889 y(In)h(a)g(m)o(ulti-threaded)i(en)o(vironmen)o(t,)e(the)h (execution)g(of)f(a)f(blo)q(c)o(king)j(comm)o(unication)f(should)189 946 y(blo)q(c)o(k)k(only)g(the)f(executing)i(thread,)f(allo)o(wing)g (the)g(thread)f(sc)o(heduler)i(to)e(de-sc)o(hedule)j(this)189 1002 y(thread)15 b(and)g(sc)o(hedule)i(another)e(thread)g(for)f (execution.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 1145 y Fn(3.5)59 b(Semantics)18 b(of)i(p)r(oint-to-p)r(oint)e (communication)75 1247 y Fx(A)g(v)m(alid)i Fv(MPI)e Fx(implemen)o (tation)h(guaran)o(tees)f(certain)h(general)f(prop)q(erties)h(of)f(p)q (oin)o(t-to-p)q(oin)o(t)h(com-)75 1303 y(m)o(unication,)d(whic)o(h)g (are)f(describ)q(ed)i(in)f(this)f(section.)75 1423 y Fv(Order)46 b Fx(Messages)17 b(are)g Fl(non-overtaking)p Fx(:)25 b(If)18 b(a)f(sender)i(sends)f(t)o(w)o(o)f(messages)g(in)i (succession)g(to)e(the)75 1479 y(same)d(destination,)i(and)f(b)q(oth)g (matc)o(h)f(the)h(same)f(receiv)o(e,)h(then)h(this)f(op)q(eration)g (cannot)f(receiv)o(e)i(the)75 1536 y(second)e(message)e(if)i(the)f (\014rst)g(one)h(is)f(still)i(p)q(ending.)21 b(If)13 b(a)g(receiv)o(er)h(p)q(osts)f(t)o(w)o(o)f(receiv)o(es)i(in)g (succession,)75 1592 y(and)k(b)q(oth)g(matc)o(h)f(the)h(same)g (message,)g(then)g(the)g(second)g(receiv)o(e)h(op)q(eration)f(cannot)g (b)q(e)g(satis\014ed)75 1649 y(b)o(y)g(this)g(message,)f(if)i(the)e (\014rst)h(one)g(is)g(still)h(p)q(ending.)29 b(This)19 b(requiremen)o(t)f(facilitates)h(matc)o(hing)e(of)75 1705 y(sends)k(to)f(receiv)o(es.)36 b(It)21 b(guaran)o(tees)f(that)f (message-passing)i(co)q(de)g(is)g(deterministic,)j(if)d(pro)q(cesses)75 1761 y(are)d(single-threaded)j(and)e(the)g(wildcard)h Fi(MPI)p 913 1761 13 2 v 14 w(ANY)p 1012 1761 V 15 w(SOURCE)c Fx(is)k(not)e(used)h(in)h(receiv)o(es.)31 b(\(Some)18 b(of)75 1818 y(the)c(calls)i(describ)q(ed)g(later,)e(suc)o(h)h(as)f Fv(MPI)p 813 1818 14 2 v 15 w(CANCEL)h Fx(or)f Fv(MPI)p 1152 1818 V 16 w(W)l(AIT)l(ANY)p Fx(,)g(are)g(additional)i(sources)e (of)75 1874 y(nondeterminism.\),)166 1931 y(If)j(a)g(pro)q(cess)h(has)f (a)g(single)h(thread)f(of)g(execution,)i(then)e(an)o(y)g(t)o(w)o(o)f (comm)o(unications)i(executed)75 1987 y(b)o(y)e(this)h(pro)q(cess)g (are)f(ordered.)23 b(On)17 b(the)f(other)g(hand,)h(if)g(the)f(pro)q (cess)h(is)g(m)o(ulti-threaded,)g(then)g(the)75 2044 y(seman)o(tics)e(of)f(thread)h(execution)h(ma)o(y)e(not)g(de\014ne)i(a) e(relativ)o(e)i(order)e(b)q(et)o(w)o(een)h(t)o(w)o(o)e(send)j(op)q (erations)75 2100 y(executed)23 b(b)o(y)f(t)o(w)o(o)f(distinct)i (threads.)40 b(The)23 b(op)q(erations)f(are)g(logically)h(concurren)o (t,)h(ev)o(en)e(if)h(one)75 2157 y(ph)o(ysically)f(precedes)e(the)g (other.)33 b(In)20 b(suc)o(h)g(a)f(case,)i(the)e(t)o(w)o(o)g(messages)g (sen)o(t)g(can)h(b)q(e)g(receiv)o(ed)h(in)75 2213 y(an)o(y)f(order.)37 b(Similarly)l(,)24 b(if)d(t)o(w)o(o)e(receiv)o(e)j(op)q(erations)f (that)f(are)g(logically)j(concurren)o(t)e(receiv)o(e)g(t)o(w)o(o)75 2270 y(successiv)o(ely)i(sen)o(t)d(messages,)h(then)g(the)g(t)o(w)o(o)f (messages)g(can)h(matc)o(h)f(the)h(t)o(w)o(o)e(receiv)o(es)j(in)f (either)75 2326 y(order.)75 2430 y Fq(Example)d(3.5)k Fx(An)16 b(example)g(of)f(non-o)o(v)o(ertaking)f(messages.)75 2534 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2591 y(IF)h(\(rank.EQ.0\))123 2647 y(THEN)170 2704 y(CALL)g (MPI_BSEND\(buf1,)e(count,)h(MPI_REAL,)f(1,)i(tag,)f(comm,)g(ierr\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 38 30 37 bop 75 -100 a Fx(30)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Fu(CALL)24 b(MPI_BSEND\(buf2,)e(count,)h(MPI_REAL,)f(1,)i(tag,)f(comm,)g(ierr\)) 123 102 y(ELSE)95 b(!)23 b(rank.EQ.1)170 158 y(CALL)h(MPI_RECV\(buf1,)e (count,)h(MPI_REAL,)f(0,)i(MPI_ANY_TAG,)e(comm,)h(status,)g(ierr\))170 214 y(CALL)h(MPI_RECV\(buf2,)e(count,)h(MPI_REAL,)f(0,)i(tag,)f(comm,)g (status,)g(ierr\))75 271 y(END)g(IF)75 372 y Fx(The)17 b(message)f(sen)o(t)h(b)o(y)g(the)f(\014rst)h(send)g(m)o(ust)f(b)q(e)i (receiv)o(ed)g(b)o(y)f(the)f(\014rst)h(receiv)o(e,)h(and)f(the)f (message)75 428 y(sen)o(t)f(b)o(y)g(the)g(second)h(send)g(m)o(ust)e(b)q (e)i(receiv)o(ed)h(b)o(y)e(the)g(second)h(receiv)o(e.)75 547 y Fv(Progress)45 b Fx(If)13 b(a)g(pair)g(of)f(matc)o(hing)h(send)h (and)f(receiv)o(es)g(ha)o(v)o(e)g(b)q(een)h(initiated)g(on)f(t)o(w)o(o) e(pro)q(cesses,)j(then)75 604 y(at)k(least)h(one)h(of)e(these)h(t)o(w)o (o)f(op)q(erations)h(will)i(complete,)f(indep)q(enden)o(tly)i(of)c (other)h(actions)g(in)h(the)75 660 y(system:)e(the)13 b(send)h(op)q(eration)f(will)h(complete,)g(unless)g(the)f(receiv)o(e)h (is)f(satis\014ed)h(b)o(y)e(another)h(message,)75 717 y(and)h(completes;)g(the)g(receiv)o(e)h(op)q(eration)f(will)h (complete,)g(unless)g(the)f(message)f(sen)o(t)g(is)i(consumed)f(b)o(y) 75 773 y(another)h(matc)o(hing)g(receiv)o(e)h(that)f(w)o(as)f(p)q (osted)h(at)g(the)g(same)g(destination)h(pro)q(cess.)75 874 y Fq(Example)i(3.6)k Fx(An)16 b(example)g(of)f(t)o(w)o(o,)e(in)o (tert)o(wined)j(matc)o(hing)f(pairs.)75 964 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1020 y(IF)h(\(rank.EQ.0\)) 123 1077 y(THEN)170 1133 y(CALL)g(MPI_BSEND\(buf1,)e(count,)h (MPI_REAL,)f(1,)i(tag1,)f(comm,)g(ierr\))170 1190 y(CALL)h (MPI_SSEND\(buf2,)e(count,)h(MPI_REAL,)f(1,)i(tag2,)f(comm,)g(ierr\)) 123 1246 y(ELSE)95 b(!)23 b(rank.EQ.1)170 1302 y(CALL)h (MPI_RECV\(buf1,)e(count,)h(MPI_REAL,)f(0,)i(tag2,)f(comm,)g(status,)g (ierr\))170 1359 y(CALL)h(MPI_RECV\(buf2,)e(count,)h(MPI_REAL,)f(0,)i (tag1,)f(comm,)g(status,)g(ierr\))75 1415 y(END)g(IF)75 1505 y Fx(Both)18 b(pro)q(cesses)g(in)o(v)o(ok)o(e)h(their)f(\014rst)g (comm)o(unication)h(call.)30 b(Since)19 b(the)f(\014rst)g(send)h(of)f (pro)q(cess)g(zero)75 1562 y(uses)h(the)f(bu\013ered)h(mo)q(de,)g(it)g (m)o(ust)f(complete,)i(irresp)q(ectiv)o(e)g(of)e(the)g(state)g(of)g (pro)q(cess)h(one.)29 b(Since)75 1618 y(no)15 b(matc)o(hing)h(receiv)o (e)g(is)g(p)q(osted,)g(the)g(message)f(will)i(b)q(e)f(copied)h(in)o(to) e(bu\013er)h(space.)21 b(\(If)15 b(insu\016cien)o(t)75 1675 y(bu\013er)h(space)g(is)h(a)o(v)m(ailable,)g(then)f(the)g(program) f(will)j(fail.\))k(The)16 b(second)h(send)f(is)h(then)f(in)o(v)o(ok)o (ed.)22 b(A)o(t)75 1731 y(that)16 b(p)q(oin)o(t,)g(a)h(matc)o(hing)f (pair)h(of)f(send)h(and)f(receiv)o(e)h(op)q(eration)g(is)g(enabled,)h (and)e(b)q(oth)h(op)q(erations)75 1788 y(m)o(ust)f(complete.)23 b(Pro)q(cess)16 b(one)g(next)g(in)o(v)o(ok)o(es)g(its)g(second)h (receiv)o(e)g(call,)g(whic)o(h)g(will)h(b)q(e)e(satis\014ed)h(b)o(y)75 1844 y(the)c(bu\013ered)g(message.)19 b(Note)12 b(that)h(pro)q(cess)g (one)g(receiv)o(ed)h(the)f(messages)f(in)i(the)f(rev)o(erse)f(order)h (they)75 1900 y(w)o(ere)i(sen)o(t.)75 2020 y Fv(F)o(airness)46 b(MPI)15 b Fx(mak)o(es)h(no)f(guaran)o(tee)h(of)f Fl(fairness)g Fx(in)i(the)f(handling)h(of)f(comm)o(unication.)23 b(Supp)q(ose)75 2076 y(that)17 b(a)g(send)h(is)g(p)q(osted.)26 b(Then)18 b(it)g(is)g(p)q(ossible)h(that)d(the)i(destination)g(pro)q(cess)g(rep)q (eatedly)g(p)q(osts)f(a)75 2133 y(receiv)o(e)i(that)e(matc)o(hes)g (this)h(send,)h(y)o(et)e(the)h(message)f(is)h(nev)o(er)g(receiv)o(ed,)h (b)q(ecause)g(it)f(is)g(eac)o(h)g(time)75 2189 y(o)o(v)o(ertak)o(en)c (b)o(y)i(another)f(message,)g(sen)o(t)h(from)f(another)g(source.)21 b(Similarly)l(,)d(supp)q(ose)f(that)e(a)g(receiv)o(e)75 2246 y(w)o(as)f(p)q(osted)i(b)o(y)f(a)g(m)o(ulti-threaded)h(pro)q (cess.)k(Then)c(it)f(is)h(p)q(ossible)h(that)d(messages)h(that)f(matc)o (h)h(this)75 2302 y(receiv)o(e)20 b(are)g(rep)q(eatedly)g(receiv)o(ed,) i(y)o(et)d(the)g(receiv)o(e)i(is)f(nev)o(er)f(satis\014ed,)i(b)q (ecause)f(it)g(is)g(o)o(v)o(ertak)o(en)75 2358 y(b)o(y)15 b(other)h(receiv)o(es)g(p)q(osted)g(at)e(this)i(no)q(de)h(\(b)o(y)e (other)g(executing)h(threads\).)21 b(It)15 b(is)h(the)g(programmer's)75 2415 y(resp)q(onsibilit)o(y)i(to)c(prev)o(en)o(t)h(starv)m(ation)g(in)h (suc)o(h)g(situations.)75 2534 y Fv(Resource)23 b(limitations)44 b Fx(An)o(y)22 b(p)q(ending)i(comm)o(unication)e(op)q(eration)h (consumes)f(system)g(resources)75 2591 y(that)14 b(are)g(limited.)22 b(Errors)14 b(ma)o(y)g(o)q(ccur)h(when)g(lac)o(k)g(of)f(resources)h (prev)o(en)o(t)f(the)h(execution)h(of)e(an)h Fv(MPI)75 2647 y Fx(call.)20 b(A)13 b(qualit)o(y)g(implemen)o(tation)h(will)g (use)f(a)f(\(small\))g(\014xed)h(amoun)o(t)f(of)g(resources)h(for)f (eac)o(h)g(p)q(ending)75 2704 y(send)i(in)f(the)g(ready)g(or)g(sync)o (hronous)g(mo)q(de)g(and)g(for)g(eac)o(h)g(p)q(ending)i(receiv)o(e.)20 b(Ho)o(w)o(ev)o(er,)12 b(bu\013er)h(space)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 39 31 38 bop 75 -100 a Fm(3.5.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b Fx(31)75 45 y(ma)o(y)13 b(b)q(e)h(consumed)g (to)e(store)h(messages)g(sen)o(t)g(in)h(standard)f(mo)q(de,)h(and)f(m)o (ust)g(b)q(e)h(consumed)g(to)f(store)75 102 y(messages)e(sen)o(t)g(in)i (bu\013ered)f(mo)q(de,)g(when)g(no)f(matc)o(hing)h(receiv)o(e)h(is)f(a) o(v)m(ailable.)20 b(The)12 b(amoun)o(t)f(of)g(space)75 158 y(a)o(v)m(ailable)17 b(for)e(bu\013ering)i(will)g(b)q(e)f(m)o(uc)o (h)g(smaller)h(than)e(program)g(data)g(memory)g(on)h(man)o(y)f (systems.)75 214 y(Then,)g(it)h(will)h(b)q(e)e(easy)g(to)g(write)g (programs)f(that)h(o)o(v)o(errun)f(a)o(v)m(ailable)j(bu\013er)f(space.) 166 271 y Fv(MPI)e Fx(allo)o(ws)h(the)g(user)g(to)f(pro)o(vide)i (bu\013er)f(memory)f(for)g(messages)g(sen)o(t)h(in)h(the)f(bu\013ered)g (mo)q(de.)75 327 y(F)l(urthermore,)f Fv(MPI)g Fx(sp)q(eci\014es)j(a)e (detailed)h(op)q(erational)g(mo)q(del)g(for)e(the)h(use)h(of)e(this)i (bu\013er.)j(An)d Fv(MPI)75 384 y Fx(implemen)o(tation)f(is)f(required) h(to)e(do)h(no)f(w)o(orse)g(than)h(implied)i(b)o(y)d(this)h(mo)q(del.) 21 b(This)14 b(allo)o(ws)g(users)g(to)75 440 y(a)o(v)o(oid)h(bu\013er)g (o)o(v)o(er\015o)o(ws)e(when)i(they)g(use)h(bu\013ered)f(sends.)20 b(Bu\013er)15 b(allo)q(cation)h(and)f(use)h(is)f(describ)q(ed)75 497 y(in)h(Section)g(3.6.)166 553 y(A)j(bu\013ered)g(send)h(op)q (eration)f(that)f(cannot)h(complete)h(b)q(ecause)f(of)g(a)g(lac)o(k)g (of)f(bu\013er)h(space)g(is)75 610 y(erroneous.)36 b(When)20 b(suc)o(h)h(a)f(situation)h(is)g(detected,)h(an)f(error)e(is)i (signalled)i(that)c(ma)o(y)h(cause)h(the)75 666 y(program)f(to)h (terminate)g(abnormally)l(.)38 b(On)22 b(the)f(other)g(hand,)i(a)e (standard)g(send)g(op)q(eration)h(that)75 723 y(cannot)16 b(complete)g(b)q(ecause)h(of)f(lac)o(k)g(of)f(bu\013er)h(space)g(will)i (merely)e(blo)q(c)o(k,)h(w)o(aiting)f(for)f(bu\013er)h(space)75 779 y(to)g(b)q(ecome)h(a)o(v)m(ailable)h(or)e(for)f(a)h(matc)o(hing)h (receiv)o(e)g(to)f(b)q(e)h(p)q(osted.)23 b(This)17 b(b)q(eha)o(vior)g (is)g(preferable)g(in)75 835 y(man)o(y)h(situations.)31 b(Consider)20 b(a)e(situation)h(where)g(a)g(pro)q(ducer)g(rep)q (eatedly)h(pro)q(duces)g(new)f(v)m(alues)75 892 y(and)j(sends)g(them)f (to)g(a)g(consumer.)39 b(Assume)22 b(that)e(the)i(pro)q(ducer)g(pro)q (duces)g(new)g(v)m(alues)h(faster)75 948 y(than)18 b(the)g(consumer)g (can)g(consume)g(them.)28 b(If)18 b(bu\013ered)g(sends)g(are)g(used,)h (then)f(a)f(bu\013er)h(o)o(v)o(er\015o)o(w)75 1005 y(will)i(result.)31 b(Additional)20 b(sync)o(hronization)g(has)e(to)g(b)q(e)h(added)h(to)e (the)g(program)g(so)g(as)g(to)g(prev)o(en)o(t)75 1061 y(this)h(from)e(o)q(ccurring.)29 b(If)19 b(standard)e(sends)i(are)f (used,)h(then)f(the)g(pro)q(ducer)h(will)h(b)q(e)f(automatically)75 1118 y(throttled,)c(as)f(its)i(send)g(op)q(erations)f(will)i(blo)q(c)o (k)f(when)f(bu\013er)h(space)f(is)h(una)o(v)m(ailable.)166 1174 y(In)c(some)f(situations,)h(a)f(lac)o(k)g(of)g(bu\013er)h(space)f (leads)h(to)f(deadlo)q(c)o(k)h(situations.)19 b(This)12 b(is)g(illustrated)75 1231 y(b)o(y)j(the)g(examples)h(b)q(elo)o(w.)75 1336 y Fq(Example)i(3.7)k Fx(An)16 b(exc)o(hange)f(of)g(messages.)75 1441 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1497 y(IF)h(\(rank.EQ.0\))123 1554 y(THEN)170 1610 y(CALL)g (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 1667 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))123 1723 y(ELSE)95 b(!)23 b(rank.EQ.1)170 1780 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(status,)g(ierr\))170 1836 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 1892 y(END)g(IF)75 1997 y Fx(This)16 b(program)f(will)i(succeed)g(ev)o(en)f (if)g(no)g(bu\013er)g(space)g(for)f(data)g(is)h(a)o(v)m(ailable.)23 b(The)16 b(standard)f(send)75 2054 y(op)q(eration)g(can)h(b)q(e)g (replaced,)g(in)g(this)f(example,)h(with)g(a)e(sync)o(hronous)i(send.) 75 2159 y Fq(Example)i(3.8)k Fx(An)16 b(attempt)e(to)h(exc)o(hange)g (messages.)75 2252 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 2308 y(IF)h(\(rank.EQ.0\))123 2365 y(THEN)170 2421 y(CALL)g(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))170 2478 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))123 2534 y(ELSE)95 b(!)23 b(rank.EQ.1)170 2591 y(CALL)h(MPI_RECV\(recvbuf,) d(count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))170 2647 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))75 2704 y(END)g(IF)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 40 32 39 bop 75 -100 a Fx(32)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(The)f(receiv)o(e)g (op)q(eration)f(of)g(the)h(\014rst)f(pro)q(cess)g(m)o(ust)g(complete)h (b)q(efore)g(its)g(send,)f(and)h(can)f(complete)75 102 y(only)g(if)g(the)g(matc)o(hing)f(send)h(of)f(the)h(second)g(pro)q (cessor)f(is)h(executed.)21 b(The)15 b(receiv)o(e)g(op)q(eration)g(of)f (the)75 158 y(second)j(pro)q(cess)g(m)o(ust)e(complete)i(b)q(efore)g (its)g(send)g(and)f(can)h(complete)g(only)g(if)g(the)f(matc)o(hing)g (send)75 214 y(of)d(the)h(\014rst)f(pro)q(cess)g(is)h(executed.)21 b(This)14 b(program)e(will)j(alw)o(a)o(ys)e(deadlo)q(c)o(k.)20 b(The)14 b(same)f(holds)h(for)f(an)o(y)75 271 y(other)i(send)h(mo)q (de.)75 377 y Fq(Example)i(3.9)k Fx(An)16 b(exc)o(hange)f(that)g (relies)h(on)f(bu\013ering.)75 471 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,) f(rank,)h(ierr\))75 527 y(IF)h(\(rank.EQ.0\))123 584 y(THEN)170 640 y(CALL)g(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h (tag,)f(comm,)g(ierr\))170 697 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(1,)h(tag,)f(comm,)g(status,)g(ierr\))123 753 y(ELSE)95 b(!)23 b(rank.EQ.1)170 810 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))170 866 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(status,)g(ierr\))75 923 y(END)g(IF)75 1016 y Fx(The)17 b(message)g(sen)o(t)g(b)o(y)g(eac)o(h)g(pro)q(cess)g(has)g (to)g(b)q(e)h(copied)g(out)f(b)q(efore)g(the)g(send)h(op)q(eration)f (returns)75 1073 y(and)e(the)g(receiv)o(e)h(op)q(eration)f(starts.)k(F) l(or)14 b(the)h(program)f(to)g(complete,)h(it)h(is)f(necessary)g(that)g (at)f(least)75 1129 y(one)22 b(of)f(the)g(t)o(w)o(o)f(messages)h(sen)o (t)g(b)q(e)h(bu\013ered.)39 b(Th)o(us,)23 b(this)f(program)e(can)h (succeed)i(only)f(if)g(the)75 1186 y(comm)o(unication)16 b(system)f(can)g(bu\013er)g(at)g(least)g Fv(count)i Fx(w)o(ords)d(of)h (data.)189 1292 y Fl(A)n(dvic)n(e)f(to)h(users.)40 b Fx(When)14 b(standard)g(send)g(op)q(erations)g(are)g(used,)g(then)h(a)f (deadlo)q(c)o(k)h(situation)189 1349 y(ma)o(y)c(o)q(ccur)h(where)h(b)q (oth)f(pro)q(cesses)g(are)g(blo)q(c)o(k)o(ed)h(b)q(ecause)g(bu\013er)f (space)g(is)h(not)e(a)o(v)m(ailable.)21 b(The)189 1405 y(same)15 b(will)i(certainly)g(happ)q(en,)f(if)g(the)g(sync)o(hronous)f (mo)q(de)h(is)g(used.)21 b(If)16 b(the)g(bu\013ered)g(mo)q(de)f(is)189 1461 y(used,)k(and)f(not)g(enough)g(bu\013er)h(space)f(is)h(a)o(v)m (ailable,)h(then)e(the)h(program)e(will)i(not)f(complete)189 1518 y(either.)34 b(Ho)o(w)o(ev)o(er,)19 b(rather)g(than)h(a)f(deadlo)q (c)o(k)i(situation,)g(w)o(e)e(shall)i(ha)o(v)o(e)e(a)h(bu\013er)f(o)o (v)o(er\015o)o(w)189 1574 y(error.)189 1650 y(A)c(program)g(is)h (\\safe")e(if)i(no)g(message)f(bu\013ering)h(is)g(required)h(for)e(the) g(program)g(to)g(complete.)189 1706 y(One)22 b(can)g(replace)h(all)f (sends)h(in)f(suc)o(h)g(program)f(with)h(sync)o(hronous)g(sends,)h(and) f(the)g(pro-)189 1762 y(gram)16 b(will)i(still)h(run)e(correctly)l(.)25 b(This)18 b(conserv)m(ativ)o(e)f(programming)f(st)o(yle)h(pro)o(vides)h (the)f(b)q(est)189 1819 y(p)q(ortabilit)o(y)l(,)e(since)g(program)f (completion)h(do)q(es)g(not)e(dep)q(end)j(on)f(the)f(amoun)o(t)g(of)f (bu\013er)i(space)189 1875 y(a)o(v)m(ailable)i(or)d(in)i(the)g(comm)o (unication)f(proto)q(col)h(used.)189 1950 y(Man)o(y)f(programmers)f (prefer)i(to)f(ha)o(v)o(e)h(more)f(leew)o(a)o(y)h(and)g(b)q(e)g(able)h (to)e(use)h(the)g(\\unsafe")f(pro-)189 2007 y(gramming)c(st)o(yle)g (sho)o(wn)h(in)g(example)h(3.9.)k(In)12 b(suc)o(h)g(cases,)g(the)g(use) g(of)f(standard)g(sends)h(is)g(lik)o(ely)189 2063 y(to)i(pro)o(vide)i (the)g(b)q(est)f(compromise)h(b)q(et)o(w)o(een)f(p)q(erformance)h(and)f (robustness:)20 b(qualit)o(y)c(imple-)189 2120 y(men)o(tations)e(will)j (pro)o(vide)f(su\016cien)o(t)g(bu\013ering)f(so)g(that)f(\\common)h (practice")g(programs)f(will)189 2176 y(not)j(deadlo)q(c)o(k.)28 b(The)18 b(bu\013ered)g(send)g(mo)q(de)g(can)f(b)q(e)i(used)f(for)f (programs)f(that)h(require)h(more)189 2233 y(bu\013ering,)k(or)e(in)i (situations)f(where)g(the)g(programmer)f(w)o(an)o(ts)f(more)i(con)o (trol.)36 b(This)22 b(mo)q(de)189 2289 y(migh)o(t)14 b(also)g(b)q(e)h(used)g(for)e(debugging)j(purp)q(oses,)e(as)g(bu\013er) g(o)o(v)o(er\015o)o(w)f(conditions)j(are)e(easier)h(to)189 2346 y(diagnose)g(than)g(deadlo)q(c)o(k)h(conditions.)189 2421 y(Non)o(blo)q(c)o(king)h(message-passing)f(op)q(erations,)h(as)f (describ)q(ed)i(in)f(Section)g(3.7,)e(can)i(b)q(e)g(used)g(to)189 2477 y(a)o(v)o(oid)f(the)g(need)h(for)f(bu\013ering)g(outgoing)g (messages.)23 b(This)16 b(prev)o(en)o(ts)g(deadlo)q(c)o(ks)h(due)g(to)f (lac)o(k)189 2534 y(of)h(bu\013er)h(space,)g(and)g(impro)o(v)o(es)f(p)q (erformance,)h(b)o(y)g(allo)o(wing)g(o)o(v)o(erlap)g(of)f(computation)h (and)189 2590 y(comm)o(unication,)12 b(and)f(a)o(v)o(oiding)h(the)f(o)o (v)o(erheads)g(of)f(allo)q(cating)j(bu\013ers)e(and)g(cop)o(ying)g (messages)189 2647 y(in)o(to)k(bu\013ers.)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 41 33 40 bop 75 -100 a Fm(3.6.)34 b(BUFFER)15 b(ALLOCA)l(TION)i(AND)f(USA) o(GE)843 b Fx(33)75 45 y Fn(3.6)59 b(Bu\013er)20 b(allo)r(cation)e(and) i(usage)75 147 y Fx(A)13 b(user)g(ma)o(y)f(sp)q(ecify)i(a)e(bu\013er)h (to)f(b)q(e)h(used)h(for)e(bu\013ering)h(messages)f(sen)o(t)h(in)g (bu\013ered)h(mo)q(de.)19 b(Bu\013er-)75 203 y(ing)d(is)f(done)h(b)o(y) f(the)g(sender.)75 354 y Fv(MPI)p 160 354 14 2 v 16 w(BUFFER)p 345 354 V 17 w(A)l(TT)l(A)o(CH\()h(bu\013er,)f(size\))117 431 y Fk(IN)155 b Fv(bu\013er)478 b Fk(initial)12 b(bu\013er)j(address) g(\(c)o(hoice\))117 506 y(IN)155 b Fv(size)519 b Fk(bu\013er)15 b(size,)f(in)g(b)o(ytes)g(\(in)o(teger\))75 631 y Fu(int)23 b(MPI)p 245 631 15 2 v 17 w(Buffer)p 406 631 V 17 w(attach\()f(void*)i (buffer,)f(int)g(size\))75 717 y(MPI)p 150 717 V 17 w(BUFFER)p 311 717 V 16 w(ATTACH\()g(BUFFER,)g(SIZE,)g(IERROR\))170 774 y()g(BUFFER\(*\))170 830 y(INTEGER)g(SIZE,)h(IERROR)166 917 y Fx(Pro)o(vides)16 b(to)f Fv(MPI)g Fx(a)h(bu\013er)g(in)h(the)e (user's)h(memory)f(to)h(b)q(e)g(used)h(for)e(bu\013ering)h(outgoing)g (mes-)75 973 y(sages.)22 b(The)16 b(bu\013er)g(is)h(used)g(only)f(b)o (y)g(messages)g(sen)o(t)f(in)i(bu\013ered)g(mo)q(de.)23 b(Only)17 b(one)f(bu\013er)g(can)g(b)q(e)75 1029 y(attac)o(hed)f(to)f (a)h(pro)q(cess)h(at)e(a)h(time.)75 1180 y Fv(MPI)p 160 1180 14 2 v 16 w(BUFFER)p 345 1180 V 17 w(DET)l(A)o(CH\()g(bu\013er,)g (size\))117 1258 y Fk(OUT)108 b Fv(bu\013er)478 b Fk(initial)12 b(bu\013er)j(address)g(\(c)o(hoice\))117 1333 y(OUT)108 b Fv(size)519 b Fk(bu\013er)15 b(size,)f(in)g(b)o(ytes)g(\(in)o (teger\))75 1457 y Fu(int)23 b(MPI)p 245 1457 15 2 v 17 w(Buffer)p 406 1457 V 17 w(detach\()f(void**)i(buffer,)e(int*)i (size\))75 1544 y(MPI)p 150 1544 V 17 w(BUFFER)p 311 1544 V 16 w(DETACH\()f(BUFFER,)g(SIZE,)g(IERROR\))170 1600 y()g(BUFFER\(*\))170 1657 y(INTEGER)g(SIZE,)h(IERROR)166 1743 y Fx(Detac)o(h)19 b(the)h(bu\013er)f(curren)o(tly)i(asso)q(ciated) f(with)g Fv(MPI)p Fx(.)e(This)j(op)q(eration)e(will)j(blo)q(c)o(k)e(un) o(til)h(all)75 1800 y(messages)16 b(curren)o(tly)h(in)g(the)f(bu\013er) g(ha)o(v)o(e)g(b)q(een)i(transmitted.)k(Up)q(on)17 b(return)f(of)g (this)h(function,)g(the)75 1856 y(user)e(ma)o(y)g(reuse)g(or)g(deallo)q (cate)h(the)g(space)f(tak)o(en)g(b)o(y)g(the)g(bu\013er.)166 1912 y(The)j(statemen)o(ts)f(made)g(in)i(this)f(section)h(describ)q(e)g (the)f(b)q(eha)o(vior)g(of)g Fv(MPI)f Fx(for)g(bu\013ered-mo)q(de)75 1969 y(sends.)38 b(When)22 b(no)f(bu\013er)g(is)g(curren)o(tly)h(asso)q (ciated,)g Fv(MPI)f Fx(b)q(eha)o(v)o(es)g(as)g(if)g(a)g(zero-sized)h (bu\013er)f(is)75 2025 y(asso)q(ciated)15 b(with)h(the)f(pro)q(cess.) 166 2082 y Fv(MPI)20 b Fx(m)o(ust)g(pro)o(vide)h(as)f(m)o(uc)o(h)h (bu\013ering)g(for)f(outgoing)g(messages)g Fl(as)h(if)30 b Fx(outgoing)20 b(message)75 2138 y(data)d(w)o(ere)h(bu\013ered)h(b)o (y)f(the)g(sending)h(pro)q(cess,)g(in)g(the)f(sp)q(eci\014ed)i (bu\013er)e(space,)h(using)f(a)g(circular,)75 2195 y(con)o (tiguous-space)e(allo)q(cation)h(p)q(olicy)l(.)24 b(W)l(e)16 b(outline)h(b)q(elo)o(w)f(a)g(mo)q(del)h(implemen)o(tation)g(that)e (de\014nes)75 2251 y(this)i(p)q(olicy)l(.)24 b Fv(MPI)16 b Fx(ma)o(y)g(pro)o(vide)g(more)g(bu\013ering,)h(and)f(ma)o(y)g(use)g (a)g(b)q(etter)g(bu\013er)h(allo)q(cation)g(algo-)75 2308 y(rithm)i(than)g(describ)q(ed)i(b)q(elo)o(w.)31 b(On)19 b(the)g(other)g(hand,)h Fv(MPI)e Fx(ma)o(y)g(signal)i(an)e (error)h(whenev)o(er)g(the)75 2364 y(simple)14 b(bu\013ering)f(allo)q (cator)f(describ)q(ed)i(b)q(elo)o(w)f(w)o(ould)g(run)g(out)f(of)g (space.)19 b(In)13 b(particular,)g(if)g(no)f(bu\013er)75 2421 y(is)k(explicitly)i(asso)q(ciated)d(with)h(the)f(pro)q(cess,)g (then)h(an)o(y)e(bu\013ered)i(send)g(ma)o(y)e(cause)i(an)f(error.)166 2477 y Fv(MPI)10 b Fx(do)q(es)g(not)g(pro)o(vide)h(mec)o(hanisms)f(for) g(querying)h(or)f(con)o(trolling)h(bu\013ering)g(done)f(b)o(y)g (standard)75 2533 y(mo)q(de)16 b(sends.)22 b(It)15 b(is)h(exp)q(ected)h (that)e(v)o(endors)h(will)h(pro)o(vide)f(suc)o(h)g(information)g(for)f (their)h(implemen-)75 2590 y(tations.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 42 34 41 bop 75 -100 a Fx(34)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fl(R)n(ationale.)38 b Fx(There)13 b(is)g(a)g(wide)g(sp)q(ectrum)h(of)e(p)q(ossible)j (implemen)o(tations)f(of)e(bu\013ered)h(comm)o(u-)189 102 y(nication:)18 b(bu\013ering)12 b(can)f(b)q(e)h(done)f(at)g (sender,)h(at)e(receiv)o(er,)i(or)f(b)q(oth;)h(bu\013ers)f(can)g(b)q(e) h(dedicated)189 158 y(to)g(one)g(sender-receiv)o(er)i(pair,)g(or)e(b)q (e)h(shared)g(b)o(y)f(all)i(comm)o(unications;)f(bu\013ering)h(can)e(b) q(e)i(done)189 214 y(in)f(real)g(or)f(in)h(virtual)g(memory;)g(it)g (can)f(use)h(dedicated)h(memory)l(,)e(or)g(memory)g(shared)h(b)o(y)f (other)189 271 y(pro)q(cesses;)19 b(bu\013er)f(space)g(ma)o(y)f(b)q(e)h (allo)q(cated)h(statically)g(or)e(b)q(e)i(c)o(hanged)f(dynamically;)j (etc..)189 327 y(It)14 b(do)q(es)h(not)f(seem)g(feasible)i(to)e(pro)o (vide)g(a)h(p)q(ortable)f(mec)o(hanism)h(for)f(querying)h(or)f(con)o (trolling)189 384 y(bu\013ering)21 b(that)g(w)o(ould)g(b)q(e)h (compatible)g(with)f(all)h(these)f(c)o(hoices,)i(y)o(et)e(pro)o(vide)h (meaningful)189 440 y(information.)e(\()p Fl(End)15 b(of)i(r)n (ationale.)p Fx(\))75 562 y Fj(3.6.1)49 b(Mo)q(del)18 b(implem)n(entation)c(of)i(bu\013ered)g(mo)q(de)75 648 y Fx(The)22 b(mo)q(del)g(implemen)o(tation)h(uses)f(the)f(pac)o(king)h (and)g(unpac)o(king)g(functions)g(describ)q(ed)i(in)e(Sec-)75 704 y(tion)15 b(3.13)f(and)i(the)f(non)o(blo)q(c)o(king)i(comm)o (unication)f(functions)f(describ)q(ed)j(in)e(Section)g(3.7.)166 761 y(W)l(e)21 b(assume)f(that)g(a)g(circular)i(queue)f(of)f(p)q (ending)j(message)d(en)o(tries)h(\(PME\))e(is)i(main)o(tained.)75 817 y(Eac)o(h)14 b(en)o(try)h(con)o(tains)f(a)g(comm)o(unication)i (request)e(handle)i(that)e(iden)o(ti\014es)i(a)e(p)q(ending)j(non)o (blo)q(c)o(king)75 874 y(send,)h(a)e(p)q(oin)o(ter)i(to)e(the)h(next)g (en)o(try)f(and)i(the)f(pac)o(k)o(ed)g(message)f(data.)25 b(The)17 b(en)o(tries)g(are)g(stored)f(in)75 930 y(successiv)o(e)j(lo)q (cations)h(in)f(the)f(bu\013er.)29 b(F)l(ree)19 b(space)f(is)h(a)o(v)m (ailable)h(b)q(et)o(w)o(een)f(the)f(queue)h(tail)g(and)g(the)75 987 y(queue)d(head.)166 1043 y(A)f(bu\013ered)h(send)g(call)g(results)f (in)i(the)e(execution)h(of)f(the)g(follo)o(wing)h(co)q(de.)143 1137 y Ft(\017)23 b Fx(T)l(ra)o(v)o(erse)10 b(sequen)o(tially)i(the)g (PME)e(queue)i(from)e(head)h(bac)o(kw)o(ard,)g(deleting)h(all)g(en)o (tries)g(for)e(com-)189 1193 y(m)o(unications)15 b(that)g(ha)o(v)o(e)f (completed,)i(up)f(to)g(the)g(\014rst)f(en)o(try)h(with)g(an)g (uncompleted)h(request;)189 1250 y(up)q(date)f(queue)h(head)g(to)f(p)q (oin)o(t)g(to)g(that)f(en)o(try)l(.)143 1344 y Ft(\017)23 b Fx(Compute)18 b(the)g(n)o(um)o(b)q(er,)h Fv(n)p Fx(,)g(of)e(b)o(ytes) h(needed)i(to)d(store)h(en)o(try)f(for)h(new)g(message)g(\(length)g(of) 189 1400 y(pac)o(k)o(ed)h(message)h(computed)g(with)g Fv(MPI)p 930 1400 14 2 v 16 w(P)l(A)o(CK)p 1061 1400 V 16 w(SIZE)g Fx(plus)h(space)f(for)f(request)h(handle)h(and)189 1456 y(p)q(oin)o(ter\).)143 1550 y Ft(\017)i Fx(Find)14 b(the)g(next)g(con)o(tiguous)f(empt)o(y)h(space)g(of)f Fv(n)h Fx(b)o(ytes)g(in)g(bu\013er)g(\(space)g(follo)o(wing)g(queue)h (tail,)189 1607 y(or)d(space)h(at)f(start)g(of)g(bu\013er)h(if)h(queue) f(tail)h(is)f(to)q(o)f(close)i(to)e(end)i(of)e(bu\013er\).)19 b(If)13 b(space)g(not)g(found)189 1663 y(then)i(raise)h(bu\013er)f(o)o (v)o(er\015o)o(w)f(error.)143 1757 y Ft(\017)23 b Fx(App)q(end)14 b(to)d(end)j(of)e(PME)g(queue)h(in)g(con)o(tiguous)g(space)f(the)h(new) g(en)o(try)f(that)f(con)o(tains)i(request)189 1813 y(handle,)j(next)f (p)q(oin)o(ter)h(and)f(pac)o(k)o(ed)g(message)g(data;)f Fv(MPI)p 1221 1813 V 16 w(P)l(A)o(CK)h Fx(is)h(used)g(to)e(pac)o(k)h (data.)143 1907 y Ft(\017)23 b Fx(P)o(ost)14 b(non)o(blo)q(c)o(king)i (send)g(\(standard)f(mo)q(de\))g(for)f(pac)o(k)o(ed)i(data.)143 2001 y Ft(\017)23 b Fx(Return)75 2144 y Fn(3.7)59 b(Nonblo)r(cking)20 b(communication)75 2246 y Fx(One)e(can)g(impro)o(v)o(e)f(p)q (erformance)g(on)h(man)o(y)e(systems)h(b)o(y)h(o)o(v)o(erlapping)f (comm)o(unication)i(and)e(com-)75 2302 y(putation.)35 b(This)21 b(is)g(esp)q(ecially)i(true)d(on)g(systems)g(where)g(comm)o (unication)h(can)g(b)q(e)g(executed)g(au-)75 2359 y(tonomously)12 b(b)o(y)h(an)g(in)o(telligen)o(t)h(comm)o(unication)g(con)o(troller.)19 b(Ligh)o(t-w)o(eigh)o(t)13 b(threads)f(are)h(one)g(mec)o(h-)75 2415 y(anism)21 b(for)g(ac)o(hieving)h(suc)o(h)g(o)o(v)o(erlap.)37 b(An)21 b(alternativ)o(e)h(mec)o(hanism)g(that)e(often)h(leads)g(to)g (b)q(etter)75 2472 y(p)q(erformance)15 b(is)g(to)g(use)g Fq(non)o(blo)q(c)o(king)j(comm)o(unication)p Fx(.)j(A)15 b(non)o(blo)q(c)o(king)h Fq(send)g(start)f Fx(call)h(ini-)75 2528 y(tiates)g(the)h(send)g(op)q(eration,)f(but)h(do)q(es)f(not)g (complete)h(it.)24 b(The)17 b(send)g(start)e(call)i(will)h(return)f(b)q (efore)75 2584 y(the)g(message)f(w)o(as)f(copied)j(out)e(of)g(the)g (send)h(bu\013er.)24 b(A)17 b(separate)e Fq(send)k(complete)e Fx(call)h(is)f(needed)75 2641 y(to)12 b(complete)i(the)f(comm)o (unication,)h(i.e.,)g(to)e(v)o(erify)h(that)g(the)g(data)f(has)h(b)q (een)h(copied)h(out)e(of)f(the)h(send)75 2697 y(bu\013er.)19 b(With)13 b(suitable)i(hardw)o(are,)d(the)h(transfer)g(of)f(data)h(out) g(of)f(the)h(sender)h(memory)f(ma)o(y)f(pro)q(ceed)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 43 35 42 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(35)75 45 y(concurren)o(tly)12 b(with)h(computations)e(done)h(at)g (the)g(sender)g(after)f(the)h(send)g(w)o(as)f(initiated)j(and)e(b)q (efore)g(it)75 102 y(completed.)21 b(Similarly)l(,)16 b(a)e(non)o(blo)q(c)o(king)i Fq(receiv)o(e)g(start)h(call)f Fx(initiates)g(the)f(receiv)o(e)g(op)q(eration,)g(but)75 158 y(do)q(es)f(not)g(complete)h(it.)k(The)14 b(call)i(will)f(return)f (b)q(efore)g(a)g(message)g(is)g(stored)f(in)o(to)h(the)g(receiv)o(e)h (bu\013er.)75 214 y(A)i(separate)f Fq(receiv)o(e)i(complete)f Fx(call)h(is)f(needed)h(to)e(complete)h(the)g(receiv)o(e)g(op)q (eration)g(and)g(v)o(erify)75 271 y(that)11 b(the)i(data)e(has)h(b)q (een)h(receiv)o(ed)h(in)o(to)e(the)g(receiv)o(e)h(bu\013er.)19 b(With)12 b(suitable)i(hardw)o(are,)d(the)i(transfer)75 327 y(of)j(data)g(in)o(to)g(the)h(receiv)o(er)g(memory)f(ma)o(y)f(pro)q (ceed)j(concurren)o(tly)f(with)g(computations)f(done)h(after)75 384 y(the)d(receiv)o(e)g(w)o(as)f(initiated)j(and)e(b)q(efore)g(it)g (completed.)20 b(The)14 b(use)g(of)f(non)o(blo)q(c)o(king)i(receiv)o (es)g(ma)o(y)e(also)75 440 y(a)o(v)o(oid)18 b(system)g(bu\013ering)h (and)f(memory-to-memory)f(cop)o(ying,)i(as)f(information)h(is)g(pro)o (vided)g(early)75 497 y(on)c(the)g(lo)q(cation)h(of)f(the)g(receiv)o(e) i(bu\013er.)166 553 y(Non)o(blo)q(c)o(king)f(send)f(start)f(calls)h (can)g(use)g(the)g(same)f(four)h(mo)q(des)g(as)f(blo)q(c)o(king)i (sends:)k Fv(standa)o(rd)p Fx(,)75 610 y Fv(bu\013ered)p Fx(,)c Fv(synchronous)i Fx(and)e Fv(ready)p Fx(.)22 b(These)16 b(carry)f(the)h(same)f(meaning.)23 b(Sends)16 b(of)g(all)g(mo)q(des,)g Fv(ready)75 666 y Fx(excepted,)f(can)f(b)q(e)g(started)f(whether)h(a)g (matc)o(hing)g(receiv)o(e)g(has)g(b)q(een)h(p)q(osted)f(or)g(not;)f(a)h (non)o(blo)q(c)o(king)75 723 y Fv(ready)e Fx(send)g(can)f(b)q(e)i (started)d(only)j(if)f(a)f(matc)o(hing)h(receiv)o(e)g(is)g(p)q(osted.) 19 b(In)12 b(all)h(cases,)f(the)f(send)h(start)f(call)75 779 y(is)g(lo)q(cal:)19 b(it)11 b(returns)f(immediately)l(,)j(irresp)q (ectiv)o(e)f(of)e(the)h(status)f(of)g(other)g(pro)q(cesses.)19 b(If)10 b(the)h(call)h(causes)75 835 y(some)17 b(system)f(resource)h (to)f(b)q(e)i(exhausted,)f(then)g(it)g(will)i(fail)f(and)f(return)g(an) f(error)h(co)q(de.)25 b(Qualit)o(y)75 892 y(implemen)o(tations)14 b(of)f Fv(MPI)f Fx(should)i(ensure)f(that)g(this)g(happ)q(ens)h(only)f (in)h(\\pathological")f(cases.)20 b(That)75 948 y(is,)13 b(an)f Fv(MPI)f Fx(implemen)o(tation)i(should)g(b)q(e)g(able)f(to)g (supp)q(ort)g(a)f(large)h(n)o(um)o(b)q(er)g(of)g(p)q(ending)i(non)o (blo)q(c)o(king)75 1005 y(op)q(erations.)166 1061 y(The)j (send-complete)i(call)f(returns)f(when)h(data)e(has)h(b)q(een)i(copied) f(out)f(of)f(the)i(send)f(bu\013er.)26 b(It)75 1118 y(ma)o(y)14 b(carry)h(additional)i(meaning,)e(dep)q(ending)j(on)d(the)g(send)h(mo)q (de.)166 1174 y(If)e(the)g(send)h(mo)q(de)f(is)h Fv(synchronous)p Fx(,)g(then)g(the)f(send)g(can)h(complete)f(only)h(if)f(a)g(matc)o (hing)g(receiv)o(e)75 1231 y(has)j(started.)24 b(That)17 b(is,)g(a)g(receiv)o(e)h(has)f(b)q(een)h(p)q(osted,)f(and)g(has)g(b)q (een)h(matc)o(hed)f(with)g(the)h(send.)25 b(In)75 1287 y(this)18 b(case,)f(the)g(send-complete)i(call)f(is)g(non-lo)q(cal.)27 b(Note)16 b(that)h(a)g(sync)o(hronous,)g(non)o(blo)q(c)o(king)h(send)75 1344 y(ma)o(y)13 b(complete,)h(if)g(matc)o(hed)g(b)o(y)f(a)g(non)o(blo) q(c)o(king)i(receiv)o(e,)g(b)q(efore)f(the)f(receiv)o(e)i(complete)f (call)h(o)q(ccurs.)75 1400 y(\(It)h(can)h(complete)g(as)f(so)q(on)g(as) g(the)h(sender)g(\\kno)o(ws")e(the)i(transfer)e(will)j(complete,)g(but) e(b)q(efore)h(the)75 1456 y(receiv)o(er)f(\\kno)o(ws")e(the)h(transfer) g(will)i(complete.\))166 1513 y(If)f(the)g(send)h(mo)q(de)f(is)h Fv(bu\013ered)g Fx(then)f(the)g(message)g(m)o(ust)g(b)q(e)g(bu\013ered) h(if)f(there)h(is)f(no)g(p)q(ending)75 1569 y(receiv)o(e.)27 b(In)18 b(this)g(case,)f(the)g(send-complete)i(call)f(is)g(lo)q(cal,)h (and)e(m)o(ust)g(succeed)h(irresp)q(ectiv)o(e)h(of)e(the)75 1626 y(status)d(of)h(a)g(matc)o(hing)g(receiv)o(e.)166 1682 y(If)f(the)g(send)g(mo)q(de)g(is)g Fv(standa)o(rd)h Fx(then)g(the)e(send-complete)j(call)f(ma)o(y)e(return)g(b)q(efore)h(a) g(matc)o(hing)75 1739 y(receiv)o(e)h(o)q(ccurred,)f(if)h(the)e(message) h(is)g(bu\013ered.)20 b(On)15 b(the)f(other)f(hand,)h(the)g(send)h(ma)o (y)e(not)g(complete)75 1795 y(un)o(til)j(a)f(matc)o(hing)g(receiv)o(e)i (o)q(ccurred,)e(and)g(the)h(message)e(w)o(as)h(copied)h(in)o(to)f(the)h (receiv)o(e)g(bu\013er.)166 1852 y(Non)o(blo)q(c)o(king)g(sends)g(can)f (b)q(e)h(matc)o(hed)f(with)h(blo)q(c)o(king)g(receiv)o(es,)g(and)f (vice-v)o(ersa.)189 1958 y Fl(A)n(dvic)n(e)h(to)i(users.)50 b Fx(The)17 b(completion)g(of)g(a)f(send)h(op)q(eration)g(ma)o(y)f(b)q (e)i(dela)o(y)o(ed,)f(for)f(standard)189 2014 y(mo)q(de,)d(and)g(m)o (ust)f(b)q(e)h(dela)o(y)o(ed,)h(for)e(sync)o(hronous)g(mo)q(de,)h(un)o (til)h(a)f(matc)o(hing)f(receiv)o(e)i(is)f(p)q(osted.)189 2071 y(The)k(use)g(of)f(non)o(blo)q(c)o(king)j(sends)e(in)h(these)f(t)o (w)o(o)e(cases)i(allo)o(ws)g(the)g(sender)h(to)e(pro)q(ceed)i(ahead)189 2127 y(of)c(the)g(receiv)o(er,)h(so)f(that)g(the)h(computation)f(is)h (more)f(toleran)o(t)g(of)g(\015uctuations)h(in)g(the)g(sp)q(eeds)189 2184 y(of)f(the)i(t)o(w)o(o)e(pro)q(cesses.)189 2259 y(Non)o(blo)q(c)o(king)k(sends)g(in)f(the)h(bu\013ered)f(and)h(ready)f (mo)q(des)g(ha)o(v)o(e)g(a)f(more)h(limited)i(impact.)26 b(A)189 2315 y(non)o(blo)q(c)o(king)16 b(send)g(will)h(return)e(as)g (so)q(on)g(as)g(p)q(ossible,)i(whereas)e(a)g(blo)q(c)o(king)h(send)g (will)h(return)189 2372 y(after)h(the)h(data)g(has)g(b)q(een)h(copied)h (out)e(of)f(the)i(sender)f(memory)l(.)32 b(The)19 b(use)h(of)f(non)o (blo)q(c)o(king)189 2428 y(sends)j(is)g(adv)m(an)o(tageous)f(in)h (these)g(cases)g(only)g(if)g(data)f(cop)o(ying)h(can)g(b)q(e)g (concurren)o(t)g(with)189 2485 y(computation.)189 2560 y(The)e(message-passing)g(mo)q(del)g(implies)i(that)d(comm)o(unication) i(is)f(initiated)h(b)o(y)f(the)g(sender.)189 2616 y(The)d(comm)o (unication)g(will)h(generally)g(ha)o(v)o(e)e(lo)o(w)o(er)g(o)o(v)o (erhead)h(if)g(a)f(receiv)o(e)i(is)f(already)g(p)q(osted)189 2673 y(when)22 b(the)h(sender)g(initiates)g(the)f(comm)o(unication)h (\(data)f(can)g(b)q(e)h(mo)o(v)o(ed)f(directly)h(to)f(the)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 44 36 43 bop 75 -100 a Fx(36)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fx(receiv)o(e)j (bu\013er,)f(and)g(there)g(is)h(no)f(need)h(to)e(queue)i(a)e(p)q (ending)j(send)f(request\).)31 b(Ho)o(w)o(ev)o(er,)18 b(a)189 102 y(receiv)o(e)g(op)q(eration)f(can)h(complete)g(only)g (after)e(the)i(matc)o(hing)f(send)h(has)f(o)q(ccurred.)27 b(The)18 b(use)189 158 y(of)13 b(non)o(blo)q(c)o(king)i(receiv)o(es)f (allo)o(ws)g(one)g(to)f(ac)o(hiev)o(e)h(lo)o(w)o(er)f(comm)o(unication) i(o)o(v)o(erheads)e(without)189 214 y(blo)q(c)o(king)j(the)f(receiv)o (er)h(while)h(it)f(w)o(aits)e(for)h(the)g(send.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))75 336 y Fj(3.7.1)49 b(Comm)o(unication)14 b(Objects)75 422 y Fx(Non)o(blo)q(c)o(king)19 b(comm)o(unications)f(use)g(opaque)g Fv(request)h Fx(ob)s(jects)e(to)g(iden)o(tify)i(comm)o(unication)g(op)q (er-)75 478 y(ations)e(and)g(matc)o(h)f(the)i(op)q(eration)f(that)f (initiates)i(the)f(comm)o(unication)h(with)f(the)g(op)q(eration)h(that) 75 535 y(terminates)f(it.)27 b(These)17 b(are)g(system)g(ob)s(jects)g (that)f(are)h(accessed)h(via)g(a)f(handle.)27 b(A)17 b(request)h(ob)s(ject)75 591 y(iden)o(ti\014es)c(v)m(arious)f(prop)q (erties)h(of)e(a)g(comm)o(unication)h(op)q(eration,)g(suc)o(h)g(as)f (the)h(send)g(mo)q(de,)g(the)g(com-)75 648 y(m)o(unication)i(bu\013er)f (that)g(is)h(asso)q(ciated)f(with)h(it,)f(its)g(con)o(text,)g(the)g (tag)f(and)i(destination)g(argumen)o(ts)75 704 y(to)e(b)q(e)h(used)f (for)g(a)g(send,)h(or)e(the)i(tag)e(and)i(source)f(argumen)o(ts)f(to)h (b)q(e)h(used)g(for)e(a)h(receiv)o(e.)20 b(In)14 b(addition,)75 761 y(this)i(ob)s(ject)e(stores)h(information)g(ab)q(out)g(the)g (status)g(of)f(the)i(p)q(ending)h(comm)o(unication)f(op)q(eration.)75 882 y Fj(3.7.2)49 b(Comm)o(unication)14 b(initiation)75 968 y Fx(W)l(e)j(use)h(the)g(same)f(naming)g(con)o(v)o(en)o(tions)h(as) f(for)g(blo)q(c)o(king)h(comm)o(unication:)25 b(a)17 b(pre\014x)h(of)f Fv(B)p Fx(,)g Fv(S)p Fx(,)g(or)75 1025 y Fv(R)f Fx(is)h(used)f(for)g Fv(bu\013ered)p Fx(,)g Fv(synchronous)i Fx(or)e Fv(ready)g Fx(mo)q(de.)22 b(In)17 b(addition)g(a)f(pre\014x)g(of)g Fv(I)f Fx(\(for)g Fv(imm)m(ediate)p Fx(\))75 1081 y(indicates)i(that)d(the)h(call)i(is)e(non)o(blo)q(c)o (king.)75 1232 y Fv(MPI)p 160 1232 14 2 v 16 w(ISEND\(buf,)g(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm)m(,)d(request\))117 1309 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1384 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 1460 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 1535 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1610 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1685 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1760 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1885 y Fu(int)23 b(MPI)p 245 1885 15 2 v 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1885 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1941 y(int)h(tag,)f(MPI)p 683 1941 V 17 w(Comm)g(comm,)g(MPI)p 1034 1941 V 17 w(Request)g(*request\))75 2027 y(MPI)p 150 2027 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g (COMM,)h(REQUEST,)e(IERROR\))170 2084 y()h(BUF\(*\))170 2140 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 2227 y Fx(Start)14 b(a)h(standard)g(mo)q(de,)g(non)o(blo)q (c)o(king)h(send.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 45 37 44 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(37)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(IBSEND\(buf,)15 b(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm)m(,)d(request\))117 122 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Ibsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(IBSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 897 y()g(BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1040 y Fx(Start)14 b(a)h(bu\013ered)h(mo)q(de,)f(non)o(blo) q(c)o(king)h(send.)75 1191 y Fv(MPI)p 160 1191 14 2 v 16 w(ISSEND\(buf,)f(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm)m(,) d(request\))117 1268 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1343 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1418 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1493 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1569 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1644 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Fu(int)23 b(MPI)p 245 1843 15 2 v 17 w(Issend\(void*)f(buf,)i (int)f(count,)g(MPI)p 1026 1843 V 17 w(Datatype)g(datatype,)f(int)i (dest,)393 1900 y(int)g(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g (MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(ISSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 2186 y Fx(Start)14 b(a)h(sync)o(hronous)g(mo)q(de,)g(non)o (blo)q(c)o(king)i(send.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 46 38 45 bop 75 -100 a Fx(38)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(IRSEND\(buf,)f(count,)g(datat)o(yp)q(e,)g(dest,)g (tag,)f(comm)m(,)d(request\))117 122 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h (bu\013er)g(elemen)o(t)e(\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(IRSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 897 y()g(BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1040 y Fx(Start)14 b(a)h(ready)g(mo)q(de)h(non)o(blo)q(c)o (king)g(send.)75 1191 y Fv(MPI)p 160 1191 14 2 v 16 w(IRECV)f(\(buf,)h (count,)g(datat)o(yp)q(e,)g(source,)g(tag,)e(comm)n(,)d(request\))117 1268 y Fk(OUT)108 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 1343 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1418 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(handle\))117 1493 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o(teger\))117 1569 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1644 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Fu(int)23 b(MPI)p 245 1843 15 2 v 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1843 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 1900 y(int)h(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g(MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(IRECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR)166 2186 y Fx(Start)14 b(a)h(non)o(blo)q(c)o(king)i (receiv)o(e.)166 2242 y(These)f(calls)h(allo)q(cate)g(a)e(comm)o (unication)i(request)f(ob)s(ject)f(and)h(asso)q(ciate)g(it)g(with)g (the)g(request)75 2298 y(handle)j(\(the)f(argumen)o(t)g Fv(request)p Fx(\).)30 b(The)18 b(request)h(can)f(b)q(e)h(used)f(later) h(to)e(query)i(the)f(status)f(of)h(the)75 2355 y(comm)o(unication)e(or) f(w)o(ait)f(for)h(its)g(completion.)166 2411 y(A)i(non)o(blo)q(c)o (king)i(send)f(call)g(indicates)h(that)e(the)g(system)g(ma)o(y)g(start) f(cop)o(ying)h(data)g(out)g(of)g(the)75 2468 y(send)g(bu\013er.)24 b(The)17 b(sender)g(should)g(not)g(access)f(an)o(y)g(part)g(of)g(the)h (send)g(bu\013er)g(after)e(a)i(non)o(blo)q(c)o(king)75 2524 y(send)f(op)q(eration)f(is)h(called,)g(un)o(til)h(the)e(send)h (completes.)166 2581 y(A)d(non)o(blo)q(c)o(king)h(receiv)o(e)g(call)g (indicates)g(that)e(the)h(system)f(ma)o(y)g(start)g(writing)h(data)f (in)o(to)h(the)g(re-)75 2637 y(ceiv)o(e)f(bu\013er.)19 b(The)11 b(receiv)o(er)h(should)h(not)e(access)g(an)o(y)g(part)g(of)g (the)g(receiv)o(e)i(bu\013er)e(after)g(a)g(non)o(blo)q(c)o(king)75 2694 y(receiv)o(e)16 b(op)q(eration)g(is)f(called,)i(un)o(til)f(the)f (receiv)o(e)h(completes.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 47 39 46 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(39)75 45 y Fj(3.7.3)49 b(Comm)o(unication)14 b(Completion)75 131 y Fx(The)j(functions)g Fv(MPI)p 452 131 14 2 v 16 w(W)l(AIT)f Fx(and)g Fv(MPI)p 768 131 V 16 w(TEST)h Fx(are)f(used)h(to) f(complete)h(a)f(non)o(blo)q(c)o(king)h(comm)o(unica-)75 187 y(tion.)i(The)13 b(completion)h(of)e(a)g(send)h(op)q(eration)g (indicates)h(that)e(the)g(sender)h(is)g(no)o(w)f(free)h(to)f(up)q(date) h(the)75 244 y(lo)q(cations)k(in)f(the)h(send)f(bu\013er)g(\(the)g (send)g(op)q(eration)g(itself)h(lea)o(v)o(es)f(the)g(con)o(ten)o(t)g (of)f(the)h(send)h(bu\013er)75 300 y(unc)o(hanged\).)25 b(It)17 b(do)q(es)g(not)f(indicate)i(that)e(the)h(message)f(has)h(b)q (een)g(receiv)o(ed,)h(rather,)e(it)h(ma)o(y)f(ha)o(v)o(e)75 357 y(b)q(een)g(bu\013ered)g(b)o(y)f(the)h(comm)o(unication)g (subsystem.)k(Ho)o(w)o(ev)o(er,)14 b(if)i(a)f Fv(synchronous)i Fx(mo)q(de)f(send)g(w)o(as)75 413 y(used,)f(the)g(completion)h(of)f (the)f(send)i(op)q(eration)f(indicates)h(that)e(a)h(matc)o(hing)g (receiv)o(e)h(w)o(as)e(initiated,)75 470 y(and)h(that)g(the)g(message)g (will)i(ev)o(en)o(tually)f(b)q(e)g(receiv)o(ed)g(b)o(y)f(this)h(matc)o (hing)f(receiv)o(e.)166 526 y(The)20 b(completion)h(of)e(a)g(receiv)o (e)i(op)q(eration)f(indicates)h(that)e(the)h(receiv)o(e)g(bu\013er)g (con)o(tains)g(the)75 583 y(receiv)o(ed)d(message,)e(the)h(receiv)o(er) h(is)f(no)o(w)f(free)h(to)f(access)h(it,)g(and)g(that)f(the)h(status)f (ob)s(ject)g(is)h(set.)22 b(It)75 639 y(not)13 b(indicate)i(that)d(the) i(matc)o(hing)f(send)h(op)q(eration)g(has)f(completed)h(\(but)g (indicates,)g(of)f(course,)h(that)75 695 y(the)h(send)h(w)o(as)e (initiated\).)166 752 y(W)l(e)27 b(shall)h(use)g(the)f(follo)o(wing)h (terminology)l(.)56 b(A)27 b Fq(n)o(ull)g Fx(handle)i(is)e(a)g(handle)h (with)g(v)m(alue)75 808 y Fi(MPI)p 152 808 13 2 v 14 w(REQUEST)p 354 808 V 14 w(NULL)p Fx(.)17 b(A)g(p)q(ersisten)o(t)g (request)f(and)h(the)g(handle)h(to)e(it)g(are)h Fq(inactiv)o(e)g Fx(if)g(the)g(request)75 865 y(is)f(not)f(asso)q(ciated)h(with)g(an)o (y)f(ongoing)h(comm)o(unication)g(\(see)g(Section)g(3.9\).)k(A)15 b(handle)i(is)f Fq(activ)o(e)g Fx(if)75 921 y(it)f(is)h(neither)g(n)o (ull)h(nor)e(inactiv)o(e.)75 1072 y Fv(MPI)p 160 1072 14 2 v 16 w(W)l(AIT\(request,)h(status\))117 1149 y Fk(INOUT)62 b Fv(request)452 b Fk(request)16 b(\(handle\))117 1225 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1349 y Fu(int)23 b(MPI)p 245 1349 15 2 v 17 w(Wait\(MPI)p 454 1349 V 16 w(Request)g(*request,)g(MPI)p 972 1349 V 17 w(Status)g(*status\))75 1435 y(MPI)p 150 1435 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 1492 y(INTEGER)g(REQUEST,) g(STATUS\(MPI)p 819 1492 V 16 w(STATUS)p 979 1492 V 16 w(SIZE\),)g(IERROR)166 1578 y Fx(A)18 b(call)h(to)e Fv(MPI)p 447 1578 14 2 v 16 w(W)l(AIT)g Fx(returns)h(when)g(the)g(op)q(eration)g (iden)o(ti\014ed)i(b)o(y)e Fv(request)h Fx(is)f(complete.)29 b(If)75 1635 y(the)16 b(comm)o(unication)h(ob)s(ject)f(asso)q(ciated)g (with)h(this)f(request)g(w)o(as)g(created)g(b)o(y)g(a)g(non)o(blo)q(c)o (king)h(send)75 1691 y(or)i(receiv)o(e)h(call,)h(then)e(the)g(ob)s (ject)g(is)h(deallo)q(cated)g(b)o(y)f(the)h(call)g(to)37 b Fv(MPI)p 1412 1691 V 16 w(W)l(AIT)19 b Fx(and)h(the)f(request)75 1748 y(handle)d(is)g(set)f(to)g Fi(MPI)p 471 1748 13 2 v 14 w(REQUEST)p 673 1748 V 14 w(NULL)p Fx(.)g Fv(MPI)p 900 1748 14 2 v 16 w(W)l(AIT)g Fx(is)h(a)f(non-lo)q(cal)i(op)q (eration.)166 1804 y(The)i(call)h(returns,)f(in)g Fv(status)p Fx(,)j(information)c(on)h(the)g(completed)h(op)q(eration.)30 b(The)19 b(con)o(ten)o(t)f(of)75 1861 y(the)d(status)f(ob)s(ject)h(for) f(a)h(receiv)o(e)h(op)q(eration)f(can)h(b)q(e)f(accessed)h(as)f (describ)q(ed)i(in)f(section)f(3.2.5.)k(The)75 1917 y(status)13 b(ob)s(ject)g(for)f(a)i(send)g(op)q(eration)f(ma)o(y)g(b)q(e)h(queried) h(b)o(y)f(a)f(call)h(to)f Fv(MPI)p 1385 1917 V 16 w(TEST)p 1515 1917 V 17 w(CANCELLED)g Fx(\(see)75 1974 y(Section)j(3.8\).)166 2030 y(One)f(is)g(allo)o(w)o(ed)f(to)g(call)h Fv(MPI)p 688 2030 V 16 w(W)l(AIT)f Fx(with)h(a)e(n)o(ull)j(or)e(inactiv)o(e)h Fv(request)h Fx(argumen)o(t.)j(In)14 b(this)h(case)75 2086 y(the)10 b(op)q(eration)h(returns)f(immediately)l(.)20 b(The)10 b Fv(status)i Fx(argumen)o(t)e(is)g(set)g(to)g(return)g Fv(tag)g(=)g(MPI)p 1672 2086 V 16 w(ANY)p 1780 2086 V 17 w(T)l(A)o(G)p Fx(,)75 2143 y Fv(source)h(=)f(MPI)p 335 2143 V 16 w(ANY)p 443 2143 V 17 w(SOURCE)p Fx(,)h(and)g(is)f(also)g (in)o(ternally)i(con\014gured)f(so)f(that)f(calls)i(to)f Fv(MPI)p 1656 2143 V 16 w(GET)p 1760 2143 V 16 w(COUNT)75 2199 y Fx(and)15 b Fv(MPI)p 248 2199 V 16 w(GET)p 352 2199 V 17 w(ELEMENTS)g Fx(return)g Fv(count)i(=)e(0)p Fx(.)189 2306 y Fl(R)n(ationale.)52 b Fx(This)18 b(mak)o(es)e Fv(MPI)p 772 2306 V 16 w(W)l(AIT)h Fx(functionally)i(equiv)m(alen)o(t)g (to)e Fv(MPI)p 1528 2306 V 16 w(W)l(AIT)l(ALL)g Fx(with)g(a)189 2362 y(list)g(of)f(length)h(one)g(and)f(adds)h(some)f(elegance.)25 b(Status)16 b(is)g(set)h(in)g(this)g(w)o(a)o(y)e(so)h(as)g(to)g(prev)o (en)o(t)189 2419 y(errors)e(due)i(to)e(accesses)i(of)f(stale)g (information.)20 b(\()p Fl(End)c(of)g(r)n(ationale.)p Fx(\))189 2525 y Fl(A)n(dvic)n(e)k(to)h(implementors.)72 b Fx(In)21 b(a)f(m)o(ulti-threaded)i(en)o(vironmen)o(t,)f(a)f(call)i (to)40 b Fv(MPI)p 1749 2525 V 16 w(W)l(AIT)189 2581 y Fx(should)12 b(blo)q(c)o(k)g(only)g(the)g(calling)h(thread,)f(allo)o (wing)g(the)g(thread)f(sc)o(heduler)i(to)e(sc)o(hedule)i(another)189 2638 y(thread)i(for)f(execution.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 48 40 47 bop 75 -100 a Fx(40)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(TEST\(request,)f(\015ag,)f(status\))117 122 y Fk(INOUT)62 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))117 197 y(OUT)108 b Fv(\015ag)518 b Fi(true)15 b Fk(if)e(op)q(eration)h(completed)f(\(logical\))117 273 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\)) 75 397 y Fu(int)23 b(MPI)p 245 397 15 2 v 17 w(Test\(MPI)p 454 397 V 16 w(Request)g(*request,)g(int)g(*flag,)g(MPI)p 1234 397 V 17 w(Status)g(*status\))75 483 y(MPI)p 150 483 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 540 y(LOGICAL)g(FLAG)170 596 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 596 V 16 w(STATUS)p 979 596 V 16 w(SIZE\),)g(IERROR)166 683 y Fx(A)17 b(call)i(to)e Fv(MPI)p 446 683 14 2 v 15 w(TEST)h Fx(returns)f Fv(\015ag)h(=)f(true)h Fx(if)g(the)g(op)q (eration)f(iden)o(ti\014ed)j(b)o(y)d Fv(request)i Fx(is)f(com-)75 739 y(plete.)37 b(In)21 b(suc)o(h)g(a)f(case,)i(the)e(status)g(ob)s (ject)g(is)h(set)f(to)g(con)o(tain)h(information)g(on)f(the)h (completed)75 796 y(op)q(eration;)15 b(if)h(the)f(comm)o(unication)h (ob)s(ject)e(w)o(as)h(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)h (send)g(or)f(receiv)o(e,)h(then)75 852 y(it)g(is)f(deallo)q(cated)i (and)f(the)f(request)h(handle)g(is)g(set)f(to)g Fi(MPI)p 1127 852 13 2 v 14 w(REQUEST)p 1329 852 V 14 w(NULL)p Fx(.)h(The)f(call)i(returns)e Fv(\015ag)75 909 y(=)g(false)p Fx(,)g(otherwise.)20 b(In)c(this)g(case,)f(the)g(v)m(alue)h(of)f(the)g (status)f(ob)s(ject)h(is)h(unde\014ned.)21 b Fv(MPI)p 1664 909 14 2 v 16 w(TEST)16 b Fx(is)f(a)75 965 y(lo)q(cal)h(op)q (eration.)166 1022 y(The)11 b(return)g(status)f(ob)s(ject)h(for)f(a)h (receiv)o(e)h(op)q(eration)f(carries)g(information)g(that)g(can)g(b)q (e)g(accessed)75 1078 y(as)19 b(describ)q(ed)h(in)g(section)g(3.2.5.)29 b(The)19 b(status)g(ob)s(ject)f(for)g(a)h(send)g(op)q(eration)h (carries)f(information)75 1134 y(that)14 b(can)i(b)q(e)g(accessed)f(b)o (y)h(a)e(call)j(to)d Fv(MPI)p 823 1134 V 16 w(TEST)p 953 1134 V 17 w(CANCELLED)h Fx(\(see)g(Section)h(3.8\).)166 1191 y(One)i(is)f(allo)o(w)o(ed)g(to)g(call)h Fv(MPI)p 702 1191 V 15 w(TEST)f Fx(with)h(a)e(n)o(ull)j(or)d(inactiv)o(e)i Fv(request)g Fx(argumen)o(t.)25 b(In)17 b(suc)o(h)g(a)75 1247 y(case)e(the)g(op)q(eration)h(returns)f Fv(\015ag)g(=)h(false)p Fx(.)166 1304 y(The)22 b(functions)h Fv(MPI)p 554 1304 V 15 w(W)l(AIT)f Fx(and)g Fv(MPI)p 881 1304 V 16 w(TEST)g Fx(can)g(b)q(e)g(used)h(to)e(complete)h(b)q(oth)g(sends)h(and)75 1360 y(receiv)o(es.)189 1467 y Fl(A)n(dvic)n(e)e(to)h(users.)79 b Fx(The)22 b(use)g(of)f(the)h(non)o(blo)q(c)o(king)h Fv(MPI)p 1267 1467 V 16 w(TEST)f Fx(call)g(allo)o(ws)g(the)g(user)g(to) 189 1523 y(sc)o(hedule)e(alternativ)o(e)f(activities)h(within)g(a)f (single)h(thread)f(of)f(execution.)32 b(An)19 b(ev)o(en)o(t-driv)o(en) 189 1579 y(thread)14 b(sc)o(heduler)h(can)f(b)q(e)h(em)o(ulated)g(with) f(p)q(erio)q(dic)i(calls)f(to)f Fv(MPI)p 1382 1579 V 16 w(TEST)p Fx(.)g(\()p Fl(End)g(of)i(advic)n(e)f(to)189 1636 y(users.)p Fx(\))75 1742 y Fq(Example)j(3.10)23 b Fx(Simple)16 b(usage)f(of)g(non)o(blo)q(c)o(king)i(op)q(erations)e (and)g Fv(MPI)p 1378 1742 V 16 w(W)l(AIT)p Fx(.)75 1836 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1892 y(IF\(rank.EQ.0\))123 1949 y(THEN)170 2005 y(CALL)h (MPI_ISEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(request,)g (ierr\))170 2062 y(****)h(do)f(some)h(computation)e(to)i(mask)f (latency)g(****)170 2118 y(CALL)h(MPI_WAIT\(request,)d(status,)i (ierr\))123 2175 y(ELSE)170 2231 y(CALL)h(MPI_IRECV\(a\(1\),)e(15,)h (MPI_REAL,)g(0,)g(tag,)h(comm,)f(request,)g(ierr\))170 2288 y(****)h(do)f(some)h(computation)e(to)i(mask)f(latency)g(****)170 2344 y(CALL)h(MPI_WAIT\(request,)d(status,)i(ierr\))75 2401 y(END)g(IF)166 2507 y Fx(A)14 b(request)g(ob)s(ject)f(can)h(b)q(e) g(deallo)q(cated)h(without)f(w)o(aiting)g(for)f(the)h(asso)q(ciated)g (comm)o(unication)75 2563 y(to)h(complete,)g(b)o(y)g(using)h(the)f (follo)o(wing)h(op)q(eration.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 49 41 48 bop 75 -100 a Fm(3.7.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b Fx(41)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(REQUEST)p 379 45 V 18 w(FREE\(request\))117 122 y Fk(INOUT)62 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 247 y Fu(int)23 b(MPI)p 245 247 15 2 v 17 w(Request)p 430 247 V 16 w(free\(MPI)p 638 247 V 17 w(Request)f(*request\))75 333 y(MPI)p 150 333 V 17 w(REQUEST)p 335 333 V 16 w(FREE\(REQUEST,)g (IERROR\))170 390 y(INTEGER)h(REQUEST,)g(IERROR)166 476 y Fx(Mark)18 b(the)h(request)g(ob)s(ject)g(for)f(deallo)q(cation)j(and) e(set)g Fv(request)i Fx(to)d Fi(MPI)p 1467 476 13 2 v 14 w(REQUEST)p 1669 476 V 14 w(NULL)p Fx(.)i(An)75 533 y(ongoing)15 b(comm)o(unication)h(that)f(is)g(asso)q(ciated)h(with)f (the)h(request)f(will)i(complete)f(successfully)l(.)22 b(The)75 589 y(request)15 b(will)i(b)q(e)f(deallo)q(cated)h(only)e (after)g(its)g(completion.)189 694 y Fl(R)n(ationale.)51 b Fx(The)17 b Fv(MPI)p 622 694 14 2 v 16 w(REQUEST)p 841 694 V 18 w(FREE)g Fx(mec)o(hanism)h(is)f(pro)o(vided)h(for)f (reasons)f(of)h(p)q(erfor-)189 751 y(mance)e(and)g(con)o(v)o(enience)i (on)e(the)h(sending)g(side.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 856 y Fl(A)n(dvic)n(e)i(to)i(users.)68 b Fx(Once)21 b(a)f(request)f(is)i(freed)f(b)o(y)g(a)f(call)i(to)f Fv(MPI)p 1419 856 V 15 w(REQUEST)p 1637 856 V 18 w(FREE)p Fx(,)g(it)g(is)189 912 y(not)c(p)q(ossible)j(to)e(c)o(hec)o(k)g(for)g (the)g(successful)i(completion)f(of)f(the)g(asso)q(ciated)g(comm)o (unication)189 969 y(with)h(calls)h(to)e Fv(MPI)p 542 969 V 16 w(W)l(AIT)h Fx(or)g Fv(MPI)p 830 969 V 15 w(TEST)p Fx(.)g(Also,)h(if)f(an)g(error)g(o)q(ccurs)g(subsequen)o(tly)h(during) 189 1025 y(the)f(comm)o(unication,)i(an)f(error)f(co)q(de)i(cannot)e(b) q(e)i(returned)f(to)f(the)h(user)g(|)g(suc)o(h)g(an)g(error)189 1082 y(m)o(ust)d(b)q(e)i(treated)f(as)f(fatal.)26 b(Questions)18 b(arise)f(as)g(to)f(ho)o(w)h(one)g(kno)o(ws)g(when)h(the)f(op)q (erations)189 1138 y(ha)o(v)o(e)f(completed)i(when)g(using)f Fv(MPI)p 840 1138 V 16 w(REQUEST)p 1059 1138 V 18 w(FREE)p Fx(.)g(Dep)q(ending)i(on)e(the)g(program)e(logic,)189 1195 y(there)j(ma)o(y)g(b)q(e)g(other)g(w)o(a)o(ys)f(in)j(whic)o(h)f (the)f(program)f(kno)o(ws)h(that)f(certain)i(op)q(erations)f(ha)o(v)o (e)189 1251 y(completed)f(and)f(this)h(mak)o(es)e(usage)h(of)g Fv(MPI)p 986 1251 V 16 w(REQUEST)p 1205 1251 V 18 w(FREE)g Fx(practical.)24 b(F)l(or)15 b(example,)i(an)189 1308 y(activ)o(e)j(send)g(request)g(could)h(b)q(e)g(freed)g(when)f(the)g (logic)h(of)f(the)g(program)f(is)h(suc)o(h)h(that)e(the)189 1364 y(receiv)o(er)i(sends)h(a)f(reply)h(to)e(the)h(message)g(sen)o(t)g (|)g(the)g(arriv)m(al)h(of)f(the)g(reply)h(informs)f(the)189 1420 y(sender)f(that)f(the)g(send)h(has)g(completed)g(and)g(the)g(send) g(bu\013er)f(can)h(b)q(e)g(reused.)34 b(An)20 b(activ)o(e)189 1477 y(receiv)o(e)e(request)f(should)h(nev)o(er)f(b)q(e)g(freed)h(as)e (the)h(receiv)o(er)h(will)h(ha)o(v)o(e)d(no)h(w)o(a)o(y)f(to)g(v)o (erify)i(that)189 1533 y(the)h(receiv)o(e)i(has)e(completed)h(and)g (the)g(receiv)o(e)g(bu\013er)g(can)f(b)q(e)h(reused.)34 b(\()p Fl(End)19 b(of)i(advic)n(e)f(to)189 1590 y(users.)p Fx(\))75 1695 y Fq(Example)e(3.11)23 b Fx(An)15 b(example)h(using)g Fv(MPI)p 863 1695 V 16 w(REQUEST)p 1082 1695 V 18 w(FREE)p Fx(.)75 1800 y Fu(CALL)23 b(MPI_COMM_RANK\(MPI_COMM_WORLD)o(,)e(rank\)) 75 1857 y(IF\(rank.EQ.0\))123 1913 y(THEN)170 1970 y(DO)j(i=1,)f(n)218 2026 y(CALL)g(MPI_ISEND\(outval,)f(1,)i(MPI_REAL,)e(1,)i(0,)f(req,)h (ierr\))218 2083 y(CALL)f(MPI_REQUEST_FREE\(req,)e(ierr\))218 2139 y(CALL)i(MPI_IRECV\(inval,)f(1,)i(MPI_REAL,)e(1,)i(0,)g(req,)f (ierr\))218 2195 y(CALL)g(MPI_WAIT\(req,)g(status,)f(ierr\))170 2252 y(END)i(DO)123 2308 y(ELSE)95 b(!)23 b(rank.EQ.1)170 2365 y(CALL)h(MPI_IRECV\(inval,)d(1,)j(MPI_REAL,)f(0,)g(0,)h(req,)f (ierr\))170 2421 y(CALL)h(MPI_WAIT\(req,)e(status\))170 2478 y(DO)i(I=1,)f(n-1)242 2534 y(CALL)g(MPI_ISEND\(outval,)f(1,)h (MPI_REAL,)g(0,)h(0,)f(req,)h(ierr\))242 2591 y(CALL)f (MPI_REQUEST_FREE\(req,)e(ierr\))242 2647 y(CALL)i(MPI_IRECV\(inval,)f (1,)i(MPI_REAL,)e(0,)i(0,)f(req,)h(ierr\))242 2704 y(CALL)f (MPI_WAIT\(req,)f(status,)h(ierr\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 42 50 42 49 bop 75 -100 a Fx(42)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Fu(END)24 b(DO)170 102 y(CALL)g(MPI_ISEND\(outval,)d(1,)j(MPI_REAL,)f(0,)g(0,)h (req,)f(ierr\))170 158 y(CALL)h(MPI_WAIT\(req,)e(status\))75 214 y(END)h(IF)75 335 y Fj(3.7.4)49 b(Semantics)13 b(of)k(Nonblo)q (cking)h(Comm)o(unications)75 421 y Fx(The)11 b(seman)o(tics)f(of)h (non)o(blo)q(c)o(king)g(comm)o(unication)h(is)f(de\014ned)h(b)o(y)e (suitably)i(extending)g(the)e(de\014nitions)75 477 y(in)16 b(Section)g(3.5.)75 597 y Fv(Order)46 b Fx(Non)o(blo)q(c)o(king)12 b(comm)o(unication)f(op)q(erations)g(are)g(ordered)g(according)g(to)f (the)h(execution)h(order)75 653 y(of)h(the)g(calls)h(that)e(initiate)i (the)f(comm)o(unication.)20 b(The)13 b(non-o)o(v)o(ertaking)g (requiremen)o(t)g(of)g(Section)h(3.5)75 709 y(is)i(extended)g(to)e(non) o(blo)q(c)o(king)j(comm)o(unication,)f(with)f(this)h(de\014nition)h(of) d(order)h(b)q(eing)i(used.)75 808 y Fq(Example)h(3.12)23 b Fx(Message)14 b(ordering)i(for)e(non)o(blo)q(c)o(king)j(op)q (erations.)75 897 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\)) 75 953 y(IF)h(\(RANK.EQ.0\))147 1010 y(THEN)218 1066 y(CALL)f(MPI_ISEND\(a,)g(1,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(r1,)h (ierr\))218 1123 y(CALL)f(MPI_ISEND\(b,)g(1,)g(MPI_REAL,)g(1,)h(0,)f (comm,)g(r2,)h(ierr\))147 1179 y(ELSE)94 b(!)24 b(rank.EQ.1)218 1236 y(CALL)f(MPI_IRECV\(a,)g(1,)g(MPI_REAL,)g(0,)h(MPI_ANY_TAG,)e (comm,)h(r1,)g(ierr\))218 1292 y(CALL)g(MPI_IRECV\(b,)g(1,)g(MPI_REAL,) g(0,)h(0,)f(comm,)g(r2,)h(ierr\))75 1349 y(END)f(IF)75 1405 y(CALL)g(MPI_WAIT\(r1,status\))75 1462 y(CALL)g (MPI_WAIT\(r2,status\))75 1550 y Fx(The)11 b(\014rst)f(send)h(of)f(pro) q(cess)h(zero)f(will)j(matc)o(h)d(the)g(\014rst)g(receiv)o(e)i(of)e (pro)q(cess)h(one,)g(ev)o(en)g(if)g(b)q(oth)f(messages)75 1607 y(are)15 b(sen)o(t)g(b)q(efore)g(pro)q(cess)h(one)f(executes)h (either)g(receiv)o(e.)75 1726 y Fv(Progress)45 b Fx(A)12 b(call)h(to)f Fv(MPI)p 545 1726 14 2 v 15 w(W)l(AIT)g Fx(that)g(completes)g(a)g(receiv)o(e)h(will)h(ev)o(en)o(tually)f (terminate)f(and)g(return)75 1782 y(if)21 b(a)f(matc)o(hing)g(send)h (has)f(b)q(een)h(started,)g(unless)g(the)g(send)g(is)f(satis\014ed)h(b) o(y)f(another)g(receiv)o(e.)36 b(In)75 1839 y(particular,)20 b(if)f(the)g(matc)o(hing)g(send)g(is)g(non)o(blo)q(c)o(king,)i(then)e (the)g(receiv)o(e)g(should)h(complete)f(ev)o(en)g(if)75 1895 y(no)c(call)i(is)e(executed)i(b)o(y)e(the)g(sender)h(to)f (complete)h(the)f(send.)20 b(Similarly)l(,)e(a)d(call)h(to)f Fv(MPI)p 1651 1895 V 15 w(W)l(AIT)h Fx(that)75 1951 y(completes)i(a)e (send)i(will)h(ev)o(en)o(tually)f(return)f(if)g(a)g(matc)o(hing)g (receiv)o(e)h(has)f(b)q(een)h(started,)f(unless)h(the)75 2008 y(receiv)o(e)e(is)g(satis\014ed)f(b)o(y)g(another)g(send,)g(and)g (ev)o(en)h(if)f(no)g(call)h(is)g(executed)g(to)e(complete)i(the)f (receiv)o(e.)75 2107 y Fq(Example)j(3.13)23 b Fx(An)15 b(illustration)i(of)d(progress)h(seman)o(tics.)75 2195 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2252 y(IF)h(\(RANK.EQ.0\))147 2308 y(THEN)218 2365 y(CALL)f (MPI_SSEND\(a,)g(1,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 2421 y(CALL)g(MPI_SEND\(b,)g(1,)g(MPI_REAL,)g(1,)h(1,)f(comm,)g(ierr\)) 147 2478 y(ELSE)94 b(!)24 b(rank.EQ.1)218 2534 y(CALL)f(MPI_IRECV\(a,)g (1,)g(MPI_REAL,)g(0,)h(0,)f(comm,)g(r,)h(ierr\))218 2591 y(CALL)f(MPI_RECV\(b,)g(1,)g(MPI_REAL,)g(0,)h(1,)f(comm,)g(ierr\))218 2647 y(CALL)g(MPI_WAIT\(r,)g(status,)g(ierr\))75 2704 y(END)g(IF)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 43 51 43 50 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(43)166 45 y(This)14 b(co)q(de)f(should)h(not)f(deadlo)q(c)o(k)h (in)g(a)e(correct)h Fv(MPI)g Fx(implemen)o(tation.)20 b(The)13 b(\014rst)g(sync)o(hronous)75 102 y(send)22 b(of)e(pro)q(cess)h(zero)g(m)o(ust)f(complete)i(after)e(pro)q(cess)i (one)f(p)q(osts)f(the)h(matc)o(hing)g(\(non)o(blo)q(c)o(king\))75 158 y(receiv)o(e)15 b(ev)o(en)f(if)h(pro)q(cess)f(one)g(has)g(not)f(y)o (et)h(reac)o(hed)g(the)g(completing)h(w)o(ait)f(call.)20 b(Th)o(us,)14 b(pro)q(cess)g(zero)75 214 y(will)j(con)o(tin)o(ue)f(and) f(execute)h(the)f(second)h(send,)f(allo)o(wing)h(pro)q(cess)g(one)f(to) g(complete)h(execution.)166 321 y(If)d(an)f Fv(MPI)p 354 321 14 2 v 16 w(TEST)h Fx(that)f(completes)h(a)g(receiv)o(e)g(is)g (rep)q(eatedly)h(called)g(with)f(the)g(same)f(argumen)o(ts,)75 377 y(and)g(a)g(matc)o(hing)g(send)h(has)f(b)q(een)h(started,)f(then)g (the)g(call)i(will)f(ev)o(en)o(tually)g(return)f Fv(\015ag)g(=)h(true)p Fx(,)g(unless)75 434 y(the)g(send)g(is)g(satis\014ed)g(b)o(y)f(another) h(receiv)o(e.)19 b(If)13 b(an)g Fv(MPI)p 1046 434 V 16 w(TEST)f Fx(that)g(completes)h(a)g(send)g(is)g(rep)q(eatedly)75 490 y(called)i(with)f(the)g(same)f(argumen)o(ts,)g(and)h(a)f(matc)o (hing)g(receiv)o(e)i(has)e(b)q(een)i(started,)e(then)h(the)g(call)g (will)75 547 y(ev)o(en)o(tually)i(return)f Fv(\015ag)g(=)h(true)p Fx(,)f(unless)i(the)e(receiv)o(e)h(is)g(satis\014ed)g(b)o(y)f(another)f (send.)75 668 y Fj(3.7.5)49 b(Multiple)17 b(Completions)75 754 y Fx(It)h(is)h(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)g(to)f(w)o (ait)g(for)f(the)i(completion)g(of)f(an)o(y)l(,)h(some,)f(or)g(all)h (the)g(op)q(erations)75 811 y(in)h(a)f(list,)i(rather)d(than)i(ha)o (ving)f(to)g(w)o(ait)g(for)f(a)h(sp)q(eci\014c)i(message.)32 b(A)19 b(call)i(to)d Fv(MPI)p 1600 811 V 16 w(W)l(AIT)l(ANY)i Fx(or)75 867 y Fv(MPI)p 160 867 V 16 w(TEST)l(ANY)d Fx(can)f(b)q(e)g (used)g(to)f(w)o(ait)h(for)f(the)h(completion)h(of)e(one)h(out)f(of)g (sev)o(eral)h(op)q(erations.)22 b(A)75 923 y(call)d(to)d Fv(MPI)p 303 923 V 16 w(W)l(AIT)l(ALL)h Fx(or)g Fv(MPI)p 665 923 V 16 w(TEST)l(ALL)h Fx(can)f(b)q(e)h(used)g(to)f(w)o(ait)g(for) g(all)h(p)q(ending)i(op)q(erations)d(in)75 980 y(a)e(list.)23 b(A)16 b(call)h(to)e Fv(MPI)p 484 980 V 15 w(W)l(AITSOME)i Fx(or)e Fv(MPI)p 892 980 V 16 w(TESTSOME)h Fx(can)g(b)q(e)h(used)f(to)f (complete)i(all)f(enabled)75 1036 y(op)q(erations)f(in)h(a)f(list.)75 1187 y Fv(MPI)p 160 1187 V 16 w(W)l(AIT)l(ANY)h(\(count,)g(a)o(rra)o(y) p 640 1187 V 14 w(of)p 691 1187 V 16 w(requests,)h(index,)f(status\)) 117 1265 y Fk(IN)155 b Fv(count)482 b Fk(list)14 b(length)f(\(in)o (teger\))117 1340 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 1340 V 15 w(of)p 468 1340 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 1415 y(OUT)108 b Fv(index)489 b Fk(index)12 b(of)f(handle)g(for)g(op)q (eration)h(that)f(completed)g(\(in)o(teger\))117 1490 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1614 y Fu(int)23 b(MPI)p 245 1614 15 2 v 17 w(Waitany\(int)g(count,)g (MPI)p 788 1614 V 16 w(Request)g(*array)p 1139 1614 V 17 w(of)p 1204 1614 V 17 w(requests,)f(int)i(*index,)393 1671 y(MPI)p 468 1671 V 17 w(Status)f(*status\))75 1757 y(MPI)p 150 1757 V 17 w(WAITANY\(COUNT,)f(ARRAY)p 645 1757 V 16 w(OF)p 709 1757 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g (IERROR\))170 1814 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1814 V 17 w(OF)p 716 1814 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 1814 V 16 w(STATUS)p 1610 1814 V 16 w(SIZE\),)170 1870 y(IERROR)166 1957 y Fx(Blo)q(c)o(ks)16 b(un)o(til)g(one)f(of)g (the)h(op)q(erations)f(asso)q(ciated)g(with)h(the)f(activ)o(e)h (requests)f(in)h(the)f(arra)o(y)f(has)75 2013 y(completed.)36 b(If)21 b(more)f(then)h(one)f(op)q(eration)h(is)g(enabled)h(and)e(can)h (terminate,)g(one)f(is)h(arbitrarily)75 2070 y(c)o(hosen.)28 b(Returns)18 b(in)h Fv(index)g Fx(the)f(index)h(of)e(that)h(request)f (in)i(the)f(arra)o(y)f(and)h(returns)f(in)i Fv(status)h Fx(the)75 2126 y(status)c(of)h(the)f(completing)j(comm)o(unication.)25 b(\(The)17 b(arra)o(y)f(is)h(indexed)i(from)d(zero)h(in)g(C,)g(and)g (from)75 2182 y(one)g(in)i(F)l(ortran.\))24 b(If)18 b(the)f(request)g (w)o(as)g(allo)q(cated)h(b)o(y)f(a)g(non)o(blo)q(c)o(king)i(comm)o (unication)f(op)q(eration,)75 2239 y(then)e(it)f(is)h(deallo)q(cated)g (and)g(the)f(request)g(handle)i(is)e(set)g(to)g Fi(MPI)p 1229 2239 13 2 v 14 w(REQUEST)p 1431 2239 V 14 w(NULL)p Fx(.)166 2295 y(The)i Fv(a)o(rra)o(y)p 359 2295 14 2 v 15 w(of)p 411 2295 V 16 w(requests)i Fx(list)e(ma)o(y)g(con)o(tain)g (n)o(ull)h(or)f(inactiv)o(e)h(handles.)26 b(If)17 b(the)g(list)h(con)o (tains)f(no)75 2352 y(activ)o(e)e(handles)i(\(list)f(has)f(length)h (zero)f(or)g(all)h(en)o(tries)g(are)f(n)o(ull)i(or)e(inactiv)o(e\),)g (then)h(the)g(call)g(returns)75 2408 y(immediately)h(with)e Fv(index)i(=)e(MPI)p 687 2408 V 16 w(UNDEFINED)p Fx(.)166 2465 y(The)k(execution)h(of)e Fv(MPI)p 610 2465 V 16 w(W)l(AIT)l(ANY\(count,)j(a)o(rra)o(y)p 1079 2465 V 14 w(of)p 1130 2465 V 16 w(requests,)h(index,)e(status\))g Fx(has)f(the)f(same)75 2521 y(e\013ect)k(as)g(the)g(execution)i(of)44 b Fv(MPI)p 727 2521 V 16 w(W)l(AIT\(&a)o(rra)o(y)p 1003 2521 V 14 w(of)p 1054 2521 V 16 w(requests[i],)26 b(status\))p Fx(,)g(where)c Fv(i)g Fx(is)h(the)g(v)m(alue)75 2578 y(returned)17 b(b)o(y)f Fv(index)p Fx(.)25 b Fv(MPI)p 546 2578 V 16 w(W)l(AIT)l(ANY)18 b Fx(with)f(an)f(arra)o(y)f(con)o (taining)j(one)e(activ)o(e)h(en)o(try)f(is)h(equiv)m(alen)o(t)75 2634 y(to)e Fv(MPI)p 216 2634 V 15 w(W)l(AIT)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 44 52 44 51 bop 75 -100 a Fx(44)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(TEST)l(ANY\(count,)g(a)o(rra)o(y)p 626 45 V 14 w(of)p 677 45 V 16 w(requests,)g(index,)f(\015ag,)f (status\))117 122 y Fk(IN)155 b Fv(count)482 b Fk(list)14 b(length)f(\(in)o(teger\))117 197 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 197 V 15 w(of)p 468 197 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 273 y(OUT)108 b Fv(index)489 b Fk(index)9 b(of)g(op)q(eration)g(that)h (completed,)f(or)g Fi(MPI)p 1651 273 13 2 v 14 w(UNDEFINED)905 329 y Fk(if)k(none)h(completed)g(\(in)o(teger\))117 404 y(OUT)108 b Fv(\015ag)518 b Fi(true)15 b Fk(if)e(one)h(of)g(the)g(op)q (erations)g(is)g(complete)f(\(logical\))117 479 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 604 y Fu(int)23 b(MPI)p 245 604 15 2 v 17 w(Testany\(int)g(count,)g (MPI)p 788 604 V 16 w(Request)g(*array)p 1139 604 V 17 w(of)p 1204 604 V 17 w(requests,)f(int)i(*index,)393 660 y(int)g(*flag,)f(MPI)p 731 660 V 16 w(Status)g(*status\))75 747 y(MPI)p 150 747 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 747 V 16 w(OF)p 709 747 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))170 803 y(LOGICAL)g(FLAG)170 860 y(INTEGER)g(COUNT,)g(ARRAY)p 651 860 V 17 w(OF)p 716 860 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 860 V 16 w(STATUS)p 1610 860 V 16 w(SIZE\),)170 916 y(IERROR)166 1002 y Fx(T)l(ests)d(for)h(completion)h(of)e(either)i(one)f(or)f(none)h (of)g(the)g(op)q(erations)g(asso)q(ciated)g(with)g(activ)o(e)75 1059 y(handles.)g(In)14 b(the)g(former)f(case,)h(it)g(returns)g Fv(\015ag)g(=)g(true)p Fx(,)h(returns)e(in)i Fv(index)g Fx(the)f(index)h(of)f(this)g(request)75 1115 y(in)h(the)g(arra)o(y)l(,) f(and)g(returns)h(in)g Fv(status)i Fx(the)e(status)f(of)g(that)g(op)q (eration;)h(if)g(the)f(request)h(w)o(as)f(allo)q(cated)75 1172 y(b)o(y)i(a)f(non)o(blo)q(c)o(king)i(comm)o(unication)g(call)f (then)g(the)g(request)g(is)g(deallo)q(cated)h(and)f(the)g(handle)h(is)f (set)75 1228 y(to)e Fi(MPI)p 207 1228 13 2 v 14 w(REQUEST)p 409 1228 V 14 w(NULL)p Fx(.)h(\(The)f(arra)o(y)f(is)i(indexed)h(from)d (zero)i(in)g(C,)f(and)g(from)g(one)g(in)h(F)l(ortran.\))j(In)75 1285 y(the)13 b(latter)g(case,)h(it)f(returns)g Fv(\015ag)h(=)f(false)p Fx(,)h(returns)f(a)g(v)m(alue)i(of)d Fi(MPI)p 1254 1285 V 15 w(UNDEFINED)g Fx(in)i Fv(index)g Fx(and)g Fv(status)75 1341 y Fx(is)f(unde\014ned.)21 b(The)12 b(arra)o(y)f(ma)o(y)h(con)o (tain)h(n)o(ull)g(or)f(inactiv)o(e)i(handles.)20 b(If)12 b(the)h(arra)o(y)e(con)o(tains)i(no)f(activ)o(e)75 1398 y(handles)18 b(then)f(the)f(call)i(returns)e(immediately)i(with)f Fv(\015ag)g(=)f(false)p Fx(,)h Fv(index)g Fx(=)g Fi(MPI)p 1528 1398 V 15 w(UNDEFINED)p Fx(,)e(and)75 1454 y Fv(status)i Fx(unde\014ned.)166 1511 y(The)i(execution)g(of)f Fv(MPI)p 609 1511 14 2 v 16 w(TEST)l(ANY\(count,)j(a)o(rra)o(y)p 1079 1511 V 15 w(of)p 1131 1511 V 16 w(requests,)g(index,)f(status\))g Fx(has)f(the)f(same)75 1567 y(e\013ect)c(as)f(the)h(execution)h(of)f Fv(MPI)p 663 1567 V 16 w(TEST\(&a)o(rra)o(y)p 940 1567 V 14 w(of)p 991 1567 V 16 w(requests[i],)j(\015ag,)c(status\))p Fx(,)j(for)d Fv(i=0,)h(1)f(,...,)g(count-)75 1623 y(1)p Fx(,)21 b(in)h(some)e(arbitrary)h(order,)g(un)o(til)h(one)f(call)h (returns)e Fv(\015ag)h(=)g(true)p Fx(,)h(or)f(all)g(fail.)38 b(In)21 b(the)g(former)75 1680 y(case,)c Fv(index)g Fx(is)g(set)g(to)f (the)h(last)f(v)m(alue)i(of)e Fv(i)p Fx(,)h(and)f(in)i(the)f(latter)f (case,)h(it)f(is)i(set)e(to)g Fi(MPI)p 1618 1680 13 2 v 14 w(UNDEFINED)p Fx(.)75 1736 y Fv(MPI)p 160 1736 14 2 v 16 w(TEST)l(ANY)g Fx(with)g(an)f(arra)o(y)f(con)o(taining)i(one)f (activ)o(e)g(en)o(try)g(is)h(equiv)m(alen)o(t)h(to)d Fv(MPI)p 1633 1736 V 16 w(TEST)p Fx(.)75 1887 y Fv(MPI)p 160 1887 V 16 w(W)l(AIT)l(ALL\()g(count,)i(a)o(rra)o(y)p 626 1887 V 15 w(of)p 678 1887 V 16 w(requests,)h(a)o(rra)o(y)p 971 1887 V 15 w(of)p 1023 1887 V 15 w(statuses\))117 1965 y Fk(IN)155 b Fv(count)482 b Fk(lists)14 b(length)g(\(in)o (teger\))117 2040 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 2040 V 15 w(of)p 468 2040 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 2115 y(OUT)108 b Fv(a)o(rra)o(y)p 416 2115 V 15 w(of)p 468 2115 V 16 w(statuses)276 b Fk(arra)o(y)14 b(of)f(status)i(ob)r(jects)g (\(arra)o(y)f(of)f(Status\))75 2239 y Fu(int)23 b(MPI)p 245 2239 15 2 v 17 w(Waitall\(int)g(count,)g(MPI)p 788 2239 V 16 w(Request)g(*array)p 1139 2239 V 17 w(of)p 1204 2239 V 17 w(requests,)393 2296 y(MPI)p 468 2296 V 17 w(Status)g(*array)p 796 2296 V 16 w(of)p 860 2296 V 17 w(statuses\))75 2382 y(MPI)p 150 2382 V 17 w(WAITALL\(COUNT,)f (ARRAY)p 645 2382 V 16 w(OF)p 709 2382 V 17 w(REQUESTS,)h(ARRAY)p 1085 2382 V 16 w(OF)p 1149 2382 V 17 w(STATUSES,)g(IERROR\))170 2439 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2439 V 17 w(OF)p 716 2439 V 17 w(REQUESTS\(*\),)170 2495 y(ARRAY)p 293 2495 V 17 w(OF)p 358 2495 V 17 w(STATUSES\(MPI)p 663 2495 V 15 w(STATUS)p 822 2495 V 17 w(SIZE,*\),)g(IERROR)166 2582 y Fx(Blo)q(c)o(ks)d(un)o(til)h(all)f(comm)o(unication)g(op)q (erations)g(asso)q(ciated)g(with)g(activ)o(e)f(handles)i(in)g(the)e (list)75 2638 y(complete,)j(and)e(return)g(the)g(status)f(of)g(all)i (these)f(op)q(erations)g(\(this)h(includes)h(the)e(case)g(where)g(no)75 2694 y(handle)j(in)f(the)f(list)i(is)f(activ)o(e\).)38 b(Both)21 b(arra)o(ys)f(ha)o(v)o(e)h(the)h(same)f(n)o(um)o(b)q(er)h(of) f(v)m(alid)i(en)o(tries.)39 b(The)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 45 53 45 52 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(45)75 45 y Fv(i)p Fx(-th)20 b(en)o(try)g(in)h Fv(a)o(rra)o(y)p 444 45 14 2 v 14 w(of)p 495 45 V 16 w(statuses)i Fx(is)e(set)f(to)f (the)i(return)f(status)f(of)h(the)g Fv(i)p Fx(-th)g(op)q(eration.)35 b(Requests)75 102 y(that)13 b(w)o(ere)h(created)g(b)o(y)h(non)o(blo)q (c)o(king)g(comm)o(unication)g(op)q(erations)f(are)g(deallo)q(cated)i (and)e(the)g(corre-)75 158 y(sp)q(onding)j(handles)g(in)g(the)f(arra)o (y)e(are)i(set)g(to)f Fi(MPI)p 968 158 13 2 v 14 w(REQUEST)p 1170 158 V 14 w(NULL)p Fx(.)h(The)g(list)h(ma)o(y)e(con)o(tain)h(n)o (ull)h(or)75 214 y(inactiv)o(e)i(handles.)29 b(The)18 b(call)g(returns)g(in)h(the)f(status)e(of)i(eac)o(h)g(suc)o(h)g(en)o (try)f Fv(tag)h(=)g(MPI)p 1654 214 14 2 v 16 w(ANY)p 1762 214 V 17 w(T)l(A)o(G)p Fx(,)75 271 y Fv(source)24 b(=)g(MPI)p 362 271 V 16 w(ANY)p 470 271 V 17 w(SOURCE)p Fx(,)g(and)g(eac)o(h)g(status)e(en)o(try)h(is)h(also)g(con\014gured)g (so)f(that)g(calls)i(to)75 327 y Fv(MPI)p 160 327 V 16 w(GET)p 264 327 V 17 w(COUNT)16 b Fx(and)f Fv(MPI)p 623 327 V 16 w(GET)p 727 327 V 17 w(ELEMENTS)g Fx(return)g Fv(count)h(=)g(0)p Fx(.)166 384 y(The)c(execution)h(of)e Fv(MPI)p 589 384 V 16 w(W)l(AIT)l(ALL\(count,)i(a)o(rra)o(y)p 1038 384 V 14 w(of)p 1089 384 V 16 w(requests,)h(a)o(rra)o(y)p 1379 384 V 15 w(of)p 1431 384 V 16 w(statuses\))g Fx(has)d(the)h(same) 75 440 y(e\013ect)17 b(as)f(the)h(execution)h(of)f Fv(MPI)p 678 440 V 16 w(W)l(AIT\(&a)o(rra)o(y)p 954 440 V 14 w(of)p 1005 440 V 16 w(request[i],)i(&a)o(rra)o(y)p 1354 440 V 15 w(of)p 1406 440 V 16 w(statuses[i]\))p Fx(,)h(for)c Fv(i=0)h(,...,)75 497 y(count-1)p Fx(,)h(in)f(some)g(arbitrary)f (order.)24 b Fv(MPI)p 840 497 V 16 w(W)l(AIT)l(ALL)16 b Fx(with)h(an)g(arra)o(y)e(of)i(length)g(one)g(is)g(equiv)m(alen)o(t) 75 553 y(to)e Fv(MPI)p 216 553 V 15 w(W)l(AIT)p Fx(.)75 704 y Fv(MPI)p 160 704 V 16 w(TEST)l(ALL\(count,)h(a)o(rra)o(y)p 613 704 V 14 w(of)p 664 704 V 16 w(requests,)h(\015ag,)e(a)o(rra)o(y)p 1054 704 V 14 w(of)p 1105 704 V 16 w(statuses\))117 781 y Fk(IN)155 b Fv(count)482 b Fk(lists)14 b(length)g(\(in)o(teger\))117 856 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 856 V 15 w(of)p 468 856 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handles\))117 932 y(OUT)108 b Fv(\015ag)518 b Fk(\(logical\))117 1007 y(OUT)108 b Fv(a)o(rra)o(y)p 416 1007 V 15 w(of)p 468 1007 V 16 w(statuses)276 b Fk(arra)o(y)14 b(of)f(status)i(ob)r(jects)g(\(arra)o(y)f(of)f(Status\))75 1131 y Fu(int)23 b(MPI)p 245 1131 15 2 v 17 w(Testall\(int)g(count,)g (MPI)p 788 1131 V 16 w(Request)g(*array)p 1139 1131 V 17 w(of)p 1204 1131 V 17 w(requests,)f(int)i(*flag,)393 1188 y(MPI)p 468 1188 V 17 w(Status)f(*array)p 796 1188 V 16 w(of)p 860 1188 V 17 w(statuses\))75 1274 y(MPI)p 150 1274 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 1274 V 16 w(OF)p 709 1274 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 1274 V 16 w(OF)p 1292 1274 V 17 w(STATUSES,)g(IERROR\))170 1331 y(LOGICAL)g(FLAG)170 1387 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1387 V 17 w(OF)p 716 1387 V 17 w(REQUESTS\(*\),)170 1443 y(ARRAY)p 293 1443 V 17 w(OF)p 358 1443 V 17 w(STATUSES\(MPI)p 663 1443 V 15 w(STATUS)p 822 1443 V 17 w(SIZE,*\),)g(IERROR)166 1530 y Fx(Returns)17 b Fv(\015ag)g(=)g(true)g Fx(if)h(all)f(comm)o (unications)h(asso)q(ciated)f(with)g(activ)o(e)g(handles)h(in)f(the)g (arra)o(y)75 1586 y(ha)o(v)o(e)f(completed)i(\(this)e(includes)j(the)d (case)h(where)g(no)f(handle)i(in)f(the)g(list)g(is)g(activ)o(e\).)23 b(In)17 b(this)g(case,)75 1643 y(eac)o(h)h(status)f(en)o(try)g(that)g (corresp)q(onds)i(to)e(an)h(activ)o(e)f(handle)j(request)d(is)i(set)e (to)h(the)f(status)g(of)h(the)75 1699 y(corresp)q(onding)h(comm)o (unication;)g(if)g(the)e(request)h(w)o(as)f(allo)q(cated)i(b)o(y)f(a)f (non)o(blo)q(c)o(king)i(comm)o(unica-)75 1756 y(tion)f(call)h(then)f (it)g(is)h(deallo)q(cated,)g(and)f(the)g(handle)i(is)e(set)g(to)f Fi(MPI)p 1293 1756 13 2 v 14 w(REQUEST)p 1495 1756 V 14 w(NULL)p Fx(.)h(Eac)o(h)g(status)75 1812 y(en)o(try)f(that)g (corresp)q(onds)h(to)f(a)g(n)o(ull)i(or)e(inactiv)o(e)h(handle)h(is)f (set)f(to)g(return)h Fv(tag)f(=)h(MPI)p 1654 1812 14 2 v 16 w(ANY)p 1762 1812 V 17 w(T)l(A)o(G)p Fx(,)75 1869 y Fv(source)e(=)f(MPI)p 345 1869 V 16 w(ANY)p 453 1869 V 17 w(SOURCE)p Fx(,)i(and)e(is)h(also)f(con\014gured)h(so)f(that)f (calls)j(to)d Fv(MPI)p 1512 1869 V 16 w(GET)p 1616 1869 V 17 w(COUNT)i Fx(and)75 1925 y Fv(MPI)p 160 1925 V 16 w(GET)p 264 1925 V 17 w(ELEMENTS)f Fx(return)g Fv(count)i(=)e(0)p Fx(.)166 1982 y(Otherwise,)h Fv(\015ag)f(=)g(false)h Fx(is)f(returned,)h(no)f(request)g(is)h(mo)q(di\014ed)h(and)e(the)g(v)m (alues)i(of)d(the)i(status)75 2038 y(en)o(tries)g(are)e(unde\014ned.)22 b(This)16 b(is)g(a)f(lo)q(cal)h(op)q(eration.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 46 54 46 53 bop 75 -100 a Fx(46)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(W)l(AITSOME\(incount,)f(a)o(rra)o(y)p 695 45 V 15 w(of)p 747 45 V 16 w(requests,)h(outcount,)f(a)o(rra)o(y)p 1236 45 V 15 w(of)p 1288 45 V 16 w(indices,)g(a)o(rra)o(y)p 1553 45 V 14 w(of)p 1604 45 V 16 w(statuses\))117 122 y Fk(IN)155 b Fv(incount)448 b Fk(length)14 b(of)f(arra)o(y)p 1176 122 13 2 v 15 w(of)p 1225 122 V 14 w(requests)j(\(in)o(teger\))117 197 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 197 14 2 v 15 w(of)p 468 197 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handles\))117 273 y(OUT)108 b Fv(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(completed)f(requests)j(\(in)o(teger\)) 117 348 y(OUT)108 b Fv(a)o(rra)o(y)p 416 348 V 15 w(of)p 468 348 V 16 w(indices)299 b Fk(arra)o(y)12 b(of)f(indices)h(of)f(op)q (erations)g(that)h(completed)f(\(arra)o(y)h(of)905 404 y(in)o(tegers\))117 479 y(OUT)108 b Fv(a)o(rra)o(y)p 416 479 V 15 w(of)p 468 479 V 16 w(statuses)276 b Fk(arra)o(y)15 b(of)g(status)h(ob)r(jects)g(for)f(op)q(erations)g(that)h(completed)905 536 y(\(arra)o(y)e(of)f(Status\))75 660 y Fu(int)23 b(MPI)p 245 660 15 2 v 17 w(Waitsome\(int)f(incount,)h(MPI)p 859 660 V 17 w(Request)g(*array)p 1211 660 V 16 w(of)p 1275 660 V 17 w(requests,)g(int)g(*outcount,)393 717 y(int)h(*array)p 636 717 V 16 w(of)p 700 717 V 17 w(indices,)f(MPI)p 1004 717 V 16 w(Status)g(*array)p 1331 717 V 17 w(of)p 1396 717 V 17 w(statuses\))75 803 y(MPI)p 150 803 V 17 w(WAITSOME\(INCOUNT,)e(ARRAY)p 716 803 V 17 w(OF)p 781 803 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 803 V 16 w(OF)p 1459 803 V 17 w(INDICES,)393 860 y(ARRAY)p 516 860 V 17 w(OF)p 581 860 V 17 w(STATUSES,)f(IERROR\))170 916 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 916 V 17 w(OF)p 764 916 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 916 V 17 w(OF)p 1514 916 V 17 w(INDICES\(*\),)170 972 y(ARRAY)p 293 972 V 17 w(OF)p 358 972 V 17 w(STATUSES\(MPI)p 663 972 V 15 w(STATUS)p 822 972 V 17 w(SIZE,*\),)h(IERROR)166 1059 y Fx(W)l(aits)d(un)o(til)h(at)e(least)i(one)f(of)g(the)g(op)q (erations)g(asso)q(ciated)g(with)h(activ)o(e)f(handles)h(in)g(the)f (list)75 1115 y(ha)o(v)o(e)15 b(completed.)21 b(Returns)15 b(in)i Fv(outcount)g Fx(the)e(n)o(um)o(b)q(er)h(of)e(requests)h(from)g (the)g(list)h Fv(a)o(rra)o(y)p 1656 1115 14 2 v 15 w(of)p 1708 1115 V 16 w(requests)75 1172 y Fx(that)c(ha)o(v)o(e)h(completed.) 20 b(Returns)13 b(in)h(the)f(\014rst)g Fv(outcount)i Fx(lo)q(cations)f(of)e(the)h(arra)o(y)f Fv(a)o(rra)o(y)p 1608 1172 V 14 w(of)p 1659 1172 V 16 w(indices)j Fx(the)75 1228 y(indices)21 b(of)d(these)h(op)q(erations)f(\(index)i(within)g (the)e(arra)o(y)g Fv(a)o(rra)o(y)p 1218 1228 V 14 w(of)p 1269 1228 V 16 w(requests)p Fx(;)k(the)d(arra)o(y)e(is)i(indexed)75 1285 y(from)12 b(zero)g(in)i(C)e(and)h(from)e(one)i(in)g(F)l(ortran\).) 18 b(Returns)13 b(in)g(the)g(\014rst)f Fv(outcount)i Fx(lo)q(cations)g(of)e(the)g(arra)o(y)75 1341 y Fv(a)o(rra)o(y)p 173 1341 V 15 w(of)p 225 1341 V 16 w(status)18 b Fx(the)g(status)e(for) g(these)h(completed)h(op)q(erations.)26 b(If)17 b(a)g(request)g(that)f (completed)i(w)o(as)75 1398 y(allo)q(cated)g(b)o(y)g(a)f(non)o(blo)q(c) o(king)h(comm)o(unication)g(call,)h(then)f(it)f(is)h(deallo)q(cated,)h (and)e(the)h(asso)q(ciated)75 1454 y(handle)e(is)g(set)f(to)g Fi(MPI)p 471 1454 13 2 v 14 w(REQUEST)p 673 1454 V 14 w(NULL)p Fx(.)166 1511 y(If)h(the)g(list)g(con)o(tains)g(no)f(activ)o (e)h(handles,)h(then)f(the)f(call)i(returns)f(immediately)h(with)f Fv(outcount)75 1567 y(=)g(0)p Fx(.)75 1718 y Fv(MPI)p 160 1718 14 2 v 16 w(TESTSOME\(incount,)h(a)o(rra)o(y)p 697 1718 V 14 w(of)p 748 1718 V 16 w(requests,)g(outcount,)f(a)o(rra)o (y)p 1237 1718 V 15 w(of)p 1289 1718 V 16 w(indices,)g(a)o(rra)o(y)p 1554 1718 V 14 w(of)p 1605 1718 V 16 w(statuses\))117 1795 y Fk(IN)155 b Fv(incount)448 b Fk(length)14 b(of)f(arra)o(y)p 1176 1795 13 2 v 15 w(of)p 1225 1795 V 14 w(requests)j(\(in)o(teger\)) 117 1870 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 1870 14 2 v 15 w(of)p 468 1870 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 1945 y(OUT)108 b Fv(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(completed)f (requests)j(\(in)o(teger\))117 2021 y(OUT)108 b Fv(a)o(rra)o(y)p 416 2021 V 15 w(of)p 468 2021 V 16 w(indices)299 b Fk(arra)o(y)12 b(of)f(indices)h(of)f(op)q(erations)g(that)h(completed)f(\(arra)o(y)h (of)905 2077 y(in)o(tegers\))117 2152 y(OUT)108 b Fv(a)o(rra)o(y)p 416 2152 V 15 w(of)p 468 2152 V 16 w(statuses)276 b Fk(arra)o(y)15 b(of)g(status)h(ob)r(jects)g(for)f(op)q(erations)g(that)h(completed)905 2209 y(\(arra)o(y)e(of)f(Status\))75 2333 y Fu(int)23 b(MPI)p 245 2333 15 2 v 17 w(Testsome\(int)f(incount,)h(MPI)p 859 2333 V 17 w(Request)g(*array)p 1211 2333 V 16 w(of)p 1275 2333 V 17 w(requests,)g(int)g(*outcount,)393 2390 y(int)h(*array)p 636 2390 V 16 w(of)p 700 2390 V 17 w(indices,)f(MPI)p 1004 2390 V 16 w(Status)g(*array)p 1331 2390 V 17 w(of)p 1396 2390 V 17 w(statuses\))75 2476 y(MPI)p 150 2476 V 17 w(TESTSOME\(INCOUNT,)e(ARRAY)p 716 2476 V 17 w(OF)p 781 2476 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 2476 V 16 w(OF)p 1459 2476 V 17 w(INDICES,)393 2532 y(ARRAY)p 516 2532 V 17 w(OF)p 581 2532 V 17 w(STATUSES,)f(IERROR\))170 2589 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 2589 V 17 w(OF)p 764 2589 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 2589 V 17 w(OF)p 1514 2589 V 17 w(INDICES\(*\),)170 2645 y(ARRAY)p 293 2645 V 17 w(OF)p 358 2645 V 17 w(STATUSES\(MPI)p 663 2645 V 15 w(STATUS)p 822 2645 V 17 w(SIZE,*\),)h(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 47 55 47 54 bop 75 -100 a Fm(3.7.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b Fx(47)166 45 y(Beha)o(v)o(es)14 b(lik)o(e)i Fv(MPI)p 508 45 14 2 v 16 w(W)l(AITSOME)p Fx(,)e(except)h(that)f(it)h(returns)f (immediately)l(.)21 b(If)15 b(no)f(op)q(eration)h(has)75 102 y(completed)h(it)g(returns)f Fv(outcount)i(=)e(0)p Fx(.)166 158 y Fv(MPI)p 251 158 V 16 w(TESTSOME)c Fx(is)f(a)g(lo)q(cal) h(op)q(eration,)g(whic)o(h)g(returns)f(immediately)l(,)j(whereas)d Fv(MPI)p 1694 158 V 16 w(W)l(AITSOME)75 214 y Fx(will)15 b(blo)q(c)o(k)f(un)o(til)g(a)f(comm)o(unication)h(completes,)g(if)g(it) g(w)o(as)e(passed)i(a)f(list)h(that)e(con)o(tains)i(at)e(least)i(one)75 271 y(activ)o(e)i(handle.)24 b(Both)16 b(calls)h(ful\014l)h(a)e Fv(fairness)h Fx(requiremen)o(t:)22 b(If)16 b(a)g(request)g(for)g(a)g (receiv)o(e)h(rep)q(eatedly)75 327 y(app)q(ears)12 b(in)g(a)g(list)g (of)g(requests)f(passed)h(to)f Fv(MPI)p 895 327 V 16 w(W)l(AITSOME)h Fx(or)g Fv(MPI)p 1296 327 V 15 w(TESTSOME)p Fx(,)h(and)f(a)f(matc)o(hing)75 384 y(send)17 b(has)f(b)q(een)i(p)q (osted,)e(then)h(the)f(receiv)o(e)i(will)g(ev)o(en)o(tually)f(succeed,) h(unless)f(the)g(send)g(is)f(satis\014ed)75 440 y(b)o(y)f(another)g (receiv)o(e;)h(and)f(similarly)i(for)e(send)g(requests.)189 547 y Fl(A)n(dvic)n(e)c(to)i(users.)37 b Fx(The)12 b(use)f(of)g Fv(MPI)p 831 547 V 16 w(TESTSOME)h Fx(is)f(lik)o(ely)i(to)e(b)q(e)h (more)e(e\016cien)o(t)i(than)f(the)h(use)189 603 y(of)h Fv(MPI)p 324 603 V 16 w(TEST)l(ANY)p Fx(.)i(The)f(former)f(returns)g (information)h(on)g(all)h(completed)g(comm)o(unications,)189 659 y(with)g(the)g(latter,)g(a)g(new)g(call)i(is)e(required)i(for)d (eac)o(h)h(comm)o(unication)h(that)f(completes.)189 735 y(A)24 b(serv)o(er)g(with)g(m)o(ultiple)i(clien)o(ts)g(can)e(use)h Fv(MPI)p 1102 735 V 15 w(W)l(AITSOME)g Fx(so)f(as)g(not)g(to)f(starv)o (e)g(an)o(y)189 791 y(clien)o(t.)35 b(Clien)o(ts)20 b(send)h(messages)e (to)g(the)h(serv)o(er)g(with)g(service)h(requests.)33 b(The)20 b(serv)o(er)g(calls)189 848 y Fv(MPI)p 274 848 V 15 w(W)l(AITSOME)h Fx(with)g(one)f(receiv)o(e)h(request)f(for)g(eac)o (h)g(clien)o(t,)i(and)f(then)f(handles)i(all)f(re-)189 904 y(ceiv)o(es)g(that)e(completed.)35 b(If)20 b(a)g(call)h(to)f Fv(MPI)p 1002 904 V 16 w(W)l(AIT)l(ANY)h Fx(is)f(used)h(instead,)g (then)g(one)f(clien)o(t)189 960 y(could)d(starv)o(e)e(while)j(requests) e(from)g(another)g(clien)o(t)h(alw)o(a)o(ys)f(sneak)g(in)h(\014rst.)23 b(\()p Fl(End)17 b(of)g(advic)n(e)189 1017 y(to)f(users.)p Fx(\))189 1123 y Fl(A)n(dvic)n(e)h(to)h(implementors.)52 b Fv(MPI)p 810 1123 V 16 w(TESTSOME)18 b Fx(should)g(complete)g(as)f (man)o(y)g(p)q(ending)i(com-)189 1180 y(m)o(unications)d(as)e(p)q (ossible.)22 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 1286 y Fq(Example)h(3.14)23 b Fx(Clien)o(t-serv)o(er)15 b(co)q(de)h(\(starv)m(ation)f(can)g(o)q(ccur\).)75 1436 y Fu(CALL)23 b(MPI_COMM_SIZE\(comm,)f(size,)h(ierr\))75 1493 y(CALL)g(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1549 y(IF\(rank)g(>)h(0\))123 1606 y(THEN)214 b(!)24 b(client)f(code)170 1662 y(DO)h(WHILE\(.TRUE.\))242 1718 y(CALL)f(MPI_ISEND\(a,)g(n,)g(MPI_REAL,)g(0,)g(tag,)h(comm,)f(request,) g(ierr\))242 1775 y(CALL)g(MPI_WAIT\(request,)f(status,)h(ierr\))170 1831 y(END)h(DO)123 1888 y(ELSE)214 b(!)24 b(rank=0)f(--)g(server)g (code)242 1944 y(DO)h(i=1,)f(size-1)314 2001 y(CALL)g (MPI_IRECV\(a\(1,i\),)e(n,)j(MPI_REAL,)f(0,)g(tag,)528 2057 y(comm,)h(request_list\(i\),)d(ierr\))242 2114 y(END)i(DO)242 2170 y(DO)h(WHILE\(.TRUE.\))314 2227 y(CALL)f(MPI_WAITANY\(size-1,)e (request_list,)h(index,)h(status,)g(ierr\))314 2283 y(CALL)g (DO_SERVICE\(a\(1,index\)\))45 b(!)23 b(handle)g(one)h(message)314 2339 y(CALL)f(MPI_IRECV\(a\(1,)f(index\),)h(n,)g(MPI_REAL,)g(0,)h(tag,) 552 2396 y(comm,)f(request_list\(index\),)f(ierr\))242 2452 y(END)h(DO)75 2509 y(END)g(IF)75 2665 y Fq(Example)18 b(3.15)23 b Fx(Same)15 b(co)q(de,)g(using)h Fv(MPI)p 849 2665 V 16 w(W)l(AITSOME)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 48 56 48 55 bop 75 -100 a Fx(48)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 102 y Fu(CALL)23 b(MPI_COMM_SIZE\(comm,)f(size,)h(ierr\))75 158 y(CALL)g (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 214 y(IF\(rank)g(>)h(0\))123 271 y(THEN)214 b(!)24 b(client)f(code)170 327 y(DO)h(WHILE\(.TRUE.\)) 242 384 y(CALL)f(MPI_ISEND\(a,)g(n,)g(MPI_REAL,)g(0,)g(tag,)h(comm,)f (request,)g(ierr\))242 440 y(CALL)g(MPI_WAIT\(request,)f(status,)h (ierr\))170 497 y(END)h(DO)123 553 y(ELSE)214 b(!)24 b(rank=0)f(--)g(server)g(code)170 610 y(DO)h(i=1,)f(size-1)242 666 y(CALL)g(MPI_IRECV\(a\(1,i\),)f(n,)h(MPI_REAL,)g(0,)h(tag,)600 723 y(comm,)f(request_list\(i\),)f(ierr\))170 779 y(END)i(DO)170 835 y(DO)g(WHILE\(.TRUE.\))242 892 y(CALL)f(MPI_WAITSOME\(size,)f (request_list,)g(numdone,)648 948 y(index_list,)g(status_list,)g (ierr\))242 1005 y(DO)i(i=1,)f(numdone)314 1061 y(CALL)g (DO_SERVICE\(a\(1,)f(index_list\(i\)\)\))314 1118 y(CALL)h (MPI_IRECV\(a\(1,)f(index_list\(i\)\),)g(n,)h(MPI_REAL,)g(0,)h(tag,)624 1174 y(comm,)f(request_list\(i\),)f(ierr\))242 1231 y(END)h(DO)170 1287 y(END)h(DO)75 1344 y(END)f(IF)75 1487 y Fn(3.8)59 b(Prob)r(e)20 b(and)f(Cancel)75 1588 y Fx(The)d Fv(MPI)p 254 1588 14 2 v 16 w(PROBE)g Fx(and)g Fv(MPI)p 605 1588 V 16 w(IPROBE)g Fx(op)q(erations)g(allo)o(w)g(incoming)h(messages)e(to) g(b)q(e)h(c)o(hec)o(k)o(ed)g(for,)75 1645 y(without)g(actually)h (receiving)h(them.)k(The)17 b(user)f(can)g(then)h(decide)g(ho)o(w)f(to) g(receiv)o(e)h(them,)f(based)g(on)75 1701 y(the)i(information)f (returned)h(b)o(y)f(the)h(prob)q(e)g(\(basically)l(,)h(the)e (information)h(returned)g(b)o(y)f Fv(status)p Fx(\).)28 b(In)75 1758 y(particular,)16 b(the)g(user)g(ma)o(y)f(allo)q(cate)h (memory)f(for)g(the)h(receiv)o(e)h(bu\013er,)e(according)h(to)f(the)h (length)g(of)75 1814 y(the)f(prob)q(ed)h(message.)166 1871 y(The)i Fv(MPI)p 347 1871 V 16 w(CANCEL)g Fx(op)q(eration)g(allo)o (ws)g(p)q(ending)i(comm)o(unications)e(to)g(b)q(e)g(canceled.)30 b(This)18 b(is)75 1927 y(required)i(for)f(clean)o(up.)33 b(P)o(osting)19 b(a)g(send)h(or)f(a)g(receiv)o(e)h(ties)f(up)h(user)f (resources)h(\(send)f(or)g(receiv)o(e)75 1983 y(bu\013ers\),)14 b(and)i(a)f(cancel)h(ma)o(y)f(b)q(e)g(needed)i(to)d(free)i(these)f (resources)g(gracefully)l(.)75 2134 y Fv(MPI)p 160 2134 V 16 w(IPROBE\(source,)h(tag,)f(comm)m(,)c(\015ag,)k(status\))117 2212 y Fk(IN)155 b Fv(source)468 b Fk(source)15 b(rank,)f(or)27 b Fi(MPI)p 1281 2212 13 2 v 15 w(ANY)p 1381 2212 V 14 w(SOURCE)13 b Fk(\(in)o(teger\))117 2287 y(IN)155 b Fv(tag)526 b Fk(tag)14 b(v)n(alue)f(or)28 b Fi(MPI)p 1226 2287 V 14 w(ANY)p 1325 2287 V 15 w(T)m(A)o(G)12 b Fk(\(in)o(teger\))117 2362 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 2437 y(OUT)108 b Fv(\015ag)518 b Fk(\(logical\))117 2512 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 2637 y Fu(int)23 b(MPI)p 245 2637 15 2 v 17 w(Iprobe\(int)g(source,)g (int)g(tag,)g(MPI)p 1002 2637 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 2693 y(MPI)p 468 2693 V 17 w(Status)g(*status\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 49 57 49 56 bop 75 -100 a Fm(3.8.)29 b(PR)o(OBE)16 b(AND)f(CANCEL)1166 b Fx(49)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(IPROBE\(SOURCE,)22 b(TAG,)h(COMM,)g(FLAG,)g(STATUS,)g(IERROR\))170 102 y(LOGICAL)g(FLAG) 170 158 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 158 V 16 w(STATUS)p 1218 158 V 16 w(SIZE\),)g(IERROR)166 244 y Fv(MPI)p 251 244 14 2 v 16 w(IPROBE\(source,)18 b(tag,)f(comm)m(,)d(\015ag,)k(status\))g Fx(returns)f Fv(\015ag)g(=)h(true)f Fx(if)h(there)f(is)h(a)e(message)75 301 y(that)f(can)i(b)q(e)f(receiv)o(ed)h(and)g(that)e(matc)o(hes)h(the) g(pattern)f(sp)q(eci\014ed)k(b)o(y)d(the)g(argumen)o(ts)f Fv(source)p Fx(,)h Fv(tag)p Fx(,)75 357 y(and)f Fv(comm)n Fx(.)i(The)e(call)i(matc)o(hes)e(the)g(same)g(message)g(that)f(w)o (ould)i(ha)o(v)o(e)f(b)q(een)h(receiv)o(ed)h(b)o(y)e(a)g(call)i(to)75 414 y Fv(MPI)p 160 414 V 16 w(RECV\(...,)f(source,)i(tag,)f(comm)m(,)d (status\))k Fx(executed)g(at)e(the)h(same)g(p)q(oin)o(t)g(in)h(the)f (program,)f(and)75 470 y(returns)c(in)i Fv(status)g Fx(the)f(same)f(v)m (alue)i(that)e(w)o(ould)g(ha)o(v)o(e)h(b)q(een)g(returned)g(b)o(y)g Fv(MPI)p 1467 470 V 15 w(RECV\(\))p Fx(.)19 b(Otherwise,)75 527 y(the)c(call)i(returns)e Fv(\015ag)g(=)g(false)p Fx(,)g(and)h(lea)o(v)o(es)f Fv(status)i Fx(unde\014ned.)166 583 y(If)g Fv(MPI)p 298 583 V 15 w(IPROBE)h Fx(returns)e Fv(\015ag)h(=)f(true)p Fx(,)h(then)g(the)g(con)o(ten)o(t)f(of)g(the)g (status)g(ob)s(ject)g(can)g(b)q(e)h(sub-)75 640 y(sequen)o(tly)i (accessed)h(as)e(describ)q(ed)i(in)g(section)f(3.2.5)e(to)h(\014nd)h (the)g(source,)g(tag)f(and)g(length)i(of)e(the)75 696 y(prob)q(ed)e(message.)166 753 y(A)d(subsequen)o(t)i(receiv)o(e)f (executed)g(with)g(the)g(same)f(con)o(text,)g(and)g(the)h(source)f(and) h(tag)f(returned)75 809 y(in)19 b Fv(status)i Fx(b)o(y)e Fv(MPI)p 415 809 V 15 w(IPROBE)h Fx(will)g(receiv)o(e)g(the)f(message)f (that)g(w)o(as)g(matc)o(hed)g(b)o(y)h(the)f(prob)q(e,)i(if)f(no)75 865 y(other)c(in)o(terv)o(ening)h(receiv)o(e)g(o)q(ccurs)f(after)g(the) g(prob)q(e.)20 b(If)15 b(the)g(receiving)i(pro)q(cess)e(is)h(m)o (ulti-threaded,)75 922 y(it)f(is)h(the)f(user's)g(resp)q(onsibilit)o(y) j(to)d(ensure)g(that)g(the)g(last)g(condition)i(holds.)166 978 y(The)f Fv(source)h Fx(argumen)o(t)e(of)h Fv(MPI)p 738 978 V 16 w(PROBE)h Fx(can)f(b)q(e)h Fi(MPI)p 1140 978 13 2 v 14 w(ANY)p 1239 978 V 15 w(SOURCE)p Fx(,)d(and)i(the)h Fv(tag)f Fx(argumen)o(t)75 1035 y(can)g(b)q(e)g Fi(MPI)p 298 1035 V 15 w(ANY)p 398 1035 V 14 w(T)m(A)o(G)p Fx(,)f(so)g(that)g (one)h(can)g(prob)q(e)g(for)f(messages)h(from)f(an)h(arbitrary)f (source)h(and/or)75 1091 y(with)h(an)g(arbitrary)f(tag.)24 b(Ho)o(w)o(ev)o(er,)16 b(a)h(sp)q(eci\014c)h(comm)o(unication)g(con)o (text)e(m)o(ust)g(b)q(e)i(pro)o(vided)g(with)75 1148 y(the)d Fv(comm)10 b Fx(argumen)o(t.)166 1204 y(It)16 b(is)g(not)f(necessary)h(to)f(receiv)o(e)h(a)f(message)h(immediately)h (after)e(it)h(has)f(b)q(een)i(prob)q(ed)f(for,)f(and)75 1261 y(the)g(same)g(message)g(ma)o(y)f(b)q(e)i(prob)q(ed)g(for)f(sev)o (eral)g(times)h(un)o(til)g(it)f(is)h(receiv)o(ed.)75 1412 y Fv(MPI)p 160 1412 14 2 v 16 w(PROBE\(source,)g(tag,)f(comm)m(,)d (status\))117 1489 y Fk(IN)155 b Fv(source)468 b Fk(source)15 b(rank,)f(or)27 b Fi(MPI)p 1281 1489 13 2 v 15 w(ANY)p 1381 1489 V 14 w(SOURCE)13 b Fk(\(in)o(teger\))117 1564 y(IN)155 b Fv(tag)526 b Fk(tag)14 b(v)n(alue,)e(or)28 b Fi(MPI)p 1237 1564 V 14 w(ANY)p 1336 1564 V 15 w(T)m(A)o(G)13 b Fk(\(in)o(teger\))117 1639 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1714 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1839 y Fu(int)23 b(MPI)p 245 1839 15 2 v 17 w(Probe\(int)g(source,)g (int)g(tag,)h(MPI)p 979 1839 V 16 w(Comm)g(comm,)f(MPI)p 1330 1839 V 17 w(Status)g(*status\))75 1925 y(MPI)p 150 1925 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g(IERROR\))170 1982 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1982 V 16 w(STATUS)p 1218 1982 V 16 w(SIZE\),)g(IERROR)166 2068 y Fv(MPI)p 251 2068 14 2 v 16 w(PROBE)d Fx(b)q(eha)o(v)o(es)f(lik) o(e)h Fv(MPI)p 777 2068 V 16 w(IPROBE)g Fx(except)f(that)f(it)i(is)f(a) g(blo)q(c)o(king)h(call)g(that)e(returns)75 2124 y(only)e(after)e(a)h (matc)o(hing)g(message)g(has)g(b)q(een)h(found.)166 2181 y(The)11 b Fv(MPI)f Fx(implemen)o(tation)i(of)e Fv(MPI)p 799 2181 V 16 w(PROBE)i Fx(and)e Fv(MPI)p 1140 2181 V 16 w(IPROBE)i Fx(needs)f(to)f(guaran)o(tee)g(progress:)75 2237 y(if)17 b(a)g(call)h(to)e Fv(MPI)p 385 2237 V 16 w(PROBE)i Fx(has)f(b)q(een)h(issued)g(b)o(y)f(a)f(pro)q(cess,)i(and)f (a)f(send)i(that)e(matc)o(hes)h(the)f(prob)q(e)75 2294 y(has)j(b)q(een)i(initiated)g(b)o(y)f(some)f(pro)q(cess,)i(then)f(the)f (call)i(to)e Fv(MPI)p 1244 2294 V 16 w(PROBE)i Fx(will)g(return,)f (unless)h(the)75 2350 y(message)c(is)i(receiv)o(ed)g(b)o(y)f(another)f (concurren)o(t)h(receiv)o(e)h(op)q(eration)f(\(that)e(is)j(executed)g (b)o(y)e(another)75 2407 y(thread)e(at)g(the)h(probing)g(pro)q(cess\).) 21 b(Similarly)l(,)d(if)e(a)f(pro)q(cess)h(busy)g(w)o(aits)f(with)h Fv(MPI)p 1573 2407 V 16 w(IPROBE)g Fx(and)g(a)75 2463 y(matc)o(hing)e(message)g(has)g(b)q(een)h(issued,)g(then)f(the)g(call)h (to)f Fv(MPI)p 1179 2463 V 16 w(IPROBE)g Fx(will)i(ev)o(en)o(tually)f (return)f Fv(\015ag)75 2520 y(=)i(true)f Fx(unless)i(the)e(message)g (is)g(receiv)o(ed)i(b)o(y)e(another)g(concurren)o(t)g(receiv)o(e)h(op)q (eration.)75 2626 y Fq(Example)i(3.16)23 b Fx(Use)15 b(blo)q(c)o(king)h(prob)q(e)g(to)f(w)o(ait)f(for)h(an)g(incoming)i (message.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 50 58 50 57 bop 75 -100 a Fx(50)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 102 y(IF)h(\(rank.EQ.0\))123 158 y(THEN)194 214 y(CALL)g(MPI_SEND\(i,)e(1,)i(MPI_INTEGER,)e(2,)h(0,) h(comm,)f(ierr\))123 271 y(ELSE)g(IF\(rank.EQ.1\))f(THEN)194 327 y(CALL)i(MPI_SEND\(x,)e(1,)i(MPI_REAL,)e(2,)i(0,)f(comm,)h(ierr\)) 123 384 y(ELSE)71 b(!)23 b(rank.EQ.2)170 440 y(DO)h(i=1,)f(2)242 497 y(CALL)g(MPI_PROBE\(MPI_ANY_SOURCE,)e(0,)624 553 y(comm,)i(status,)g(ierr\))242 610 y(IF)h(\(status\(MPI_SOURCE\))d(=)j (0\))314 666 y(THEN)75 723 y(100)214 b(CALL)24 b(MPI_RECV\(i,)e(1,)i (MPI_INTEGER,)e(0,)h(0,)h(status,)f(ierr\))314 779 y(ELSE)75 835 y(200)214 b(CALL)24 b(MPI_RECV\(x,)e(1,)i(MPI_REAL,)e(1,)i(0,)f (status,)g(ierr\))242 892 y(END)g(IF)170 948 y(END)h(DO)75 1005 y(END)f(IF)75 1111 y Fx(Eac)o(h)15 b(message)g(is)g(receiv)o(ed)i (with)e(the)h(righ)o(t)f(t)o(yp)q(e.)75 1217 y Fq(Example)j(3.17)23 b Fx(A)15 b(similar)h(program)e(to)h(the)g(previous)h(example,)g(but)f (no)o(w)g(it)g(has)g(a)g(problem.)75 1311 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1368 y(IF)h(\(rank.EQ.0\)) 123 1424 y(THEN)194 1481 y(CALL)g(MPI_SEND\(i,)e(1,)i(MPI_INTEGER,)e (2,)h(0,)h(comm,)f(ierr\))123 1537 y(ELSE)g(IF\(rank.EQ.1\))f(THEN)194 1593 y(CALL)i(MPI_SEND\(x,)e(1,)i(MPI_REAL,)e(2,)i(0,)f(comm,)h(ierr\)) 123 1650 y(ELSE)170 1706 y(DO)g(i=1,)f(2)242 1763 y(CALL)g (MPI_PROBE\(MPI_ANY_SOURCE,)e(0,)624 1819 y(comm,)i(status,)g(ierr\)) 242 1876 y(IF)h(\(status\(MPI_SOURCE\))d(=)j(0\))314 1932 y(THEN)75 1989 y(100)214 b(CALL)24 b(MPI_RECV\(i,)e(1,)i (MPI_INTEGER,)e(MPI_ANY_SOURCE,)695 2045 y(0,)i(status,)f(ierr\))314 2102 y(ELSE)75 2158 y(200)214 b(CALL)24 b(MPI_RECV\(x,)e(1,)i (MPI_REAL,)e(MPI_ANY_SOURCE,)695 2214 y(0,)i(status,)f(ierr\))242 2271 y(END)g(IF)170 2327 y(END)h(DO)75 2384 y(END)f(IF)166 2478 y Fx(W)l(e)16 b(sligh)o(tly)h(mo)q(di\014ed)g(example)f(3.16,)e (using)j Fi(MPI)p 1075 2478 13 2 v 14 w(ANY)p 1174 2478 V 15 w(SOURCE)d Fx(as)h(the)h Fv(source)g Fx(argumen)o(t)f(in)75 2534 y(the)f(t)o(w)o(o)g(receiv)o(e)h(calls)g(in)h(statemen)o(ts)d(lab) q(eled)j(100)e(and)g(200.)19 b(The)c(program)e(is)i(no)o(w)f (incorrect:)20 b(the)75 2591 y(receiv)o(e)f(op)q(eration)f(ma)o(y)f (receiv)o(e)i(a)f(message)f(that)h(is)g(distinct)h(from)f(the)g (message)f(prob)q(ed)i(b)o(y)f(the)75 2647 y(preceding)f(call)f(to)f Fv(MPI)p 504 2647 14 2 v 15 w(PROBE)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 51 59 51 58 bop 75 -100 a Fm(3.8.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b Fx(51)189 45 y Fl(A)n(dvic)n(e)13 b(to)i(implementors.)38 b Fx(A)13 b(call)i(to)d Fv(MPI)p 971 45 14 2 v 16 w(PROBE\(source,)j (tag,)e(comm)m(,)d(status\))15 b Fx(will)g(matc)o(h)189 102 y(the)k(message)g(that)g(w)o(ould)h(ha)o(v)o(e)f(b)q(een)i(receiv)o (ed)f(b)o(y)g(a)f(call)h(to)f Fv(MPI)p 1431 102 V 16 w(RECV\(...,)h(source,)g(tag,)189 158 y(comm)m(,)9 b(status\))k Fx(executed)g(at)e(the)h(same)g(p)q(oin)o(t.)19 b(Supp)q(ose)13 b(that)e(this)h(message)g(has)f(source)h Fv(s)p Fx(,)h(tag)189 214 y Fv(t)i Fx(and)h(comm)o(unicator)f Fv(c)p Fx(.)20 b(If)c(the)f(tag)g(argumen)o(t)f(in)j(the)e(prob)q(e)h(call)g(has)f(v)m (alue)i Fi(MPI)p 1684 214 13 2 v 14 w(ANY)p 1783 214 V 15 w(T)m(A)o(G)189 271 y Fx(then)c(the)g(message)f(prob)q(ed)i(will)g (b)q(e)g(the)f(earliest)g(p)q(ending)i(message)e(from)f(source)h Fv(s)g Fx(with)g(com-)189 327 y(m)o(unicator)g Fv(c)h Fx(and)f(an)o(y)g(tag;)g(in)i(an)o(y)e(case,)g(the)h(message)f(prob)q (ed)h(will)h(b)q(e)f(the)g(earliest)g(p)q(ending)189 384 y(message)f(from)g(source)h Fv(s)g Fx(with)g(tag)e Fv(t)j Fx(and)e(comm)o(unicator)h Fv(c)g Fx(\(this)f(is)i(the)e (message)h(that)f(w)o(ould)189 440 y(ha)o(v)o(e)j(b)q(een)h(receiv)o (ed,)g(so)f(as)g(to)g(preserv)o(e)g(message)g(order\).)22 b(This)17 b(message)f(con)o(tin)o(ues)h(as)f(the)189 497 y(earliest)g(p)q(ending)h(message)e(from)g(source)g Fv(s)h Fx(with)g(tag)f Fv(t)g Fx(and)h(comm)o(unicator)f Fv(c)p Fx(,)h(un)o(til)g(it)g(is)g(re-)189 553 y(ceiv)o(ed.)k(A)12 b(receiv)o(e)h(op)q(eration)g(subsequen)o(t)g(to)e(the)i(prob)q(e)g (that)e(uses)i(the)f(same)g(comm)o(unicator)189 610 y(as)j(the)h(prob)q (e)g(and)g(uses)g(the)f(tag)g(and)h(source)g(v)m(alues)h(returned)f(b)o (y)f(the)h(prob)q(e,)g(m)o(ust)f(receiv)o(e)189 666 y(this)h(message,)f (unless)i(it)f(has)g(already)g(b)q(een)h(receiv)o(ed)g(b)o(y)f(another) g(receiv)o(e)h(op)q(eration.)22 b(\()p Fl(End)189 723 y(of)16 b(advic)n(e)g(to)h(implementors.)p Fx(\))75 923 y Fv(MPI)p 160 923 14 2 v 16 w(CANCEL\(request\))117 1001 y Fk(IN)155 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1125 y Fu(int)23 b(MPI)p 245 1125 15 2 v 17 w(Cancel\(MPI)p 502 1125 V 16 w(Request)g(*request\))75 1211 y(MPI)p 150 1211 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1268 y(INTEGER)h(REQUEST,)g(IERROR)166 1354 y Fx(A)15 b(call)i(to)e Fv(MPI)p 440 1354 14 2 v 16 w(CANCEL)g Fx(marks)g(for)g(cancellation)i(a)e(p)q(ending,)i(non)o(blo)q(c)o(king) g(comm)o(unication)75 1411 y(op)q(eration)d(\(send)g(or)f(receiv)o (e\).)20 b(The)13 b(cancel)i(call)g(is)f(lo)q(cal.)20 b(It)14 b(returns)f(immediately)l(,)j(p)q(ossibly)f(b)q(efore)75 1467 y(the)h(comm)o(unication)h(is)g(actually)g(canceled.)25 b(It)16 b(is)h(still)g(necessary)g(to)e(complete)i(a)f(comm)o (unication)75 1524 y(that)g(has)g(b)q(een)i(mark)o(ed)e(for)f (cancellation,)k(using)e(a)f(call)h(to)f Fv(MPI)p 1249 1524 V 16 w(W)l(AIT)g Fx(or)g Fv(MPI)p 1533 1524 V 16 w(TEST)h Fx(\(or)e(an)o(y)h(of)75 1580 y(the)f(deriv)o(ed)i(op)q (erations\).)166 1637 y(If)i(a)g(comm)o(unication)h(is)g(mark)o(ed)f (for)g(cancellation,)j(then)d(a)g Fv(MPI)p 1370 1637 V 16 w(W)l(AIT)g Fx(call)i(for)d(that)h(com-)75 1693 y(m)o(unication)i(is)f(guaran)o(teed)f(to)h(return,)g(irresp)q(ectiv)o (e)h(of)f(the)g(activities)h(of)e(other)g(pro)q(cesses)i(\(i.e.,)75 1750 y Fv(MPI)p 160 1750 V 16 w(W)l(AIT)f Fx(b)q(eha)o(v)o(es)g(as)g(a) g(lo)q(cal)i(function\);)g(similarly)g(if)f Fv(MPI)p 1235 1750 V 16 w(TEST)f Fx(is)h(rep)q(eatedly)g(called)h(in)f(a)75 1806 y(busy)13 b(w)o(ait)g(lo)q(op)h(for)e(a)h(canceled)h(comm)o (unication,)g(then)g Fv(MPI)p 1173 1806 V 15 w(TEST)g Fx(will)g(ev)o(en)o(tually)g(b)q(e)g(successful.)166 1862 y Fv(MPI)p 251 1862 V 16 w(CANCEL)e Fx(can)f(b)q(e)i(used)f(to)f (cancel)i(a)e(comm)o(unication)h(that)f(uses)h(a)f(p)q(ersisten)o(t)h (request)g(\(see)75 1919 y(Sec.)26 b(3.9\),)15 b(in)j(the)f(same)f(w)o (a)o(y)g(it)h(is)h(used)f(for)g(nonp)q(ersisten)o(t)g(requests.)25 b(A)17 b(successful)h(cancellation)75 1975 y(cancels)c(the)f(activ)o(e) g(comm)o(unication,)h(but)f(not)f(the)h(request)g(itself.)20 b(After)13 b(the)g(call)h(to)e Fv(MPI)p 1690 1975 V 16 w(CANCEL)75 2032 y Fx(and)j(the)g(subsequen)o(t)h(call)g(to)e Fv(MPI)p 697 2032 V 16 w(W)l(AIT)h Fx(or)f Fv(MPI)p 978 2032 V 16 w(TEST)p Fx(,)h(the)g(request)g(b)q(ecomes)h(inactiv)o(e)g (and)f(can)75 2088 y(b)q(e)h(activ)m(ated)g(for)e(a)h(new)g(comm)o (unication.)166 2145 y(The)j(successful)i(cancellation)g(of)e(a)g (bu\013ered)h(send)g(frees)f(the)g(bu\013er)h(space)f(o)q(ccupied)j(b)o (y)d(the)75 2201 y(p)q(ending)f(message.)166 2258 y(Either)i(the)f (cancellation)i(succeeds,)f(or)f(the)g(comm)o(unication)h(succeeds,)h (but)e(not)g(b)q(oth.)29 b(If)18 b(a)75 2314 y(send)h(is)h(mark)o(ed)e (for)g(cancellation,)j(then)e(it)g(m)o(ust)f(b)q(e)i(the)f(case)g(that) f(either)h(the)g(send)g(completes)75 2371 y(normally)l(,)f(in)h(whic)o (h)f(case)f(the)h(message)f(sen)o(t)g(w)o(as)g(receiv)o(ed)h(at)f(the)h (destination)g(pro)q(cess,)g(or)f(that)75 2427 y(the)f(send)g(is)g (successfully)i(canceled,)f(in)g(whic)o(h)f(case)g(no)g(part)f(of)g (the)h(message)f(w)o(as)g(receiv)o(ed)i(at)e(the)75 2483 y(destination.)21 b(Then,)15 b(an)o(y)f(matc)o(hing)h(receiv)o(e)h(has) e(to)h(b)q(e)g(satis\014ed)h(b)o(y)e(another)h(send.)20 b(If)15 b(a)g(receiv)o(e)g(is)75 2540 y(mark)o(ed)d(for)f (cancellation,)k(then)d(it)h(m)o(ust)e(b)q(e)i(the)g(case)f(that)f (either)i(the)g(receiv)o(e)g(completes)g(normally)l(,)75 2596 y(or)18 b(that)g(the)g(receiv)o(e)i(is)f(successfully)h(canceled,) h(in)e(whic)o(h)h(case)e(no)h(part)f(of)g(the)g(receiv)o(e)i(bu\013er)e (is)75 2653 y(altered.)i(Then,)c(an)o(y)e(matc)o(hing)i(send)g(has)f (to)f(b)q(e)i(satis\014ed)g(b)o(y)f(another)g(receiv)o(e.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 52 60 52 59 bop 75 -100 a Fx(52)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fx(If)f(the)f(op)q (eration)h(has)g(b)q(een)g(canceled,)h(then)f(information)g(to)f(that)f (e\013ect)i(will)h(b)q(e)f(returned)g(in)75 102 y(the)f(status)g (argumen)o(t)f(of)h(the)g(op)q(eration)g(that)g(completes)h(the)f(comm) o(unication.)75 252 y Fv(MPI)p 160 252 14 2 v 16 w(TEST)p 290 252 V 16 w(CANCELLED\(status,)i(\015ag\))117 330 y Fk(IN)155 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))117 405 y(OUT)108 b Fv(\015ag)518 b Fk(\(logical\))75 529 y Fu(int)23 b(MPI)p 245 529 15 2 v 17 w(Test)p 358 529 V 17 w(cancelled\(MPI)p 687 529 V 15 w(Status)g(status,)g(int)h (*flag\))75 616 y(MPI)p 150 616 V 17 w(TEST)p 263 616 V 16 w(CANCELLED\(STATUS,)e(FLAG,)h(IERROR\))170 672 y(LOGICAL)g(FLAG)170 729 y(INTEGER)g(STATUS\(MPI)p 604 729 V 16 w(STATUS)p 764 729 V 17 w(SIZE\),)g(IERROR)166 815 y Fx(Returns)12 b Fv(\015ag)f(=)h(true)g Fx(if)f(the)h(comm)o (unication)g(asso)q(ciated)f(with)h(the)g(status)e(ob)s(ject)h(w)o(as)f (canceled)75 872 y(successfully)l(.)27 b(In)18 b(suc)o(h)f(a)g(case,)g (all)g(other)g(\014elds)h(of)e Fv(status)j Fx(\(suc)o(h)e(as)g Fv(count)h Fx(or)e Fv(tag)p Fx(\))h(are)f(unde\014ned.)75 928 y(Returns)c Fv(\015ag)g(=)g(false)p Fx(,)g(otherwise.)19 b(If)12 b(a)g(receiv)o(e)g(op)q(eration)g(migh)o(t)g(b)q(e)h(canceled)g (then)f(one)g(should)h(call)75 984 y Fv(MPI)p 160 984 14 2 v 16 w(TEST)p 290 984 V 16 w(CANCELLED)g Fx(\014rst,)f(to)g(c)o (hec)o(k)g(whether)h(the)f(op)q(eration)h(w)o(as)e(canceled,)j(b)q (efore)f(c)o(hec)o(king)75 1041 y(on)i(the)g(other)g(\014elds)i(of)d (the)i(return)f(status.)189 1147 y Fl(A)n(dvic)n(e)k(to)h(users.)64 b Fx(Cancel)20 b(can)g(b)q(e)f(an)h(exp)q(ensiv)o(e)g(op)q(eration)g (that)e(should)j(b)q(e)f(used)f(only)189 1204 y(exceptionally)l(.)j(\() p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))189 1310 y Fl(A)n(dvic)n(e)e(to)i(implementors.)39 b Fx(If)15 b(a)f(send)h(op)q(eration)g(uses)f(an)g(\\eager")g(proto)q(col)g (\(data)g(is)g(trans-)189 1366 y(ferred)e(to)f(the)h(receiv)o(er)h(b)q (efore)f(a)g(matc)o(hing)f(receiv)o(e)i(is)g(p)q(osted\),)f(then)g(the) g(cancellation)i(of)e(this)189 1423 y(send)i(ma)o(y)g(require)h(comm)o (unication)g(with)f(the)g(in)o(tended)i(receiv)o(er)f(in)f(order)g(to)g (free)g(allo)q(cated)189 1479 y(bu\013ers.)k(On)13 b(some)f(systems)g (this)h(ma)o(y)f(require)h(an)f(in)o(terrupt)h(to)e(the)i(in)o(tended)h (receiv)o(er.)19 b(Note)189 1536 y(that,)14 b(while)j(comm)o(unication) g(ma)o(y)e(b)q(e)h(needed)h(to)e(implemen)o(t)i Fv(MPI)p 1415 1536 V 16 w(CANCEL)p Fx(,)e(this)h(is)g(still)h(a)189 1592 y(lo)q(cal)g(op)q(eration,)f(since)i(its)e(completion)h(do)q(es)g (not)f(dep)q(end)i(on)e(the)g(co)q(de)h(executed)g(b)o(y)f(other)189 1649 y(pro)q(cesses.)21 b(If)15 b(pro)q(cessing)i(is)f(required)g(on)f (another)g(pro)q(cess,)h(this)g(should)g(b)q(e)g(transparen)o(t)f(to) 189 1705 y(the)h(application)j(\(hence)e(the)g(need)h(for)e(an)h(in)o (terrupt)g(and)g(an)f(in)o(terrupt)h(handler\).)26 b(\()p Fl(End)17 b(of)189 1762 y(advic)n(e)f(to)g(implementors.)p Fx(\))75 1905 y Fn(3.9)59 b(P)n(ersistent)19 b(communication)f (requests)75 2006 y Fx(Often)f(a)f(comm)o(unication)g(with)h(the)f (same)g(argumen)o(t)g(list)h(is)g(rep)q(eatedly)g(executed)g(within)h (the)e(in-)75 2063 y(ner)k(lo)q(op)h(of)f(a)g(parallel)i(computation.) 35 b(In)20 b(suc)o(h)h(a)f(situation,)h(it)g(ma)o(y)e(b)q(e)i(p)q (ossible)h(to)e(optimize)75 2119 y(the)c(comm)o(unication)g(b)o(y)g (binding)h(the)f(list)g(of)f(comm)o(unication)i(argumen)o(ts)d(to)h(a)h Fq(p)q(ersisten)o(t)f Fx(com-)75 2176 y(m)o(unication)k(request)e(once) h(and,)g(then,)g(rep)q(eatedly)h(using)f(the)g(request)g(to)e(initiate) j(and)f(complete)75 2232 y(messages.)g(The)12 b(p)q(ersisten)o(t)g (request)g(th)o(us)f(created)h(can)g(b)q(e)g(though)o(t)f(of)g(as)g(a)h (comm)o(unication)g(p)q(ort)f(or)75 2289 y(a)k(\\half-c)o(hannel.")21 b(It)15 b(do)q(es)h(not)e(pro)o(vide)i(the)f(full)i(functionalit)o(y)f (of)f(a)f(con)o(v)o(en)o(tional)i(c)o(hannel,)g(since)75 2345 y(there)i(is)h(no)f(binding)i(of)e(the)g(send)h(p)q(ort)f(to)f (the)i(receiv)o(e)g(p)q(ort.)28 b(This)19 b(construct)f(allo)o(ws)g (reduction)75 2401 y(of)c(the)g(o)o(v)o(erhead)g(for)g(comm)o (unication)h(b)q(et)o(w)o(een)g(the)f(pro)q(cess)h(and)f(comm)o (unication)h(con)o(troller,)g(but)75 2458 y(not)e(of)g(the)g(o)o(v)o (erhead)g(for)g(comm)o(unication)h(b)q(et)o(w)o(een)g(one)f(comm)o (unication)h(con)o(troller)g(and)g(another.)75 2514 y(It)19 b(is)g(not)f(necessary)g(that)g(messages)g(send)h(with)g(a)f(p)q (ersisten)o(t)h(request)g(b)q(e)g(receiv)o(ed)g(b)o(y)g(a)f(receiv)o(e) 75 2571 y(op)q(eration)d(using)h(a)f(p)q(ersisten)o(t)h(request,)f(or)f (vice)j(v)o(ersa.)166 2627 y(A)j(p)q(ersisten)o(t)g(comm)o(unication)g (request)g(is)g(created)g(using)h(one)e(of)h(the)f(four)h(follo)o(wing) g(calls.)75 2684 y(These)c(calls)g(in)o(v)o(olv)o(e)g(no)f(comm)o (unication.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 53 61 53 60 bop 75 -100 a Fm(3.9.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)661 b Fx(53)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SEND)p 293 45 V 17 w(INIT\(buf,)14 b(count,)i(datat)o(yp)q(e,)h (dest,)f(tag,)f(comm)m(,)d(request\))117 122 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Send)p 358 698 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1091 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(SEND)p 263 840 V 16 w(INIT\(BUF,)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 897 y()g (BUF\(*\))170 953 y(INTEGER)g(REQUEST,)g(COUNT,)g(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 1040 y Fx(Creates)15 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g(request)f(for)g(a)f (standard)h(mo)q(de)h(send)f(op)q(eration,)h(and)75 1096 y(binds)f(to)f(it)g(all)i(the)e(argumen)o(ts)f(of)h(a)g(send)h(op)q (eration.)75 1247 y Fv(MPI)p 160 1247 14 2 v 16 w(BSEND)p 323 1247 V 17 w(INIT\(buf,)f(count,)h(datat)o(yp)q(e,)g(dest,)g(tag,)f (comm)m(,)d(request\))117 1324 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1400 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 1475 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 1550 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1625 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1700 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1775 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1900 y Fu(int)23 b(MPI)p 245 1900 15 2 v 17 w(Bsend)p 382 1900 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 1900 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 1956 y(int)g(tag,)f(MPI)p 683 1956 V 17 w(Comm)g(comm,)g(MPI)p 1034 1956 V 17 w(Request)g(*request\))75 2043 y(MPI)p 150 2043 V 17 w(BSEND)p 287 2043 V 16 w(INIT\(BUF,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2099 y()g(BUF\(*\))170 2156 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 2242 y Fx(Creates)14 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g (request)f(for)g(a)g(bu\013ered)g(mo)q(de)h(send.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 54 62 54 61 bop 75 -100 a Fx(54)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SSEND)p 318 45 V 17 w(INIT\(buf,)e(count,)h(datat)o (yp)q(e,)g(dest,)g(tag,)f(comm)m(,)d(request\))117 122 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Ssend)p 382 698 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(SSEND)p 287 840 V 16 w(INIT\(BUF,)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 897 y()g (BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h (COMM,)f(REQUEST,)f(IERROR)166 1040 y Fx(Creates)14 b(a)h(p)q(ersisten) o(t)h(comm)o(unication)g(ob)s(ject)f(for)f(a)h(sync)o(hronous)g(mo)q (de)h(send)f(op)q(eration.)75 1191 y Fv(MPI)p 160 1191 14 2 v 16 w(RSEND)p 322 1191 V 17 w(INIT\(buf,)g(count,)h(datat)o(yp)q (e,)g(dest,)g(tag,)f(comm)m(,)d(request\))117 1268 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1343 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 1418 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 1493 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1569 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1644 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Fu(int)23 b(MPI)p 245 1843 15 2 v 17 w(Rsend)p 382 1843 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 1843 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 1900 y(int)g(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g(MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(RSEND)p 287 1986 V 16 w(INIT\(BUF,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)166 2186 y Fx(Creates)14 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g(ob)s(ject)f (for)f(a)h(ready)g(mo)q(de)h(send)f(op)q(eration.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 55 63 55 62 bop 75 -100 a Fm(3.9.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)661 b Fx(55)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(RECV)p 291 45 V 17 w(INIT\(buf,)15 b(count,)h(datat)o(yp)q(e,)g (source,)g(tag,)e(comm)m(,)e(request\))117 122 y Fk(OUT)108 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(receiv)o(ed)j(\(in)o(teger\))117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(source)j(or)d(MPI)p 1309 348 13 2 v 16 w(ANY)p 1418 348 V 15 w(SOUR)o(CE)h(\(in)o(teger\)) 117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(or)h(MPI)p 1270 423 V 15 w(ANY)p 1378 423 V 16 w(T)m(A)o(G)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Recv)p 358 698 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1091 698 V 17 w(Datatype)g(datatype,)f(int)i(source,) 393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(RECV)p 263 840 V 16 w(INIT\(BUF,)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 897 y()g(BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 1040 y Fx(Creates)13 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)f(request)h(for)e(a)h(receiv) o(e)h(op)q(eration.)20 b(The)14 b(argumen)o(t)f Fv(buf)75 1096 y Fx(is)f(mark)o(ed)g(as)f Fv(OUT)h Fx(b)q(ecause)h(the)f(user)g (giv)o(es)g(p)q(ermission)h(to)e(write)h(on)g(the)f(receiv)o(e)i (bu\013er)f(b)o(y)g(passing)75 1153 y(the)j(argumen)o(t)g Fv(MPI)p 442 1153 14 2 v 15 w(RECV)p 572 1153 V 18 w(INIT)p Fx(.)166 1209 y(A)h(p)q(ersisten)o(t)h(comm)o(unication)f(request)h(is) f(inactiv)o(e)h(after)f(it)g(w)o(as)f(created)h(|)h(no)f(activ)o(e)g (com-)75 1266 y(m)o(unication)g(is)g(attac)o(hed)e(to)h(the)g(request.) 166 1322 y(A)k(comm)o(unication)g(\(send)g(or)g(receiv)o(e\))g(that)f (uses)h(a)f(p)q(ersisten)o(t)i(request)e(is)i(initiated)g(b)o(y)f(the) 75 1379 y(function)d Fv(MPI)p 338 1379 V 16 w(ST)l(ART)p Fx(.)75 1530 y Fv(MPI)p 160 1530 V 16 w(ST)l(ART\(request\))117 1607 y Fk(INOUT)62 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1731 y Fu(int)23 b(MPI)p 245 1731 15 2 v 17 w(Start\(MPI)p 478 1731 V 16 w(Request)g(*request\))75 1818 y(MPI)p 150 1818 V 17 w(START\(REQUEST,)f(IERROR\))170 1874 y(INTEGER)h(REQUEST,)g(IERROR)166 1961 y Fv(request)18 b Fx(is)f(a)f(handle)h(returned)g(b)o(y)f(one)h(of)f(the)g(previous)h (four)f(calls.)25 b(The)16 b(asso)q(ciated)h(request)75 2017 y(should)f(b)q(e)g(inactiv)o(e.)21 b(The)15 b(request)h(b)q (ecomes)f(activ)o(e)h(once)f(the)h(call)g(is)g(made.)166 2073 y(If)e(the)g(request)g(is)h(for)e(a)h(send)h(with)f(ready)g(mo)q (de,)g(then)g(a)g(matc)o(hing)g(receiv)o(e)h(should)g(b)q(e)g(p)q (osted)75 2130 y(b)q(efore)j(the)g(call)h(is)g(made.)28 b(The)18 b(comm)o(unication)h(bu\013er)e(should)i(not)f(b)q(e)g (accessed)h(after)e(the)h(call,)75 2186 y(and)d(un)o(til)i(the)e(op)q (eration)g(completes.)166 2243 y(The)22 b(call)i(is)e(lo)q(cal,)j(with) e(similar)g(seman)o(tics)f(to)g(the)g(non)o(blo)q(c)o(king)i(comm)o (unication)f(op)q(era-)75 2299 y(tions)d(describ)q(ed)i(in)e(section)h (3.7.)33 b(That)19 b(is,)i(a)f(call)h(to)e Fv(MPI)p 1174 2299 14 2 v 16 w(ST)l(ART)i Fx(with)f(a)f(request)h(created)g(b)o(y)75 2356 y Fv(MPI)p 160 2356 V 16 w(SEND)p 293 2356 V 17 w(INIT)13 b Fx(starts)f(a)h(comm)o(unication)i(in)g(the)e(same)h (manner)g(as)f(a)g(call)i(to)e Fv(MPI)p 1601 2356 V 16 w(ISEND)p Fx(;)g(a)h(call)75 2412 y(to)h Fv(MPI)p 216 2412 V 16 w(ST)l(ART)h Fx(with)g(a)f(request)g(created)h(b)o(y)f Fv(MPI)p 995 2412 V 16 w(BSEND)p 1158 2412 V 17 w(INIT)g Fx(starts)f(a)h(comm)o(unication)i(in)f(the)75 2469 y(same)f(manner)g (as)g(a)g(call)h(to)f Fv(MPI)p 671 2469 V 16 w(IBSEND)p Fx(;)g(and)g(so)g(on.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 56 64 56 63 bop 75 -100 a Fx(56)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ST)l(ART)l(ALL\(count,)f(a)o(rra)o(y)p 641 45 V 15 w(of)p 693 45 V 16 w(requests\))117 122 y Fk(IN)155 b Fv(count)482 b Fk(list)14 b(length)f(\(in)o(teger\))117 197 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 197 V 15 w(of)p 468 197 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handle\))75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Startall\(int)f(count,)h(MPI)p 811 322 V 17 w(Request)g(*array)p 1163 322 V 17 w(of)p 1228 322 V 16 w(requests\))75 408 y(MPI)p 150 408 V 17 w(STARTALL\(COUNT,)f (ARRAY)p 669 408 V 16 w(OF)p 733 408 V 17 w(REQUESTS,)h(IERROR\))170 465 y(INTEGER)g(COUNT,)g(ARRAY)p 651 465 V 17 w(OF)p 716 465 V 17 w(REQUESTS\(*\),)f(IERROR)166 551 y Fx(Start)i(all)j(comm) o(unications)f(asso)q(ciated)g(with)f(requests)h(in)g Fv(a)o(rra)o(y)p 1399 551 14 2 v 15 w(of)p 1451 551 V 16 w(requests)p Fx(.)52 b(A)26 b(call)h(to)75 608 y Fv(MPI)p 160 608 V 16 w(ST)l(ART)l(ALL\(count,)76 b(a)o(rra)o(y)p 701 608 V 14 w(of)p 752 608 V 16 w(requests\))65 b Fx(has)d(the)h(same) f(e\013ect)h(as)f(calls)i(to)75 664 y Fv(MPI)p 160 664 V 16 w(ST)l(ART\(&a)o(rra)o(y)p 465 664 V 15 w(of)p 517 664 V 16 w(requests[i]\))p Fx(,)17 b(executed)f(for)f Fv(i=0)g(,...,)e(count-1)p Fx(,)j(in)g(some)f(arbitrary)g(order.)166 721 y(A)i(comm)o(unication)g(started)f(with)h(a)g(call)h(to)e Fv(MPI)p 1063 721 V 16 w(ST)l(ART)h Fx(or)f Fv(MPI)p 1377 721 V 16 w(ST)l(ART)l(ALL)i Fx(is)f(completed)75 777 y(b)o(y)11 b(a)h(call)g(to)f Fv(MPI)p 385 777 V 16 w(W)l(AIT)p Fx(,)g Fv(MPI)p 620 777 V 16 w(TEST)p Fx(,)h(or)f(one)g(of) g(the)h(deriv)o(ed)h(functions)f(describ)q(ed)h(in)g(section)f(3.7.5.) 75 834 y(The)18 b(request)f(b)q(ecomes)h(inactiv)o(e)h(after)d (successful)j(completion)g(of)e(suc)o(h)h(call.)27 b(The)18 b(request)f(is)h(not)75 890 y(deallo)q(cated)f(and)e(it)g(can)h(b)q(e)g (activ)m(ated)f(anew)h(b)o(y)f(an)g Fv(MPI)p 1116 890 V 16 w(ST)l(ART)h Fx(or)e Fv(MPI)p 1427 890 V 16 w(ST)l(ART)l(ALL)i Fx(call.)166 946 y(A)f(p)q(ersisten)o(t)h(request)f(is)h(deallo)q (cated)g(b)o(y)g(a)e(call)j(to)d Fv(MPI)p 1190 946 V 16 w(REQUEST)p 1409 946 V 18 w(FREE)i Fx(\(Section)f(3.7.3\).)166 1003 y(The)g(call)g(to)f Fv(MPI)p 481 1003 V 16 w(REQUEST)p 700 1003 V 17 w(FREE)h Fx(can)g(o)q(ccur)g(at)e(an)o(y)h(p)q(oin)o(t)h (in)h(the)e(program)f(after)h(the)g(p)q(er-)75 1059 y(sisten)o(t)h (request)g(w)o(as)f(created.)20 b(Ho)o(w)o(ev)o(er,)14 b(the)h(request)g(will)h(b)q(e)g(deallo)q(cated)g(only)g(after)e(it)h (b)q(ecomes)75 1116 y(inactiv)o(e.)33 b(Activ)o(e)20 b(receiv)o(e)g(requests)f(should)i(not)e(b)q(e)h(freed.)32 b(Otherwise,)21 b(it)e(will)i(not)e(b)q(e)h(p)q(ossible)75 1172 y(to)c(c)o(hec)o(k)g(that)f(the)i(receiv)o(e)g(has)f(completed.)23 b(It)17 b(is)f(preferable,)h(in)g(general,)g(to)e(free)h(requests)g (when)75 1229 y(they)h(are)g(inactiv)o(e.)27 b(If)17 b(this)g(rule)h(is)g(follo)o(w)o(ed,)g(then)f(the)g(functions)h (describ)q(ed)h(in)f(this)g(section)f(will)75 1285 y(b)q(e)f(in)o(v)o (ok)o(ed)f(in)h(a)f(sequence)i(of)d(the)i(form)166 1401 y Fq(Create)h Fx(\()p Fq(Start)h(Complete)p Fx(\))725 1384 y Fh(\003)762 1401 y Fq(F)l(ree)e Fp(;)166 1516 y Fx(where)g Ft(\003)g Fx(indicates)i(zero)e(or)f(more)h(rep)q (etitions.)24 b(If)16 b(the)g(same)g(comm)o(unication)h(ob)s(ject)f(is) g(used)75 1573 y(in)i(sev)o(eral)f(concurren)o(t)f(threads,)h(it)g(is)g (the)g(user's)g(resp)q(onsibilit)o(y)i(to)d(co)q(ordinate)i(calls)f(so) g(that)f(the)75 1629 y(correct)f(sequence)h(is)g(ob)q(ey)o(ed.)166 1686 y(A)c(send)g(op)q(eration)f(initiated)j(with)d Fv(MPI)p 874 1686 V 16 w(ST)l(ART)i Fx(can)e(b)q(e)i(matc)o(hed)e(with)h(an)o(y) f(receiv)o(e)i(op)q(eration)75 1742 y(and,)g(lik)o(ewise,)h(a)e(receiv) o(e)i(op)q(eration)e(initiated)i(with)f Fv(MPI)p 1098 1742 V 16 w(ST)l(ART)g Fx(can)g(receiv)o(e)g(messages)f(generated)75 1798 y(b)o(y)j(an)o(y)g(send)h(op)q(eration.)75 1942 y Fn(3.10)59 b(Send-receive)75 2043 y Fx(The)18 b Fq(send-receiv)o(e)g Fx(op)q(erations)g(com)o(bine)h(in)g(one)f(call)h(the)f(sending)i(of)d (a)h(message)g(to)f(one)h(desti-)75 2100 y(nation)g(and)f(the)h (receiving)h(of)e(another)g(message,)h(from)e(another)h(pro)q(cess.)28 b(The)17 b(t)o(w)o(o)g(\(source)g(and)75 2156 y(destination\))i(are)f (p)q(ossibly)j(the)d(same.)30 b(A)19 b(send-receiv)o(e)h(op)q(eration)f (is)g(v)o(ery)f(useful)i(for)e(executing)75 2213 y(a)g(shift)i(op)q (eration)f(across)f(a)g(c)o(hain)i(of)e(pro)q(cesses.)31 b(If)19 b(blo)q(c)o(king)h(sends)f(and)g(receiv)o(es)h(are)e(used)i (for)75 2269 y(suc)o(h)f(a)f(shift,)h(then)g(one)f(needs)h(to)f(order)g (the)g(sends)h(and)g(receiv)o(es)g(correctly)g(\(for)e(example,)j(ev)o (en)75 2325 y(pro)q(cesses)d(send,)f(then)h(receiv)o(e,)g(o)q(dd)f(pro) q(cesses)h(receiv)o(e)g(\014rst,)f(then)g(send\))h(so)f(as)g(to)f(prev) o(en)o(t)h(cyclic)75 2382 y(dep)q(endencies)j(that)c(ma)o(y)g(lead)h (to)f(deadlo)q(c)o(k.)23 b(When)16 b(a)f(send-receiv)o(e)j(op)q (eration)e(is)g(used,)g(the)g(com-)75 2438 y(m)o(unication)j(subsystem) e(tak)o(es)g(care)h(of)f(these)h(issues.)29 b(The)18 b(send-receiv)o(e)h(op)q(eration)f(can)g(b)q(e)g(used)75 2495 y(in)e(conjunction)f(with)h(the)e(functions)i(describ)q(ed)h(in)e (Chapter)g(6)f(in)i(order)e(to)h(p)q(erform)f(shifts)h(on)g(v)m(ar-)75 2551 y(ious)20 b(logical)h(top)q(ologies.)33 b(Also,)21 b(a)e(send-receiv)o(e)j(op)q(eration)d(is)h(useful)h(for)e(implemen)o (ting)j(remote)75 2608 y(pro)q(cedure)16 b(calls.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 57 65 57 64 bop 75 -100 a Fm(3.10.)34 b(SEND-RECEIVE)1277 b Fx(57)166 45 y(A)16 b(message)f(sen)o(t)g(b)o(y)h(a)f(send-receiv)o(e)j (op)q(eration)d(can)h(b)q(e)h(receiv)o(ed)f(b)o(y)g(a)f(regular)h (receiv)o(e)h(op)q(er-)75 102 y(ation)f(or)f(prob)q(ed)i(b)o(y)f(a)g (prob)q(e)g(op)q(eration;)g(a)g(send-receiv)o(e)i(op)q(eration)e(can)g (receiv)o(e)h(a)f(message)f(sen)o(t)75 158 y(b)o(y)g(a)g(regular)g (send)h(op)q(eration.)75 309 y Fv(MPI)p 160 309 14 2 v 16 w(SENDRECV\(sendbuf,)24 b(sendcount,)f(sendt)o(yp)q(e,)g(dest,)f (sendtag,)g(recvbuf,)g(recvcount,)f(recvt)o(yp)q(e,)75 365 y(source,)15 b(recvtag,)g(comm)m(,)d(status\))117 443 y Fk(IN)155 b Fv(sendbuf)444 b Fk(initial)12 b(address)j(of)f(send) g(bu\013er)h(\(c)o(hoice\))117 518 y(IN)155 b Fv(sendcount)399 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 593 y(IN)155 b Fv(sendt)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(bu\013er)g (\(handle\))117 668 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 743 y(IN)155 b Fv(sendtag)443 b Fk(send)15 b(tag)e(\(in)o(teger\))117 818 y(OUT)108 b Fv(recvbuf)450 b Fk(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 893 y(IN)155 b Fv(recvcount)405 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 969 y(IN)155 b Fv(recvt)o(yp)q(e)430 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(receiv)o(e)i(bu\013er)g (\(handle\))117 1044 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o(teger\))117 1119 y(IN)155 b Fv(recvtag)449 b Fk(receiv)o(e)15 b(tag)f(\(in)o(teger\))117 1194 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1269 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\)) 75 1394 y Fu(int)23 b(MPI)p 245 1394 15 2 v 17 w(Sendrecv\(void)f (*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1394 V 17 w(Datatype)f(sendtype,)393 1450 y(int)i(dest,)f(int)g(sendtag,)g(void)g (*recvbuf,)g(int)g(recvcount,)393 1506 y(MPI)p 468 1506 V 17 w(Datatype)g(recvtype,)f(int)i(source,)f(MPI)p 1297 1506 V 16 w(Datatype)g(recvtag,)393 1563 y(MPI)p 468 1563 V 17 w(Comm)g(comm,)g(MPI)p 819 1563 V 17 w(Status)g(*status\))75 1649 y(MPI)p 150 1649 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVBUF,)393 1706 y(RECVCOUNT,)h (RECVTYPE,)f(SOURCE,)h(RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 1762 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1819 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h (RECVTYPE,)170 1875 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 1875 V 16 w(STATUS)p 1122 1875 V 16 w(SIZE\),)h(IERROR)166 1962 y Fx(Execute)19 b(a)e(blo)q(c)o(king)j(send)f(and)f(receiv)o(e)h (op)q(eration.)29 b(Both)18 b(send)h(and)f(receiv)o(e)h(use)g(the)f (same)75 2018 y(comm)o(unicator,)i(but)g(p)q(ossibly)i(di\013eren)o(t)e (tags.)33 b(The)20 b(send)g(bu\013er)g(and)g(receiv)o(e)h(bu\013ers)f (m)o(ust)f(b)q(e)75 2075 y(disjoin)o(t,)c(and)h(ma)o(y)e(ha)o(v)o(e)h (di\013eren)o(t)h(lengths)f(and)h(datat)o(yp)q(es.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 58 66 58 65 bop 75 -100 a Fx(58)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SENDRECV)p 408 45 V 17 w(REPLA)o(CE\(buf,)i(count,)f (datat)o(yp)q(e,)h(dest,)f(sendtag,)h(source,)f(recvtag,)f(comm)m(,)d (sta-)75 102 y(tus\))117 179 y Fk(INOUT)62 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(and)g(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 254 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)9 b(of)g(elemen)o(ts)h(in)f(send)i(and)e(receiv)o(e)i(bu\013er)g(\(in)o (teger\))117 329 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(and)f(receiv)o(e)h(bu\013er)g (\(handle\))117 404 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 479 y(IN)155 b Fv(sendtag)443 b Fk(send)15 b(message)f(tag)f(\(in)o(teger\))117 554 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o (teger\))117 630 y(IN)155 b Fv(recvtag)449 b Fk(receiv)o(e)15 b(message)f(tag)f(\(in)o(teger\))117 705 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 780 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 904 y Fu(int)23 b(MPI)p 245 904 15 2 v 17 w(Sendrecv)p 454 904 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 904 V 17 w(Datatype)g(datatype,)393 961 y(int)h(dest,)f(int)g (sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 961 V 16 w(Comm)h(comm,)393 1017 y(MPI)p 468 1017 V 17 w(Status)f(*status\))75 1104 y(MPI)p 150 1104 V 17 w(SENDRECV)p 359 1104 V 16 w(REPLACE\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g (SOURCE,)g(RECVTAG,)393 1160 y(COMM,)g(STATUS,)g(IERROR\))170 1217 y()g(BUF\(*\))170 1273 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)g(COMM,)170 1329 y(STATUS\(MPI)p 413 1329 V 16 w(STATUS)p 573 1329 V 17 w(SIZE\),)g(IERROR)166 1416 y Fx(Execute)18 b(a)f(blo)q(c)o(king)i (send)f(and)g(receiv)o(e.)27 b(The)18 b(same)f(bu\013er)h(is)g(used)g (b)q(oth)f(for)g(the)h(send)g(and)75 1472 y(for)d(the)g(receiv)o(e,)h (so)e(that)h(the)g(message)g(sen)o(t)g(is)h(replaced)g(b)o(y)f(the)g (message)g(receiv)o(ed.)166 1529 y(The)f(seman)o(tics)g(of)f(a)h (send-receiv)o(e)h(op)q(eration)f(is)g(what)g(w)o(ould)g(obtain)g(if)g (the)g(caller)h(fork)o(ed)e(t)o(w)o(o)75 1585 y(concurren)o(t)j (threads,)f(one)i(to)e(execute)h(the)g(send,)h(and)f(one)g(to)f (execute)i(the)f(receiv)o(e,)g(follo)o(w)o(ed)g(b)o(y)g(a)75 1642 y(join)g(of)e(these)i(t)o(w)o(o)e(threads.)189 1748 y Fl(A)n(dvic)n(e)d(to)h(implementors.)37 b Fx(Additional)12 b(in)o(termediate)f(bu\013ering)g(is)g(needed)h(for)e(the)h(\\replace") 189 1804 y(v)m(arian)o(t.)19 b(\()p Fl(End)d(of)g(advic)n(e)h(to)f (implementors.)p Fx(\))75 1948 y Fn(3.11)59 b(Null)21 b(p)n(ro)r(cesses)75 2049 y Fx(In)15 b(man)o(y)e(instances,)i(it)f(is)h (con)o(v)o(enien)o(t)g(to)e(sp)q(ecify)j(a)d(\\dumm)o(y")h(source)g(or) g(destination)h(for)e(comm)o(u-)75 2106 y(nication.)23 b(This)16 b(simpli\014es)i(the)e(co)q(de)g(that)g(is)g(needed)h(for)e (dealing)i(with)f(b)q(oundaries,)h(for)e(example,)75 2162 y(in)h(the)f(case)h(of)e(a)h(non-circular)i(shift)e(done)h(with)g (calls)g(to)e(send-receiv)o(e.)166 2219 y(The)f(sp)q(ecial)h(v)m(alue) 25 b Fi(MPI)p 605 2219 13 2 v 14 w(PROC)p 731 2219 V 13 w(NULL)14 b Fx(can)e(b)q(e)h(used)g(instead)g(of)f(a)g(rank)g (wherev)o(er)h(a)f(source)g(or)g(a)75 2275 y(destination)j(argumen)o(t) e(is)h(required)i(in)e(a)g(call.)21 b(A)14 b(comm)o(unication)g(with)h (pro)q(cess)26 b Fi(MPI)p 1634 2275 V 15 w(PROC)p 1761 2275 V 13 w(NULL)75 2331 y Fx(has)11 b(no)h(e\013ect.)18 b(A)12 b(send)g(to)21 b Fi(MPI)p 633 2331 V 15 w(PROC)p 760 2331 V 13 w(NULL)13 b Fx(succeeds)f(and)g(returns)f(as)g(so)q(on)h (as)f(p)q(ossible.)20 b(A)12 b(receiv)o(e)75 2388 y(from)24 b Fi(MPI)p 269 2388 V 14 w(PROC)p 395 2388 V 13 w(NULL)14 b Fx(succeeds)g(and)f(returns)f(as)h(so)q(on)f(as)h(p)q(ossible)h(with) f(no)g(mo)q(di\014cations)h(to)e(the)75 2444 y(receiv)o(e)17 b(bu\013er.)22 b(When)17 b(a)e(receiv)o(e)i(with)g Fv(source)f Fx(=)31 b Fi(MPI)p 1080 2444 V 15 w(PROC)p 1207 2444 V 13 w(NULL)17 b Fx(is)f(executed)h(then)g(the)f(status)75 2501 y(ob)s(ject)f(returns)30 b Fv(source)16 b Fx(=)29 b Fi(MPI)p 661 2501 V 14 w(PROC)p 787 2501 V 14 w(NULL)p Fx(,)15 b Fv(tag)g Fx(=)30 b Fi(MPI)p 1147 2501 V 14 w(ANY)p 1246 2501 V 15 w(T)m(A)o(G)14 b Fx(and)h Fv(count)i(=)e(0)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 59 67 59 66 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(59)75 45 y Fn(3.12)59 b(Derived)19 b(datat)n(yp)r(es)75 147 y Fx(Up)13 b(to)f(here,)h(all)h(p)q(oin)o(t)f(to)f(p)q(oin)o(t)i (comm)o(unication)f(ha)o(v)o(e)f(in)o(v)o(olv)o(ed)i(only)f(con)o (tiguous)g(bu\013ers)g(con)o(tain-)75 203 y(ing)h(a)f(sequence)i(of)e (elemen)o(ts)i(of)e(the)g(same)h(t)o(yp)q(e.)19 b(This)14 b(is)g(to)q(o)f(constraining)h(on)g(t)o(w)o(o)e(accoun)o(ts.)19 b(One)75 259 y(often)d(w)o(an)o(ts)f(to)g(pass)h(messages)f(that)h(con) o(tain)g(v)m(alues)h(with)f(di\013eren)o(t)h(datat)o(yp)q(es)e(\(e.g.,) g(an)h(in)o(teger)75 316 y(coun)o(t,)e(follo)o(w)o(ed)g(b)o(y)g(a)g (sequence)h(of)e(real)i(n)o(um)o(b)q(ers\);)f(and)g(one)g(often)g(w)o (an)o(ts)f(to)g(send)i(noncon)o(tiguous)75 372 y(data)k(\(e.g.,)g(a)g (sub-blo)q(c)o(k)i(of)e(a)h(matrix\).)32 b(One)20 b(solution)h(is)f(to) f(pac)o(k)g(noncon)o(tiguous)h(data)f(in)o(to)g(a)75 429 y(con)o(tiguous)e(bu\013er)h(at)e(the)i(sender)g(site)f(and)h (unpac)o(k)f(it)h(bac)o(k)f(at)g(the)g(receiv)o(er)h(site.)27 b(This)18 b(has)f(the)75 485 y(disadv)m(an)o(tage)12 b(of)g(requiring)h(additional)h(memory-to-memory)d(cop)o(y)h(op)q (erations)g(at)g(b)q(oth)g(sites,)h(ev)o(en)75 542 y(when)j(the)g(comm) o(unication)g(subsystem)g(has)f(scatter-gather)g(capabilities.)24 b(Instead,)15 b Fv(MPI)g Fx(pro)o(vides)75 598 y(mec)o(hanisms)h(to)f (sp)q(ecify)i(more)f(general,)g(mixed,)g(and)g(noncon)o(tiguous)g(comm) o(unication)h(bu\013ers.)k(It)75 655 y(is)16 b(up)g(to)f(the)h (implemen)o(tation)h(to)e(decide)i(whether)f(data)f(should)h(b)q(e)g (\014rst)g(pac)o(k)o(ed)f(in)i(a)e(con)o(tiguous)75 711 y(bu\013er)c(b)q(efore)h(b)q(eing)h(transmitted,)e(or)g(whether)h(it)g (can)f(b)q(e)h(collected)i(directly)e(from)f(where)h(it)f(resides.)166 768 y(The)j(general)h(mec)o(hanisms)g(pro)o(vided)g(here)f(allo)o(w)h (one)f(to)g(transfer)f(directly)l(,)j(without)e(cop)o(ying,)75 824 y(ob)s(jects)j(of)h(v)m(arious)h(shap)q(e)f(and)h(size.)29 b(It)18 b(is)h(not)f(assumed)g(that)f(the)h Fv(MPI)g Fx(library)h(is)f(cognizan)o(t)g(of)75 880 y(the)d(ob)s(jects)f (declared)i(in)g(the)f(host)f(language.)20 b(Th)o(us,)15 b(if)g(one)g(w)o(an)o(ts)f(to)g(transfer)g(a)g(structure,)h(or)f(an)75 937 y(arra)o(y)g(section,)h(it)g(will)h(b)q(e)f(necessary)g(to)g(pro)o (vide)g(in)g Fv(MPI)g Fx(a)f(de\014nition)j(of)d(a)g(comm)o(unication)i (bu\013er)75 993 y(that)d(mimics)j(the)e(de\014nition)i(of)e(the)g (structure)g(or)f(arra)o(y)g(section)i(in)g(question.)20 b(These)15 b(facilities)h(can)75 1050 y(b)q(e)j(used)g(b)o(y)f(library) h(designers)h(to)d(de\014ne)j(comm)o(unication)f(functions)g(that)f (can)g(transfer)g(ob)s(jects)75 1106 y(de\014ned)e(in)g(the)f(host)f (language)h(|)g(b)o(y)g(deco)q(ding)h(their)f(de\014nitions)i(as)d(a)o (v)m(ailable)i(in)g(a)e(sym)o(b)q(ol)i(table)75 1163 y(or)f(a)g(dop)q(e)g(v)o(ector.)20 b(Suc)o(h)15 b(higher-lev)o(el)j (comm)o(unication)e(functions)g(are)f(not)g(part)f(of)h Fv(MPI)p Fx(.)166 1219 y(More)d(general)g(comm)o(unication)i(bu\013ers) e(are)g(sp)q(eci\014ed)i(b)o(y)f(replacing)g(the)g(basic)g(datat)o(yp)q (es)f(that)75 1276 y(ha)o(v)o(e)e(b)q(een)i(used)f(so)f(far)g(with)h (deriv)o(ed)g(datat)o(yp)q(es)f(that)g(are)g(constructed)g(from)g (basic)h(datat)o(yp)q(es)f(using)75 1332 y(the)j(constructors)e (describ)q(ed)j(in)g(this)f(section.)19 b(These)13 b(metho)q(ds)f(of)g (constructing)h(deriv)o(ed)h(datat)o(yp)q(es)75 1389 y(can)h(b)q(e)h(applied)h(recursiv)o(ely)l(.)166 1445 y(A)e Fq(general)j(datat)o(yp)q(e)e Fx(is)g(an)f(opaque)h(ob)s(ject)e (that)h(sp)q(eci\014es)i(t)o(w)o(o)d(things:)143 1536 y Ft(\017)23 b Fx(A)15 b(sequence)h(of)f(basic)h(datat)o(yp)q(es)143 1628 y Ft(\017)23 b Fx(A)15 b(sequence)h(of)f(in)o(teger)g(\(b)o(yte\)) g(displacemen)o(ts)166 1719 y(The)22 b(displacemen)o(ts)g(are)g(not)f (required)h(to)f(b)q(e)h(p)q(ositiv)o(e,)i(distinct,)g(or)d(in)h (increasing)h(order.)75 1776 y(Therefore,)15 b(the)g(order)h(of)e (items)i(need)g(not)f(coincide)j(with)d(their)h(order)g(in)g(store,)e (and)i(an)f(item)h(ma)o(y)75 1832 y(app)q(ear)j(more)g(than)g(once.)33 b(W)l(e)19 b(call)i(suc)o(h)e(a)g(pair)h(of)f(sequences)h(\(or)f (sequence)h(of)f(pairs\))g(a)g Fq(t)o(yp)q(e)75 1889 y(map)p Fx(.)24 b(The)17 b(sequence)h(of)f(basic)g(datat)o(yp)q(es)f (\(displacemen)o(ts)i(ignored\))f(is)g(the)g Fq(t)o(yp)q(e)i(signature) e Fx(of)75 1945 y(the)e(datat)o(yp)q(e.)166 2002 y(Let)189 2092 y Fp(T)6 b(y)r(pemap)12 b Fx(=)h Ft(f)p Fx(\()p Fp(ty)r(pe)562 2099 y Fg(0)581 2092 y Fp(;)8 b(disp)686 2099 y Fg(0)705 2092 y Fx(\))p Fp(;)g(:::;)g Fx(\()o Fp(ty)r(pe)905 2099 y Ff(n)p Fh(\000)p Fg(1)971 2092 y Fp(;)g(disp)1076 2099 y Ff(n)p Fh(\000)p Fg(1)1144 2092 y Fx(\))p Ft(g)p Fp(;)75 2183 y Fx(b)q(e)16 b(suc)o(h)f(a)g(t)o (yp)q(e)h(map,)e(where)i Fp(ty)r(pe)709 2190 y Ff(i)738 2183 y Fx(are)f(basic)h(t)o(yp)q(es,)f(and)g Fp(disp)1233 2190 y Ff(i)1262 2183 y Fx(are)g(displacemen)o(ts.)21 b(Let)189 2274 y Fp(T)6 b(y)r(pesig)13 b Fx(=)g Ft(f)p Fp(ty)r(pe)517 2281 y Fg(0)537 2274 y Fp(;)8 b(:::;)g(ty)r(pe)702 2281 y Ff(n)p Fh(\000)p Fg(1)768 2274 y Ft(g)75 2365 y Fx(b)q(e)14 b(the)f(asso)q(ciated)h(t)o(yp)q(e)f(signature.)19 b(This)14 b(t)o(yp)q(e)f(map,)g(together)g(with)g(a)g(base)g(address)h Fl(buf)p Fx(,)f(sp)q(eci\014es)75 2421 y(a)19 b(comm)o(unication)i (bu\013er:)28 b(the)20 b(comm)o(unication)h(bu\013er)e(that)g(consists) h(of)f Fp(n)h Fx(en)o(tries,)h(where)f(the)75 2478 y Fp(i)p Fx(-th)c(en)o(try)f(is)i(at)e(address)h Fp(buf)f Fx(+)c Fp(disp)763 2485 y Ff(i)793 2478 y Fx(and)16 b(has)g(t)o(yp)q(e) g Fp(ty)r(pe)1151 2485 y Ff(i)1165 2478 y Fx(.)23 b(A)16 b(message)f(assem)o(bled)i(from)e(suc)o(h)h(a)75 2534 y(comm)o(unication)g(bu\013er)f(will)i(consist)e(of)g Fp(n)h Fx(v)m(alues,)g(of)e(the)i(t)o(yp)q(es)f(de\014ned)h(b)o(y)g Fp(T)6 b(y)r(pesig)r Fx(.)166 2591 y(W)l(e)11 b(can)f(use)h(a)f(handle) i(to)e(a)g(general)h(datat)o(yp)q(e)f(as)h(an)f(argumen)o(t)g(in)h(a)f (send)i(or)e(receiv)o(e)h(op)q(eration,)75 2647 y(instead)h(of)e(a)h (basic)g(datat)o(yp)q(e)g(argumen)o(t.)18 b(The)11 b(op)q(eration)g Fv(MPI)p 1198 2647 14 2 v 16 w(SEND\(buf,)h(1,)f(datat)o(yp)q(e,...\)) 19 b Fx(will)13 b(use)75 2704 y(the)18 b(send)g(bu\013er)f(de\014ned)i (b)o(y)e(the)h(base)f(address)h Fv(buf)g Fx(and)g(the)f(general)h (datat)o(yp)q(e)f(asso)q(ciated)h(with)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 60 68 60 67 bop 75 -100 a Fx(60)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(datat)o(yp)q(e)p Fx(;)23 b(it)c(will)i(generate)f(a)f(message)g(with)g(the)h(t)o(yp)q(e) f(signature)h(determined)h(b)o(y)e(the)g Fv(datat)o(yp)q(e)75 102 y Fx(argumen)o(t.)28 b Fv(MPI)p 390 102 14 2 v 16 w(RECV\(buf,)20 b(1,)e(datat)o(yp)q(e,...\))29 b Fx(will)20 b(use)e(the)h(receiv)o(e)g(bu\013er)f(de\014ned)h(b)o(y)g(the)f(base)75 158 y(address)d Fv(buf)h Fx(and)g(the)f(general)h(datat)o(yp)q(e)e (asso)q(ciated)i(with)f Fv(datat)o(yp)q(e)p Fx(.)166 214 y(General)h(datat)o(yp)q(es)f(can)h(b)q(e)h(used)f(in)h(all)g(send) f(and)h(receiv)o(e)f(op)q(erations.)22 b(W)l(e)16 b(discuss,)h(in)g (Sec.)75 271 y(3.12.5,)c(the)i(case)g(where)h(the)f(second)h(argumen)o (t)e Fv(count)j Fx(has)e(v)m(alue)h Fp(>)d Fx(1.)166 327 y(The)e(basic)g(datat)o(yp)q(es)f(presen)o(ted)i(in)f(section)g (3.2.2)f(are)g(particular)h(cases)g(of)f(a)h(general)g(datat)o(yp)q(e,) 75 384 y(and)19 b(are)f(prede\014ned.)32 b(Th)o(us,)19 b Fv(MPI)p 711 384 V 16 w(INT)f Fx(is)h(a)g(prede\014ned)h(handle)g(to) e(a)g(datat)o(yp)q(e)g(with)h(t)o(yp)q(e)g(map)75 440 y Ft(f)p Fx(\()p Fv(int)p Fp(;)8 b Fx(0\))p Ft(g)p Fx(,)13 b(with)j(one)g(en)o(try)f(of)g(t)o(yp)q(e)g Fv(int)h Fx(and)g(displacemen)o(t)h(zero.)j(The)15 b(other)g(basic)i(datat)o(yp) q(es)d(are)75 497 y(similar.)166 553 y(The)h Fq(exten)o(t)f Fx(of)g(a)g(datat)o(yp)q(e)g(is)h(de\014ned)g(to)f(b)q(e)h(the)g(span)f (from)g(the)g(\014rst)g(b)o(yte)h(to)e(the)i(last)f(b)o(yte)75 610 y(o)q(ccupied)19 b(b)o(y)d(en)o(tries)h(in)h(this)f(datat)o(yp)q (e,)f(rounded)h(up)g(to)f(satisfy)h(alignmen)o(t)g(requiremen)o(ts.)25 b(That)75 666 y(is,)15 b(if)189 752 y Fp(T)6 b(y)r(pemap)12 b Fx(=)h Ft(f)p Fx(\()p Fp(ty)r(pe)562 759 y Fg(0)581 752 y Fp(;)8 b(disp)686 759 y Fg(0)705 752 y Fx(\))p Fp(;)g(:::;)g Fx(\()o Fp(ty)r(pe)905 759 y Ff(n)p Fh(\000)p Fg(1)971 752 y Fp(;)g(disp)1076 759 y Ff(n)p Fh(\000)p Fg(1)1144 752 y Fx(\))p Ft(g)p Fp(;)75 837 y Fx(then)283 922 y Fp(l)q(b)p Fx(\()p Fp(T)e(y)r(pemap)p Fx(\))40 b(=)h(min)688 951 y Ff(j)742 922 y Fp(disp)826 929 y Ff(j)844 922 y Fp(;)272 1013 y(ub)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))40 b(=)h(max)693 1041 y Ff(j)743 1013 y Fx(\()p Fp(disp)845 1020 y Ff(j)873 1013 y Fx(+)10 b Fp(siz)r(eof)5 b Fx(\()p Fp(ty)r(pe)1150 1020 y Ff(j)1169 1013 y Fx(\)\))p Fp(;)21 b Fx(and)189 1103 y Fp(extent)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))42 b(=)f Fp(ub)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))j Ft(\000)h Fp(l)q(b)p Fx(\()p Fp(T)c(y)r(pemap)p Fx(\))j(+)h Fp(\017:)456 b Fx(\(3.1\))75 1188 y(If)17 b Fp(ty)r(pe)206 1195 y Ff(i)237 1188 y Fx(requires)g(alignmen)o(t)g (to)f(a)h(b)o(yte)f(address)h(that)f(is)h(is)g(a)f(m)o(ultiple)j(of)d Fp(k)1474 1195 y Ff(i)1488 1188 y Fx(,)g(then)h Fp(\017)g Fx(is)g(the)g(least)75 1245 y(nonnegativ)o(e)f(incremen)o(t)g(needed)g (to)f(round)g Fp(extent)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))16 b(to)e(the)h(next)h(m)o(ultiple)h(of)d(max)1794 1252 y Ff(i)1816 1245 y Fp(k)1840 1252 y Ff(i)1854 1245 y Fx(.)75 1339 y Fq(Example)k(3.18)23 b Fx(Assume)12 b(that)g Fp(T)6 b(y)r(pe)13 b Fx(=)g Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)j Fx(\(a)h Fv(double)i Fx(at)e(displacemen)o(t)i(zero,)75 1396 y(follo)o(w)o(ed)k(b)o(y)g(a)f Fv(cha)o(r)g Fx(at)g(displacemen)o (t)j(eigh)o(t\).)27 b(Assume,)18 b(furthermore,)f(that)g(doubles)i(ha)o (v)o(e)e(to)g(b)q(e)75 1452 y(strictly)d(aligned)g(at)e(addresses)i (that)e(are)h(m)o(ultiples)h(of)f(eigh)o(t.)19 b(Then,)14 b(the)f(exten)o(t)g(of)f(this)i(datat)o(yp)q(e)e(is)75 1508 y(16)e(\(9)g(rounded)h(to)f(the)h(next)g(m)o(ultiple)h(of)f(8\).) 17 b(A)11 b(datat)o(yp)q(e)f(that)g(consists)h(of)f(a)g(c)o(haracter)g (immediately)75 1565 y(follo)o(w)o(ed)16 b(b)o(y)f(a)g(double)h(will)h (also)e(ha)o(v)o(e)g(an)g(exten)o(t)g(of)g(16.)189 1659 y Fl(R)n(ationale.)39 b Fx(The)15 b(de\014nition)h(of)e(exten)o(t)g(is) h(motiv)m(ated)f(b)o(y)h(the)f(assumption)h(that)e(the)i(amoun)o(t)189 1716 y(of)i(padding)i(added)g(at)e(the)h(end)g(of)g(eac)o(h)g (structure)f(in)i(an)f(arra)o(y)e(of)i(structures)f(is)i(the)f(least) 189 1772 y(needed)d(to)e(ful\014ll)j(alignmen)o(t)e(constrain)o(ts.)19 b(More)14 b(explicit)i(con)o(trol)d(of)g(the)h(exten)o(t)g(is)g(pro)o (vided)189 1829 y(in)i(section)g(3.12.3.)k(Suc)o(h)c(explicit)i(con)o (trol)e(is)g(needed)h(in)f(cases)g(where)g(the)g(assumption)g(do)q(es) 189 1885 y(not)e(hold,)i(for)f(example,)g(where)h(union)g(t)o(yp)q(es)f (are)g(used.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))75 2005 y Fj(3.12.1)49 b(Datat)o(yp)q(e)17 b(constructo)o(rs)75 2091 y Fv(Contiguous)47 b Fx(The)16 b(simplest)i(datat)o(yp)q(e)e (constructor)f(is)i Fv(MPI)p 1171 2091 V 16 w(TYPE)p 1304 2091 V 17 w(CONTIGUOUS)h Fx(whic)o(h)f(allo)o(ws)75 2148 y(replication)g(of)e(a)g(datat)o(yp)q(e)f(in)o(to)h(con)o(tiguous) h(lo)q(cations.)75 2299 y Fv(MPI)p 160 2299 V 16 w(TYPE)p 293 2299 V 17 w(CONTIGUOUS\(count,)h(oldt)o(yp)q(e,)f(newt)o(yp)q(e\)) 117 2376 y Fk(IN)155 b Fv(count)482 b Fk(replication)14 b(coun)o(t)g(\(nonnegativ)o(e)g(in)o(teger\))117 2449 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 2523 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2647 y Fu(int)23 b(MPI)p 245 2647 15 2 v 17 w(Type)p 358 2647 V 17 w(contiguous\(int)f (count,)h(MPI)p 972 2647 V 17 w(Datatype)f(oldtype,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)h(*newtype\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 61 69 61 68 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(61)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(CONTIGUOUS\(COUNT,)22 b(OLDTYPE,)h(NEWTYPE,)f(IERROR\)) 170 102 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(NEWTYPE,)g(IERROR)166 188 y Fv(newt)o(yp)q(e)c Fx(is)f(the)f(datat)o(yp)q(e)g(obtained)h(b)o (y)f(concatenating)h Fv(count)h Fx(copies)f(of)f Fv(oldt)o(yp)q(e)p Fx(.)27 b(Concate-)75 244 y(nation)15 b(is)h(de\014ned)h(using)f Fl(extent)e Fx(as)h(the)h(size)g(of)e(the)i(concatenated)f(copies.)75 339 y Fq(Example)j(3.19)23 b Fx(Let)17 b Fv(oldt)o(yp)q(e)g Fx(ha)o(v)o(e)g(t)o(yp)q(e)g(map)f Ft(f)p Fx(\()p Fv(double)q Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)p Fp(;)15 b Fx(with)i(exten)o(t)f(16,)h(and)f(let)75 396 y Fv(count)e Fx(=)f(3.)19 b(The)d(t)o(yp)q(e)f(map)g(of)g(the)g (datat)o(yp)q(e)g(returned)g(b)o(y)h Fv(newt)o(yp)q(e)h Fx(is)189 482 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(16\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(24\))p Fp(;)g Fx(\()p Fv(double)n Fp(;)g Fx(32\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(40\))p Ft(g)p Fx(;)75 567 y(i.e.,)15 b(alternating)g Fv(double)i Fx(and)e Fv(cha)o(r)g Fx(elemen)o(ts,)h(with)f(displacemen)o(ts)i(0)p Fp(;)8 b Fx(8)p Fp(;)g Fx(16)p Fp(;)f Fx(24)p Fp(;)g Fx(32)o Fp(;)h Fx(4)o(0.)166 745 y(In)16 b(general,)f(assume)g(that)g(the)g(t)o (yp)q(e)g(map)g(of)g Fv(oldt)o(yp)q(e)h Fx(is)189 831 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 838 y Fg(0)333 831 y Fp(;)8 b(disp)438 838 y Fg(0)457 831 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 838 y Ff(n)p Fh(\000)p Fg(1)723 831 y Fp(;)h(disp)828 838 y Ff(n)p Fh(\000)p Fg(1)895 831 y Fx(\))p Ft(g)p Fp(;)75 917 y Fx(with)16 b(exten)o(t)e Fp(ex)p Fx(.)20 b(Then)c Fv(newt)o(yp)q(e)h Fx(has)e(a)g(t)o(yp)q(e)g (map)g(with)h Fv(count)11 b Ft(\001)f Fv(n)16 b Fx(en)o(tries)f (de\014ned)i(b)o(y:)189 1003 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1010 y Fg(0)333 1003 y Fp(;)8 b(disp)438 1010 y Fg(0)457 1003 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1010 y Ff(n)p Fh(\000)p Fg(1)723 1003 y Fp(;)h(disp)828 1010 y Ff(n)p Fh(\000)p Fg(1)895 1003 y Fx(\))p Fp(;)g Fx(\()p Fp(ty)r(pe)1036 1010 y Fg(0)1055 1003 y Fp(;)g(disp)1160 1010 y Fg(0)1189 1003 y Fx(+)i Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)1482 1010 y Ff(n)p Fh(\000)p Fg(1)1548 1003 y Fp(;)g(disp)1653 1010 y Ff(n)p Fh(\000)p Fg(1)1730 1003 y Fx(+)j Fp(ex)p Fx(\))p Fp(;)189 1098 y(:::;)d Fx(\()p Fp(ty)r(pe)351 1105 y Fg(0)368 1098 y Fp(;)g(disp)473 1105 y Fg(0)502 1098 y Fx(+)i Fp(ex)h Ft(\001)e Fx(\()p Fv(count)j Ft(\000)e Fx(1\)\))p Fp(;)e(:::;)f Fx(\()p Fp(ty)r(pe)1048 1105 y Ff(n)p Fh(\000)p Fg(1)1114 1098 y Fp(;)h(disp)1219 1105 y Ff(n)p Fh(\000)p Fg(1)1296 1098 y Fx(+)j Fp(ex)f Ft(\001)g Fx(\()p Fv(count)h Ft(\000)f Fx(1\)\))p Ft(g)p Fp(:)75 1337 y Fv(V)o(ecto)o(r)45 b Fx(The)16 b(function)g Fv(MPI)p 600 1337 14 2 v 16 w(TYPE)p 733 1337 V 17 w(VECTOR)g Fx(is)g(a)f(more)g(general)h(constructor)e (that)h(allo)o(ws)g(repli-)75 1394 y(cation)20 b(of)g(a)g(datat)o(yp)q (e)f(in)o(to)h(lo)q(cations)h(that)e(consist)i(of)e(equally)j(spaced)f (blo)q(c)o(ks.)35 b(Eac)o(h)20 b(blo)q(c)o(k)g(is)75 1450 y(obtained)g(b)o(y)f(concatenating)h(the)f(same)g(n)o(um)o(b)q(er) h(of)f(copies)h(of)f(the)g(old)h(datat)o(yp)q(e.)32 b(The)19 b(spacing)75 1507 y(b)q(et)o(w)o(een)d(blo)q(c)o(ks)f(is)h(a)f(m)o (ultiple)i(of)e(the)g(exten)o(t)g(of)g(the)g(old)h(datat)o(yp)q(e.)75 1658 y Fv(MPI)p 160 1658 V 16 w(TYPE)p 293 1658 V 17 w(VECTOR\()g(count,)g(blo)q(cklength,)g(stride,)g(oldt)o(yp)q(e,)g (newt)o(yp)q(e\))117 1735 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o (teger\))117 1808 y(IN)155 b Fv(blo)q(cklength)371 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e (\(nonnegativ)o(e)h(in)o(te-)905 1865 y(ger\))117 1938 y(IN)155 b Fv(stride)484 b Fk(n)o(um)o(b)q(er)13 b(of)f(elemen)o(ts)h (b)q(et)o(w)o(een)h(start)g(of)f(eac)o(h)g(blo)q(c)o(k)g(\(in)o(te-)905 1995 y(ger\))117 2068 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i(\(handle\))117 2142 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2266 y Fu(int)23 b(MPI)p 245 2266 15 2 v 17 w(Type)p 358 2266 V 17 w(vector\(int)f(count,)h(int)h(blocklength,)e(int)i (stride,)393 2323 y(MPI)p 468 2323 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 2323 V 17 w(Datatype)h(*newtype\))75 2409 y(MPI)p 150 2409 V 17 w(TYPE)p 263 2409 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR\))170 2466 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 2591 y Fq(Example)18 b(3.20)23 b Fx(Assume,)15 b(again,)f(that)h Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g(map)g Ft(f)p Fx(\()p Fv(double)q Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)p Fp(;)13 b Fx(with)j(ex-)75 2647 y(ten)o(t)g(16.)21 b(A)c(call)g(to)e Fv(MPI)p 527 2647 14 2 v 16 w(TYPE)p 660 2647 V 17 w(VECTOR\()i(2,)e(3,)h(4,)f(oldt) o(yp)q(e,)i(newt)o(yp)q(e\))h Fx(will)g(create)d(the)h(datat)o(yp)q(e) 75 2704 y(with)g(t)o(yp)q(e)f(map,)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 62 70 62 69 bop 75 -100 a Fx(62)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(16\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(24\))p Fp(;)g Fx(\()p Fv(double)n Fp(;)g Fx(32\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(40\))p Fp(;)189 151 y Fx(\()p Fv(double)q Fp(;)g Fx(64\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(72\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(80\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(88\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(96\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(104\))p Ft(g)p Fp(:)75 258 y Fx(That)15 b(is,)g(t)o(w)o(o)f (blo)q(c)o(ks)i(with)f(three)h(copies)g(eac)o(h)f(of)g(the)g(old)h(t)o (yp)q(e,)f(starting)f(4)c Ft(\001)g Fx(16)k(apart.)75 364 y Fq(Example)k(3.21)23 b Fx(A)c(call)i(to)d Fv(MPI)p 692 364 14 2 v 16 w(TYPE)p 825 364 V 17 w(VECTOR\(3,)j(1,)e(-2,)h(oldt) o(yp)q(e,)i(newt)o(yp)q(e\))f Fx(will)g(create)e(the)75 420 y(datat)o(yp)q(e,)189 514 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Ft(\000)p Fx(32\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Ft(\000)p Fx(24\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Ft(\000)p Fx(64\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Ft(\000)p Fx(56\))p Ft(g)p Fp(:)166 703 y Fx(In)16 b(general,)f(assume)g(that)g Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g(map,)189 797 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 804 y Fg(0)333 797 y Fp(;)8 b(disp)438 804 y Fg(0)457 797 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 804 y Ff(n)p Fh(\000)p Fg(1)723 797 y Fp(;)h(disp)828 804 y Ff(n)p Fh(\000)p Fg(1)895 797 y Fx(\))p Ft(g)p Fp(;)75 891 y Fx(with)14 b(exten)o(t)f Fp(ex)p Fx(.)19 b(Let)13 b Fv(bl)h Fx(b)q(e)g(the)f Fv(blo)q(cklength)p Fx(.)21 b(The)14 b(newly)g(created)f(datat)o(yp)q(e)g(has)g(a)g(t)o(yp) q(e)h(map)f(with)75 947 y Fv(count)e Ft(\001)f Fv(bl)h Ft(\001)e Fp(n)16 b Fx(en)o(tries:)189 1041 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1048 y Fg(0)333 1041 y Fp(;)8 b(disp)438 1048 y Fg(0)457 1041 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1048 y Ff(n)p Fh(\000)p Fg(1)723 1041 y Fp(;)h(disp)828 1048 y Ff(n)p Fh(\000)p Fg(1)895 1041 y Fx(\))p Fp(;)189 1147 y Fx(\()p Fp(ty)r(pe)291 1154 y Fg(0)310 1147 y Fp(;)g(disp)415 1154 y Fg(0)444 1147 y Fx(+)i Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)737 1154 y Ff(n)p Fh(\000)p Fg(1)803 1147 y Fp(;)g(disp)908 1154 y Ff(n)p Fh(\000)p Fg(1)985 1147 y Fx(+)j Fp(ex)p Fx(\))p Fp(;)d(:::;)189 1254 y Fx(\()p Fp(ty)r(pe)291 1261 y Fg(0)310 1254 y Fp(;)g(disp)415 1261 y Fg(0)444 1254 y Fx(+)i(\()p Fv(bl)g Ft(\000)h Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)918 1261 y Ff(n)p Fh(\000)p Fg(1)984 1254 y Fp(;)g(disp)1089 1261 y Ff(n)p Fh(\000)p Fg(1)1166 1254 y Fx(+)j(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)189 1360 y Fx(\()p Fp(ty)r(pe)291 1367 y Fg(0)310 1360 y Fp(;)e(disp)415 1367 y Fg(0)444 1360 y Fx(+)i Fv(stride)h Ft(\001)f Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)874 1367 y Ff(n)p Fh(\000)p Fg(1)940 1360 y Fp(;)g(disp)1045 1367 y Ff(n)p Fh(\000)p Fg(1)1122 1360 y Fx(+)j Fv(stride)g Ft(\001)e Fp(ex)p Fx(\))p Fp(;)f(:::;)189 1466 y Fx(\()p Fp(ty)r(pe)291 1473 y Fg(0)310 1466 y Fp(;)g(disp)415 1473 y Fg(0)444 1466 y Fx(+)i(\()p Fv(stride)h Fx(+)g Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()o Fp(ty)r(pe)1077 1473 y Ff(n)p Fh(\000)p Fg(1)1143 1466 y Fp(;)g(disp)1248 1473 y Ff(n)p Fh(\000)p Fg(1)1326 1466 y Fx(+)i(\()p Fv(stride)h Fx(+)f Fv(bl)h Ft(\000)f Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(::::;)189 1572 y Fx(\()p Fp(ty)r(pe)291 1579 y Fg(0)310 1572 y Fp(;)g(disp)415 1579 y Fg(0)444 1572 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)g Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 1679 y Fx(\()p Fp(ty)r(pe)291 1686 y Ff(n)p Fh(\000)p Fg(1)359 1679 y Fp(;)g(disp)464 1686 y Ff(n)p Fh(\000)p Fg(1)542 1679 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)f Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)e(:::;)189 1785 y Fx(\()p Fp(ty)r(pe)291 1792 y Fg(0)310 1785 y Fp(;)g(disp)415 1792 y Fg(0)444 1785 y Fx(+)i(\()p Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)f Fx(1\))g(+)g Fv(bl)h Ft(\000)f Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(:::;)189 1891 y Fx(\()p Fp(ty)r(pe)291 1898 y Ff(n)p Fh(\000)p Fg(1)359 1891 y Fp(;)g(disp)464 1898 y Ff(n)p Fh(\000)p Fg(1)542 1891 y Fx(+)i(\()p Fv(stride)h Ft(\001)e Fx(\()p Fv(count)j Ft(\000)e Fx(1\))g(+)g Fv(bl)g Ft(\000)h Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Ft(g)p Fp(:)166 2080 y Fx(A)16 b(call)i(to)e Fv(MPI)p 443 2080 V 16 w(TYPE)p 576 2080 V 16 w(CONTIGUOUS\(count,)k(oldt)o(yp)q(e,)d(newt)o(yp)q(e\))h Fx(is)f(equiv)m(alen)o(t)h(to)e(a)g(call)i(to)75 2137 y Fv(MPI)p 160 2137 V 16 w(TYPE)p 293 2137 V 17 w(VECTOR\(count,)g(1,)f (1,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p Fx(,)h(or)e(to)g(a)h(call)g (to)f Fv(MPI)p 1495 2137 V 16 w(TYPE)p 1628 2137 V 17 w(VECTOR\(1,)75 2193 y(count,)g(n,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p Fx(,)h Fv(n)e Fx(arbitrary)l(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 63 71 63 70 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(63)75 45 y Fv(Hvecto)o(r)45 b Fx(The)19 b(function)h Fv(MPI)p 631 45 14 2 v 16 w(TYPE)p 764 45 V 17 w(HVECTOR)g Fx(is)f(iden)o(tical)i(to)d Fv(MPI)p 1389 45 V 16 w(TYPE)p 1522 45 V 17 w(VECTOR)p Fx(,)h(except)75 102 y(that)h Fv(stride)h Fx(is)f(giv)o(en)h(in)g(b)o(ytes,)g(rather)f(than)g(in)h (elemen)o(ts.)36 b(The)20 b(use)h(for)e(b)q(oth)i(t)o(yp)q(es)f(of)g(v) o(ector)75 158 y(constructors)14 b(is)i(illustrated)h(in)f(Sec.)k (3.12.7.)e(\()p Fv(H)d Fx(stands)g(for)g(\\heterogeneous"\).)75 309 y Fv(MPI)p 160 309 V 16 w(TYPE)p 293 309 V 17 w(HVECTOR\()h(count,) g(blo)q(cklength,)h(stride,)e(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))117 386 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o(teger\))117 461 y(IN)155 b Fv(blo)q(cklength)371 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 518 y(ger\))117 593 y(IN)155 b Fv(stride)484 b Fk(n)o(um)o(b)q(er)13 b(of)g(b)o(ytes)h(b)q(et)o(w)o(een)h(start)f (of)f(eac)o(h)h(blo)q(c)o(k)f(\(in)o(teger\))117 668 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 743 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 868 y Fu(int)23 b(MPI)p 245 868 15 2 v 17 w(Type)p 358 868 V 17 w(hvector\(int)f (count,)h(int)h(blocklength,)e(MPI)p 1306 868 V 17 w(Aint)h(stride,)393 924 y(MPI)p 468 924 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 924 V 17 w(Datatype)h(*newtype\))75 1011 y(MPI)p 150 1011 V 17 w(TYPE)p 263 1011 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)f(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1067 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)166 1236 y Fx(Assume)15 b(that)g Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g (map,)189 1330 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1337 y Fg(0)333 1330 y Fp(;)8 b(disp)438 1337 y Fg(0)457 1330 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1337 y Ff(n)p Fh(\000)p Fg(1)723 1330 y Fp(;)h(disp)828 1337 y Ff(n)p Fh(\000)p Fg(1)895 1330 y Fx(\))p Ft(g)p Fp(;)75 1424 y Fx(with)14 b(exten)o(t)f Fp(ex)p Fx(.)19 b(Let)13 b Fv(bl)h Fx(b)q(e)g(the)f Fv(blo)q(cklength)p Fx(.)21 b(The)14 b(newly)g(created)f(datat)o(yp)q(e)g(has)g(a)g(t)o(yp)q(e)h (map)f(with)75 1480 y Fv(count)e Ft(\001)f Fv(bl)h Ft(\001)e Fp(n)16 b Fx(en)o(tries:)189 1574 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1581 y Fg(0)333 1574 y Fp(;)8 b(disp)438 1581 y Fg(0)457 1574 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1581 y Ff(n)p Fh(\000)p Fg(1)723 1574 y Fp(;)h(disp)828 1581 y Ff(n)p Fh(\000)p Fg(1)895 1574 y Fx(\))p Fp(;)189 1680 y Fx(\()p Fp(ty)r(pe)291 1687 y Fg(0)310 1680 y Fp(;)g(disp)415 1687 y Fg(0)444 1680 y Fx(+)i Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)737 1687 y Ff(n)p Fh(\000)p Fg(1)803 1680 y Fp(;)g(disp)908 1687 y Ff(n)p Fh(\000)p Fg(1)985 1680 y Fx(+)j Fp(ex)p Fx(\))p Fp(;)d(:::;)189 1787 y Fx(\()p Fp(ty)r(pe)291 1794 y Fg(0)310 1787 y Fp(;)g(disp)415 1794 y Fg(0)444 1787 y Fx(+)i(\()p Fv(bl)g Ft(\000)h Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)918 1794 y Ff(n)p Fh(\000)p Fg(1)984 1787 y Fp(;)g(disp)1089 1794 y Ff(n)p Fh(\000)p Fg(1)1166 1787 y Fx(+)j(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)189 1893 y Fx(\()p Fp(ty)r(pe)291 1900 y Fg(0)310 1893 y Fp(;)e(disp)415 1900 y Fg(0)444 1893 y Fx(+)i Fv(stride)q Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)794 1900 y Ff(n)p Fh(\000)p Fg(1)860 1893 y Fp(;)g(disp)965 1900 y Ff(n)p Fh(\000)p Fg(1)1042 1893 y Fx(+)j Fv(stride)q Fx(\))p Fp(;)d(:::;)189 1999 y Fx(\()p Fp(ty)r(pe)291 2006 y Fg(0)310 1999 y Fp(;)g(disp)415 2006 y Fg(0)444 1999 y Fx(+)i Fv(stride)h Fx(+)g(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)e(:::;)189 2105 y Fx(\()p Fp(ty)r(pe)291 2112 y Ff(n)p Fh(\000)p Fg(1)359 2105 y Fp(;)g(disp)464 2112 y Ff(n)p Fh(\000)p Fg(1)542 2105 y Fx(+)i Fv(stride)h Fx(+)g(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(::::;)189 2212 y Fx(\()p Fp(ty)r(pe)291 2219 y Fg(0)310 2212 y Fp(;)g(disp)415 2219 y Fg(0)444 2212 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)g Fx(1\)\))p Fp(;)d(:::;)f Fx(\()p Fp(ty)r(pe)1047 2219 y Ff(n)p Fh(\000)p Fg(1)1113 2212 y Fp(;)h(disp)1218 2219 y Ff(n)p Fh(\000)p Fg(1)1295 2212 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)g Fx(1\)\))p Fp(;)d(:::;)189 2318 y Fx(\()p Fp(ty)r(pe)291 2325 y Fg(0)310 2318 y Fp(;)g(disp)415 2325 y Fg(0)444 2318 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)g Fx(1\))e(+)i(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(:::;)189 2424 y Fx(\()p Fp(ty)r(pe)291 2431 y Ff(n)p Fh(\000)p Fg(1)359 2424 y Fp(;)g(disp)464 2431 y Ff(n)p Fh(\000)p Fg(1)542 2424 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)f Fx(1\))g(+)g(\()p Fv(bl)g Ft(\000)h Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Ft(g)p Fp(:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 64 72 64 71 bop 75 -100 a Fx(64)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(Indexed)47 b Fx(The)19 b(function)h Fv(MPI)p 629 45 14 2 v 16 w(TYPE)p 762 45 V 17 w(INDEXED)f Fx(allo)o(ws)h(replication)h(of)d(an)i(old)f (datat)o(yp)q(e)g(in)o(to)g(a)75 102 y(sequence)g(of)f(blo)q(c)o(ks)h (\(eac)o(h)f(blo)q(c)o(k)h(is)g(a)f(concatenation)g(of)g(the)g(old)h (datat)o(yp)q(e\),)e(where)i(eac)o(h)f(blo)q(c)o(k)75 158 y(can)23 b(con)o(tain)f(a)g(di\013eren)o(t)h(n)o(um)o(b)q(er)g(of)f (copies)h(and)g(ha)o(v)o(e)f(a)g(di\013eren)o(t)g(displacemen)o(t.)43 b(All)24 b(blo)q(c)o(k)75 214 y(displacemen)o(ts)17 b(are)e(m)o (ultiples)h(of)f(the)h(old)f(t)o(yp)q(e)h(exten)o(t.)75 365 y Fv(MPI)p 160 365 V 16 w(TYPE)p 293 365 V 17 w(INDEXED\()c(count,) i(a)o(rra)o(y)p 762 365 V 14 w(of)p 813 365 V 16 w(blo)q(cklengths,)h (a)o(rra)o(y)p 1185 365 V 15 w(of)p 1237 365 V 16 w(displacements,)e (oldt)o(yp)q(e,)h(newt)o(yp)q(e\))117 499 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f (n)o(um)o(b)q(er)h(of)f(en)o(tries)i(in)905 556 y Fi(a)o(rra)o(y)p 992 556 13 2 v 15 w(of)p 1041 556 V 15 w(displacements)17 b Fk(and)h Fi(a)o(rra)o(y)p 1483 556 V 15 w(of)p 1532 556 V 15 w(blo)q(cklengths)i Fk(\(non-)905 612 y(negativ)o(e)14 b(in)o(teger\))117 687 y(IN)155 b Fv(a)o(rra)o(y)p 416 687 14 2 v 15 w(of)p 468 687 V 16 w(blo)q(cklengths)191 b Fk(n)o(um)o(b)q(er)18 b(of)f(elemen)o(ts)h(p)q(er)i(blo)q(c)o(k)d (\(arra)o(y)i(of)e(nonnegativ)o(e)905 744 y(in)o(tegers\))117 819 y(IN)155 b Fv(a)o(rra)o(y)p 416 819 V 15 w(of)p 468 819 V 16 w(displacements)162 b Fk(displacemen)o(t)19 b(for)h(eac)o(h)g(blo)q(c)o(k,)g(in)g(m)o(ultiples)e(of)h Fi(oldt)o(yp)q(e)905 875 y Fk(exten)o(t)c(\(arra)o(y)f(of)f(in)o (teger\))117 950 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i(\(handle\))117 1025 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1150 y Fu(int)23 b(MPI)p 245 1150 15 2 v 17 w(Type)p 358 1150 V 17 w(indexed\(int)f (count,)h(int)h(*array)p 1068 1150 V 16 w(of)p 1132 1150 V 17 w(blocklengths,)393 1206 y(int)g(*array)p 636 1206 V 16 w(of)p 700 1206 V 17 w(displacements,)e(MPI)p 1147 1206 V 17 w(Datatype)g(oldtype,)393 1263 y(MPI)p 468 1263 V 17 w(Datatype)h(*newtype\))75 1349 y(MPI)p 150 1349 V 17 w(TYPE)p 263 1349 V 16 w(INDEXED\(COUNT,)f(ARRAY)p 757 1349 V 17 w(OF)p 822 1349 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 1349 V 16 w(OF)p 1357 1349 V 17 w(DISPLACEMENTS,)393 1406 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1462 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1462 V 17 w(OF)p 716 1462 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1462 V 17 w(OF)p 1323 1462 V 17 w(DISPLACEMENTS\(*\),)170 1519 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1655 y Fq(Example)18 b(3.22)23 b Fx(Let)14 b Fv(oldt)o(yp)q(e)h Fx(ha)o(v)o(e)f(t)o(yp)q(e)g (map)g Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)p Fp(;)k Fx(with)i(exten)o(t) g(16.)19 b(Let)14 b Fv(B)h(=)75 1711 y(\(3,)e(1\))h Fx(and)g(let)h Fv(D)f(=)g(\(4,)f(0\))p Fx(.)19 b(A)14 b(call)i(to)d Fv(MPI)p 855 1711 14 2 v 16 w(TYPE)p 988 1711 V 17 w(INDEXED\(2,)g(B,)h (D,)f(oldt)o(yp)q(e,)j(newt)o(yp)q(e\))f Fx(returns)75 1768 y(a)g(datat)o(yp)q(e)g(with)g(t)o(yp)q(e)g(map,)189 1862 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(64\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(72\))p Fp(;)g Fx(\()p Fv(double)n Fp(;)g Fx(80\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(88\))p Fp(;)g Fx(\()p Fv(double)n Fp(;)g Fx(96\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(104\))p Fp(;)189 1968 y Fx(\()p Fv(double)q Fp(;)g Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(8\))p Ft(g)p Fp(:)75 2074 y Fx(That)15 b(is,)i(three)f(copies)g(of)g(the)g(old)g(t)o(yp)q(e)g(starting)g(at)f (displacemen)o(t)i(64,)e(and)h(one)h(cop)o(y)e(starting)h(at)75 2131 y(displacemen)o(t)h(0.)166 2320 y(In)f(general,)f(assume)g(that)g Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g(map,)189 2413 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 2420 y Fg(0)333 2413 y Fp(;)8 b(disp)438 2420 y Fg(0)457 2413 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 2420 y Ff(n)p Fh(\000)p Fg(1)723 2413 y Fp(;)h(disp)828 2420 y Ff(n)p Fh(\000)p Fg(1)895 2413 y Fx(\))p Ft(g)p Fp(;)75 2507 y Fx(with)41 b(exten)o(t)f Fl(ex)p Fx(.)96 b(Let)41 b Fv(B)g Fx(b)q(e)g(the)g Fv(a)o(rra)o(y)p 986 2507 V 14 w(of)p 1037 2507 V 16 w(blo)q(cklength)i Fx(argumen)o(t)d(and)g Fv(D)h Fx(b)q(e)g(the)75 2564 y Fv(a)o(rra)o(y)p 173 2564 V 15 w(of)p 225 2564 V 16 w(displacements)15 b Fx(argumen)o(t.)k(The)c(newly)h(created)f(datat)o (yp)q(e)g(has)g Fp(n)c Ft(\001)1450 2532 y Fe(P)1494 2545 y Fi(count)q Fh(\000)p Fg(1)1494 2577 y Ff(i)p Fg(=0)1646 2564 y Fv(B)p Fx([)p Fv(i)p Fx(])j(en)o(tries:)189 2657 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 2664 y Fg(0)333 2657 y Fp(;)8 b(disp)438 2664 y Fg(0)467 2657 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)874 2664 y Ff(n)p Fh(\000)p Fg(1)939 2657 y Fp(;)g(disp)1044 2664 y Ff(n)p Fh(\000)p Fg(1)1122 2657 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 65 73 65 72 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(65)189 45 y(\()p Fp(ty)r(pe)291 52 y Fg(0)310 45 y Fp(;)8 b(disp)415 52 y Fg(0)444 45 y Fx(+)i(\()p Fv(D)p Fx([)p Fv(0)p Fx(])f(+)h Fv(B)p Fx([)p Fv(0)p Fx(])g Ft(\000)g Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(:::;)g Fx(\()p Fp(ty)r(pe)1098 52 y Ff(n)p Fh(\000)p Fg(1)1164 45 y Fp(;)g(disp)1269 52 y Ff(n)p Fh(\000)p Fg(1)1347 45 y Fx(+)i(\()p Fv(D)p Fx([)p Fv(0)p Fx(])f(+)h Fv(B)p Fx([)p Fv(0)p Fx(])f Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 151 y Fx(\()p Fp(ty)r(pe)291 158 y Fg(0)310 151 y Fp(;)g(disp)415 158 y Fg(0)444 151 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)1013 158 y Ff(n)p Fh(\000)p Fg(1)1079 151 y Fp(;)g(disp)1184 158 y Ff(n)p Fh(\000)p Fg(1)1261 151 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 258 y Fx(\()p Fp(ty)r(pe)291 265 y Fg(0)310 258 y Fp(;)g(disp)415 265 y Fg(0)444 258 y Fx(+)i(\()p Fv(D)p Fx([)p Fv(count)h Ft(\000)f Fv(1)p Fx(])g(+)g Fv(B)p Fx([)p Fv(count)i Ft(\000)e Fv(1)p Fx(])g Ft(\000)g Fx(1\))f Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 364 y Fx(\()p Fp(ty)r(pe)291 371 y Ff(n)p Fh(\000)p Fg(1)359 364 y Fp(;)g(disp)464 371 y Ff(n)p Fh(\000)p Fg(1)542 364 y Fx(+)i(\()p Fv(D)p Fx([)p Fv(count)h Ft(\000)f Fv(1)p Fx(])g(+)g Fv(B)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Ft(g)p Fp(:)166 553 y Fx(A)j(call)g(to)f Fv(MPI)p 431 553 14 2 v 16 w(TYPE)p 564 553 V 17 w(VECTOR\(count,)j(blo)q(cklength,) g(stride,)e(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))g Fx(is)f(equiv)m(alen)o (t)75 609 y(to)i(a)f(call)j(to)d Fv(MPI)p 392 609 V 16 w(TYPE)p 525 609 V 17 w(INDEXED\(count,)i(B,)f(D,)f(oldt)o(yp)q(e,)i (newt)o(yp)q(e\))h Fx(where)189 703 y Fv(D)p Fx([)p Fv(j)p Fx(])11 b(=)i Fp(j)g Ft(\001)d Fv(stride)q Fp(;)22 b(j)15 b Fx(=)e(0)p Fp(;)8 b(:::;)g Fv(count)g Ft(\000)j Fx(1)p Fp(;)75 797 y Fx(and)189 891 y Fv(B)p Fx([)p Fv(j)p Fx(])h(=)h Fv(blo)q(cklength)r Fp(;)22 b(j)15 b Fx(=)e(0)p Fp(;)8 b(:::;)g Fv(count)h Ft(\000)h Fx(1)p Fp(:)75 1011 y Fv(Hindexed)47 b Fx(The)11 b(function)h Fv(MPI)p 643 1011 V 15 w(TYPE)p 775 1011 V 17 w(HINDEXED)f Fx(is)g(iden)o(tical)i(to)d Fv(MPI)p 1383 1011 V 15 w(TYPE)p 1515 1011 V 17 w(INDEXED)p Fx(,)g(except)75 1067 y(that)21 b(blo)q(c)o(k)i(displacemen)o(ts)g(in)g Fv(a)o(rra)o(y)p 761 1067 V 14 w(of)p 812 1067 V 16 w(displacements)f Fx(are)g(sp)q(eci\014ed)i(in)e(b)o(ytes,)h(rather)e(than)h(in)75 1124 y(m)o(ultiples)17 b(of)e(the)g Fv(oldt)o(yp)q(e)h Fx(exten)o(t.)75 1275 y Fv(MPI)p 160 1275 V 16 w(TYPE)p 293 1275 V 17 w(HINDEXED\()21 b(count,)k(a)o(rra)o(y)p 814 1275 V 14 w(of)p 865 1275 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1248 1275 V 15 w(of)p 1300 1275 V 16 w(displacements,)d(oldt)o(yp)q(e,) h(new-)75 1331 y(t)o(yp)q(e\))117 1408 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f (n)o(um)o(b)q(er)h(of)f(en)o(tries)i(in)905 1465 y Fi(a)o(rra)o(y)p 992 1465 13 2 v 15 w(of)p 1041 1465 V 15 w(displacements)17 b Fk(and)h Fi(a)o(rra)o(y)p 1483 1465 V 15 w(of)p 1532 1465 V 14 w(blo)q(cklengths)i Fk(\(in)o(te-)905 1521 y(ger\))117 1596 y(IN)155 b Fv(a)o(rra)o(y)p 416 1596 14 2 v 15 w(of)p 468 1596 V 16 w(blo)q(cklengths)191 b Fk(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)h(in)f(eac)o(h)h(blo)q(c)o (k)g(\(arra)o(y)f(of)g(nonnega-)905 1653 y(tiv)o(e)f(in)o(tegers\))117 1728 y(IN)155 b Fv(a)o(rra)o(y)p 416 1728 V 15 w(of)p 468 1728 V 16 w(displacements)162 b Fk(b)o(yte)14 b(displacemen)o(t)g (of)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1803 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 1878 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2003 y Fu(int)23 b(MPI)p 245 2003 15 2 v 17 w(Type)p 358 2003 V 17 w(hindexed\(int)f (count,)h(int)h(*array)p 1092 2003 V 16 w(of)p 1156 2003 V 17 w(blocklengths,)393 2059 y(MPI)p 468 2059 V 17 w(Aint)f(*array)p 748 2059 V 17 w(of)p 813 2059 V 17 w(displacements,)e(MPI)p 1259 2059 V 17 w(Datatype)i(oldtype,)393 2116 y(MPI)p 468 2116 V 17 w(Datatype)g(*newtype\))75 2202 y(MPI)p 150 2202 V 17 w(TYPE)p 263 2202 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 2202 V 17 w(OF)p 846 2202 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 2202 V 16 w(OF)p 1381 2202 V 17 w(DISPLACEMENTS,)393 2259 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 2315 y(INTEGER)g(COUNT,)g (ARRAY)p 651 2315 V 17 w(OF)p 716 2315 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 2315 V 17 w(OF)p 1323 2315 V 17 w(DISPLACEMENTS\(*\),)170 2371 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)166 2541 y Fx(Assume)15 b(that)g Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g(map,)189 2634 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 2641 y Fg(0)333 2634 y Fp(;)8 b(disp)438 2641 y Fg(0)457 2634 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 2641 y Ff(n)p Fh(\000)p Fg(1)723 2634 y Fp(;)h(disp)828 2641 y Ff(n)p Fh(\000)p Fg(1)895 2634 y Fx(\))p Ft(g)p Fp(;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 66 74 66 73 bop 75 -100 a Fx(66)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(with)41 b(exten)o(t)e Fp(ex)p Fx(.)95 b(Let)41 b Fv(B)f Fx(b)q(e)h(the)g Fv(a)o(rra)o(y)p 988 45 14 2 v 14 w(of)p 1039 45 V 16 w(blo)q(cklength)h Fx(argumen)o(t)e(and)g Fv(D)g Fx(b)q(e)h(the)75 102 y Fv(a)o(rra)o(y)p 173 102 V 15 w(of)p 225 102 V 16 w(displacements)21 b Fx(argumen)o(t.)39 b(The)22 b(newly)h(created)f (datat)o(yp)q(e)f(has)h(a)f(t)o(yp)q(e)h(map)f(with)i Fp(n)14 b Ft(\001)75 126 y Fe(P)119 139 y Fi(count)q Fh(\000)p Fg(1)119 171 y Ff(i)p Fg(=0)271 158 y Fv(B)p Fx([)p Fv(i)p Fx(])g(en)o(tries:)189 251 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 258 y Fg(0)333 251 y Fp(;)8 b(disp)438 258 y Fg(0)467 251 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])o(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)794 258 y Ff(n)p Fh(\000)p Fg(1)859 251 y Fp(;)g(disp)964 258 y Ff(n)p Fh(\000)p Fg(1)1042 251 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])o(\))p Fp(;)e(:::;)189 355 y Fx(\()p Fp(ty)r(pe)291 362 y Fg(0)310 355 y Fp(;)g(disp)415 362 y Fg(0)444 355 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f(+)i(\()p Fv(B)p Fx([)p Fv(0)p Fx(])e Ft(\000)h Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(:::;)189 460 y Fx(\()p Fp(ty)r(pe)291 467 y Ff(n)p Fh(\000)p Fg(1)359 460 y Fp(;)g(disp)464 467 y Ff(n)p Fh(\000)p Fg(1)542 460 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f(+)h(\()p Fv(B)p Fx([)p Fv(0)p Fx(])f Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 564 y Fx(\()p Fp(ty)r(pe)291 571 y Fg(0)310 564 y Fp(;)g(disp)415 571 y Fg(0)444 564 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])o(\))p Fp(;)d(:::;)g Fx(\()p Fp(ty)r(pe)933 571 y Ff(n)p Fh(\000)p Fg(1)999 564 y Fp(;)g(disp)1104 571 y Ff(n)p Fh(\000)p Fg(1)1181 564 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])o(\))p Fp(;)d(:::;)189 669 y Fx(\()p Fp(ty)r(pe)291 676 y Fg(0)310 669 y Fp(;)g(disp)415 676 y Fg(0)444 669 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e(+)i(\()p Fv(B)p Fx([)p Fv(count)g Ft(\000)f Fv(1)p Fx(])g Ft(\000)g Fx(1\))f Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 774 y Fx(\()p Fp(ty)r(pe)291 781 y Ff(n)p Fh(\000)p Fg(1)359 774 y Fp(;)g(disp)464 781 y Ff(n)p Fh(\000)p Fg(1)542 774 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)f Fv(1)p Fx(])g(+)g(\()p Fv(B)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Ft(g)p Fp(:)75 1024 y Fv(Struct)47 b(MPI)p 323 1024 V 16 w(TYPE)p 456 1024 V 16 w(STRUCT)20 b Fx(is)g(the)g(most)f(general)g(t)o(yp)q(e)h (constructor.)32 b(It)19 b(further)g(generalizes)75 1081 y(the)13 b(previous)h(one)f(in)h(that)e(it)h(allo)o(ws)g(eac)o(h)g(blo) q(c)o(k)h(to)e(consist)h(of)g(replications)h(of)f(di\013eren)o(t)g (datat)o(yp)q(es.)75 1232 y Fv(MPI)p 160 1232 V 16 w(TYPE)p 293 1232 V 17 w(STRUCT\(count,)f(a)o(rra)o(y)p 729 1232 V 15 w(of)p 781 1232 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1151 1232 V 14 w(of)p 1202 1232 V 16 w(displacements,)e(a)o(rra)o(y)p 1599 1232 V 15 w(of)p 1651 1232 V 15 w(t)o(yp)q(es,)i(new-)75 1288 y(t)o(yp)q(e\))117 1365 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)18 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))g({)g (also)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)905 1422 y(in)c(arra)o(ys)h Fi(a)o(rra)o(y)p 1167 1422 13 2 v 15 w(of)p 1216 1422 V 14 w(t)o(yp)q(es)p Fk(,)h Fi(a)o(rra)o(y)p 1432 1422 V 15 w(of)p 1481 1422 V 14 w(displacements)e Fk(and)g Fi(a)o(r-)905 1478 y(ra)o(y)p 959 1478 V 15 w(of)p 1008 1478 V 15 w(blo)q(cklengths)117 1553 y Fk(IN)155 b Fv(a)o(rra)o(y)p 416 1553 14 2 v 15 w(of)p 468 1553 V 16 w(blo)q(cklength)208 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o (k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1628 y(IN)155 b Fv(a)o(rra)o(y)p 416 1628 V 15 w(of)p 468 1628 V 16 w(displacements)162 b Fk(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h (blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1703 y(IN)155 b Fv(a)o(rra)o(y)p 416 1703 V 15 w(of)p 468 1703 V 16 w(t)o(yp)q(es)327 b Fk(t)o(yp)q(e)20 b(of)f(elemen)o(ts)g(in) g(eac)o(h)h(blo)q(c)o(k)f(\(arra)o(y)g(of)g(handles)g(to)905 1759 y(datat)o(yp)q(e)14 b(ob)r(jects\))117 1834 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1959 y Fu(int)23 b(MPI)p 245 1959 15 2 v 17 w(Type)p 358 1959 V 17 w(struct\(int)f(count,)h(int)h(*array)p 1044 1959 V 16 w(of)p 1108 1959 V 17 w(blocklengths,)393 2015 y(MPI)p 468 2015 V 17 w(Aint)f(*array)p 748 2015 V 17 w(of)p 813 2015 V 17 w(displacements,)e(MPI)p 1259 2015 V 17 w(Datatype)i(*array)p 1635 2015 V 16 w(of)p 1699 2015 V 17 w(types,)393 2072 y(MPI)p 468 2072 V 17 w(Datatype)g(*newtype\))75 2158 y(MPI)p 150 2158 V 17 w(TYPE)p 263 2158 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 2158 V 17 w(OF)p 798 2158 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 2158 V 16 w(OF)p 1333 2158 V 17 w(DISPLACEMENTS,)393 2214 y(ARRAY)p 516 2214 V 17 w(OF)p 581 2214 V 17 w(TYPES,)h(NEWTYPE,)f (IERROR\))170 2271 y(INTEGER)h(COUNT,)g(ARRAY)p 651 2271 V 17 w(OF)p 716 2271 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 2271 V 17 w(OF)p 1323 2271 V 17 w(DISPLACEMENTS\(*\),)170 2327 y(ARRAY)p 293 2327 V 17 w(OF)p 358 2327 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)75 2462 y Fq(Example)18 b(3.23)23 b Fx(Let)15 b Fv(t)o(yp)q(e1)h Fx(ha)o(v)o(e)f(t)o(yp)q(e)g(map,)189 2555 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)p Fp(;)75 2647 y Fx(with)k(exten)o(t)f(16.)18 b(Let)11 b Fv(B)h(=)g(\(2,)f(1,)g(3\))p Fx(,)g Fv(D)h(=)f(\(0,)g(16,)h(26\))p Fx(,)e(and)i Fv(T)f(=)h(\(MPI)p 1321 2647 14 2 v 15 w(FLO)o(A)l(T,)f(t)o(yp)q(e1,)i(MPI)p 1711 2647 V 16 w(CHAR\))p Fx(.)75 2704 y(Then)h(a)f(call)h(to)e Fv(MPI)p 447 2704 V 16 w(TYPE)p 580 2704 V 17 w(STRUCT\(3,)h(B,)g(D,)g (T,)g(newt)o(yp)q(e\))h Fx(returns)f(a)g(datat)o(yp)q(e)g(with)g(t)o (yp)q(e)h(map,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 67 75 67 74 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(67)189 45 y Ft(f)p Fx(\()p Fv(\015oat)o Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(\015oat)n Fp(;)g Fx(4\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(16\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(24\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(26\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(27\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(28\))p Ft(g)p Fp(:)75 128 y Fx(That)16 b(is,)h(t)o(w)o(o)e(copies)i(of)f Fv(MPI)p 610 128 14 2 v 15 w(FLO)o(A)l(T)h Fx(starting)f(at)f(0,)h(follo)o(w)o (ed)h(b)o(y)f(one)h(cop)o(y)f(of)g Fv(t)o(yp)q(e1)h Fx(starting)f(at)75 185 y(16,)d(follo)o(w)o(ed)h(b)o(y)f(three)h(copies)h(of)e Fv(MPI)p 765 185 V 16 w(CHAR)p Fx(,)h(starting)f(at)g(26.)18 b(\(W)l(e)c(assume)f(that)g(a)g(\015oat)g(o)q(ccupies)75 241 y(four)i(b)o(ytes.\))166 416 y(In)h(general,)f(let)h Fv(T)f Fx(b)q(e)h(the)f Fv(a)o(rra)o(y)p 742 416 V 14 w(of)p 793 416 V 16 w(t)o(yp)q(es)i Fx(argumen)o(t,)d(where)i Fv(T[i])f Fx(is)g(a)g(handle)i(to,)189 499 y Fp(ty)r(pemap)360 506 y Ff(i)387 499 y Fx(=)c Ft(f)p Fx(\()p Fp(ty)r(pe)560 480 y Ff(i)560 510 y Fg(0)579 499 y Fp(;)8 b(disp)684 480 y Ff(i)684 510 y Fg(0)702 499 y Fx(\))p Fp(;)g(:::;)g Fx(\()p Fp(ty)r(pe)903 480 y Ff(i)903 510 y(n)924 515 y Fd(i)935 510 y Fh(\000)p Fg(1)982 499 y Fp(;)g(disp)1087 480 y Ff(i)1087 510 y(n)1108 515 y Fd(i)1120 510 y Fh(\000)p Fg(1)1168 499 y Fx(\))p Ft(g)p Fp(;)75 582 y Fx(with)i(exten)o(t)g Fp(ex)354 589 y Ff(i)369 582 y Fx(.)18 b(Let)10 b Fv(B)g Fx(b)q(e)h(the)g Fv(a)o(rra)o(y)p 745 582 V 14 w(of)p 796 582 V 16 w(blo)q(cklength)h Fx(argumen)o(t)d(and)i Fv(D)f Fx(b)q(e)h(the)f Fv(a)o(rra)o(y)p 1590 582 V 14 w(of)p 1641 582 V 16 w(displacements)75 639 y Fx(argumen)o(t.)19 b(Let)c Fv(c)g Fx(b)q(e)h(the)f Fv(count)h Fx(argumen)o(t.)j(Then)c (the)g(newly)h(created)f(datat)o(yp)q(e)f(has)h(a)f(t)o(yp)q(e)h(map)75 695 y(with)179 663 y Fe(P)222 676 y Fi(c)p Fh(\000)p Fg(1)222 708 y Ff(i)p Fg(=0)296 695 y Fv(B)p Fx([)p Fv(i)p Fx(])9 b Ft(\001)h Fp(n)422 702 y Ff(i)451 695 y Fx(en)o(tries:)189 779 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 760 y Fg(0)314 790 y(0)333 779 y Fp(;)e(disp)438 760 y Fg(0)438 790 y(0)467 779 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])o(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)794 760 y Fg(0)794 790 y Ff(n)815 795 y Fs(0)832 779 y Fp(;)g(disp)937 760 y Fg(0)937 790 y Ff(n)958 795 y Fs(0)986 779 y Fx(+)j Fv(D)p Fx([)p Fv(0)p Fx(])o(\))p Fp(;)d(:::;)189 870 y Fx(\()p Fp(ty)r(pe)291 851 y Fg(0)291 881 y(0)310 870 y Fp(;)g(disp)415 851 y Fg(0)415 881 y(0)444 870 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f(+)i(\()p Fv(B)p Fx([)p Fv(0)p Fx(])e Ft(\000)h Fx(1\))g Ft(\001)f Fp(ex)897 877 y Fg(0)917 870 y Fx(\))p Fp(;)f(:::;)g Fx(\()p Fp(ty)r(pe)1118 851 y Fg(0)1118 881 y Ff(n)1139 886 y Fs(0)1156 870 y Fp(;)g(disp)1261 851 y Fg(0)1261 881 y Ff(n)1282 886 y Fs(0)1311 870 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f(+)i(\()p Fv(B)p Fx([)p Fv(0)p Fx(])e Ft(\000)h Fv(1)p Fx(\))g Ft(\001)f Fp(ex)1764 877 y Fg(0)1784 870 y Fx(\))p Fp(;)f(:::;)189 962 y Fx(\()p Fp(ty)r(pe)291 943 y Fi(c)p Fh(\000)p Fg(1)291 974 y(0)356 962 y Fp(;)g(disp)461 943 y Fi(c)p Fh(\000)p Fg(1)461 974 y(0)536 962 y Fx(+)i Fv(D)p Fx([)p Fv(c)g Ft(\000)g Fv(1)p Fx(]\))p Fp(;)e(:::;)g Fx(\()o Fp(ty)r(pe)938 943 y Fi(c)p Fh(\000)p Fg(1)938 974 y Ff(n)959 981 y Fi(c)p Fc(\000)p Fs(1)1017 974 y Fh(\000)p Fg(1)1064 962 y Fp(;)g(disp)1169 943 y Fi(c)p Fh(\000)p Fg(1)1169 974 y Ff(n)1190 981 y Fi(c)p Fc(\000)p Fs(1)1249 974 y Fh(\000)p Fg(1)1306 962 y Fx(+)j Fv(D)p Fx([)p Fv(c)e Ft(\000)i Fv(1)p Fx(])o(\))p Fp(;)d(:::;)189 1064 y Fx(\()p Fp(ty)r(pe)291 1045 y Fi(c)p Fh(\000)p Fg(1)291 1077 y(0)356 1064 y Fp(;)g(disp)461 1045 y Fi(c)p Fh(\000)p Fg(1)461 1077 y(0)536 1064 y Fx(+)i Fv(D)p Fx([)p Fv(c)g Ft(\000)g Fv(1)p Fx(])g(+)g(\()p Fv(B)p Fx([)p Fv(c)g Ft(\000)h Fv(1)p Fx(])e Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)1141 1071 y Fi(c)p Fh(\000)p Fg(1)1207 1064 y Fx(\))p Fp(;)e(:::;)189 1156 y Fx(\()p Fp(ty)r(pe)291 1137 y Fi(c)p Fh(\000)p Fg(1)291 1168 y Ff(n)312 1175 y Fi(c)p Fc(\000)p Fs(1)372 1168 y Fh(\000)p Fg(1)419 1156 y Fp(;)g(disp)524 1137 y Fi(c)p Fh(\000)p Fg(1)524 1168 y Ff(n)545 1175 y Fi(c)p Fc(\000)p Fs(1)604 1168 y Fh(\000)p Fg(1)661 1156 y Fx(+)j Fv(D)p Fx([)p Fv(c)e Ft(\000)i Fv(1)p Fx(])e(+)i(\()p Fv(B)p Fx([)p Fv(c)f Ft(\000)g Fv(1)p Fx(])f Ft(\000)i Fv(1)p Fx(\))e Ft(\001)h Fp(ex)1266 1163 y Fi(c)p Fh(\000)p Fg(1)1332 1156 y Fx(\))p Ft(g)p Fp(:)166 1335 y Fx(A)15 b(call)h(to)f Fv(MPI)p 439 1335 V 16 w(TYPE)p 572 1335 V 16 w(HINDEXED\()g(count,)h(B,)f(D,)g(oldt)o(yp)q(e,)h(newt)o(yp)q (e\))g Fx(is)g(equiv)m(alen)o(t)h(to)d(a)h(call)75 1391 y(to)d Fv(MPI)p 213 1391 V 16 w(TYPE)p 346 1391 V 17 w(STRUCT\()h(count,)i(B,)e(D,)f(T,)h(newt)o(yp)q(e\))p Fx(,)h(where)f(eac)o(h)g(en)o(try)g(of)g Fv(T)g Fx(is)g(equal)h(to)e Fv(oldt)o(yp)q(e)p Fx(.)75 1511 y Fj(3.12.2)49 b(Address)16 b(and)g(extent)f(functions)75 1597 y Fx(The)g(displacemen)o(ts)i(in)e (a)g(general)h(datat)o(yp)q(e)e(are)h(relativ)o(e)g(to)g(some)f (initial)j(bu\013er)e(address.)20 b Fq(Abso-)75 1653 y(lute)15 b(addresses)c Fx(can)h(b)q(e)h(substituted)f(for)g(these)g (displacemen)o(ts:)20 b(w)o(e)12 b(treat)f(them)h(as)g(displacemen)o (ts)75 1710 y(relativ)o(e)17 b(to)e(\\address)h(zero,")g(the)g(start)f (of)h(the)g(address)h(space.)23 b(This)17 b(initial)h(address)e(zero)g (is)h(indi-)75 1766 y(cated)f(b)o(y)g(the)g(constan)o(t)f Fi(MPI)p 601 1766 13 2 v 14 w(BOTTOM)p Fx(.)g(Th)o(us,)g(a)h(datat)o (yp)q(e)f(can)h(sp)q(ecify)h(the)f(absolute)g(address)g(of)75 1823 y(the)e(en)o(tries)g(in)g(the)g(comm)o(unication)g(bu\013er,)g(in) h(whic)o(h)f(case)g(the)g Fv(buf)g Fx(argumen)o(t)f(is)h(passed)g(the)g (v)m(alue)75 1879 y Fi(MPI)p 152 1879 V 14 w(BOTTOM)p Fx(.)166 1936 y(The)31 b(address)g(of)f(a)h(lo)q(cation)g(in)h(memory)e (can)h(b)q(e)h(found)f(b)o(y)g(in)o(v)o(oking)g(the)g(function)75 1992 y Fv(MPI)p 160 1992 14 2 v 16 w(ADDRESS)p Fx(.)75 2143 y Fv(MPI)p 160 2143 V 16 w(ADDRESS\(lo)q(cation,)16 b(address\))117 2220 y Fk(IN)155 b Fv(lo)q(cation)437 b Fk(lo)q(cation)13 b(in)h(caller)f(memory)f(\(c)o(hoice\))117 2293 y(OUT)108 b Fv(address)449 b Fk(address)15 b(of)f(lo)q(cation)f (\(in)o(teger\))75 2418 y Fu(int)23 b(MPI)p 245 2418 15 2 v 17 w(Address\(void*)f(location,)h(MPI)p 907 2418 V 17 w(Aint)g(*address\))75 2504 y(MPI)p 150 2504 V 17 w(ADDRESS\(LOCATION,)e(ADDRESS,)i(IERROR\))170 2561 y()g (LOCATION\(*\))170 2617 y(INTEGER)g(ADDRESS,)g(IERROR)166 2704 y Fx(Returns)16 b(the)f(\(b)o(yte\))f(address)i(of)e Fv(lo)q(cation)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 68 76 68 75 bop 75 -100 a Fx(68)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fq(Example)h(3.24)23 b Fx(Using)15 b Fv(MPI)p 619 45 14 2 v 16 w(ADDRESS)h Fx(for)f(an)g(arra)o(y)l(.)147 151 y Fu(REAL)23 b(A\(100,100\))147 208 y(INTEGER)f(I1,)i(I2,)f(DIFF)147 264 y(CALL)g (MPI_ADDRESS\(A\(1,1\),)e(I1,)j(IERROR\))147 321 y(CALL)f (MPI_ADDRESS\(A\(10,10\),)e(I2,)i(IERROR\))147 377 y(DIFF)g(=)h(I2)f(-) h(I1)75 434 y(!)g(The)f(value)g(of)h(DIFF)f(is)h(909*sizeofreal;)e(the) h(values)g(of)h(I1)f(and)h(I2)f(are)75 490 y(!)h(implementation)e (dependent.)189 596 y Fl(A)n(dvic)n(e)e(to)i(users.)73 b Fx(C)20 b(users)h(ma)o(y)f(b)q(e)h(tempted)g(to)f(a)o(v)o(oid)h(the)f (usage)h(of)f Fv(MPI)p 1659 596 V 16 w(ADDRESS)189 653 y Fx(and)e(rely)g(on)g(the)g(a)o(v)m(ailabilit)o(y)i(of)d(the)h (address)g(op)q(erator)f(&.)29 b(Note,)17 b(ho)o(w)o(ev)o(er,)h(that)f Fi(&)h Fl(c)n(ast-)189 709 y(expr)n(ession)c Fx(is)j(a)e(p)q(oin)o (ter,)i(not)e(an)h(address.)22 b(ANSI)17 b(C)f(do)q(es)g(not)g(require) g(that)g(the)g(v)m(alue)h(of)e(a)189 766 y(p)q(oin)o(ter)f(\(or)f(the)h (p)q(oin)o(ter)g(cast)g(to)f Fi(int)p Fx(\))h(b)q(e)g(the)g(absolute)g (address)g(of)f(the)h(ob)s(ject)g(p)q(oin)o(ted)g(at)g(|)189 822 y(although)j(this)g(is)h(the)f(common)g(case.)25 b(F)l(urthermore,)17 b(referencing)h(ma)o(y)e(not)h(ha)o(v)o(e)g(a)f (unique)189 879 y(de\014nition)j(on)f(mac)o(hines)h(with)g(a)e(segmen)o (ted)h(address)g(space.)29 b(The)18 b(use)h(of)e Fv(MPI)p 1659 879 V 16 w(ADDRESS)189 935 y Fx(to)g(\\reference")h(C)f(v)m (ariables)i(guaran)o(tees)e(p)q(ortabilit)o(y)i(to)e(suc)o(h)h(mac)o (hines)g(as)g(w)o(ell.)28 b(\()p Fl(End)18 b(of)189 992 y(advic)n(e)e(to)g(users.)p Fx(\))166 1098 y(The)f(follo)o(wing)h (auxiliary)h(functions)f(pro)o(vide)f(useful)i(information)e(on)g (deriv)o(ed)h(datat)o(yp)q(es.)75 1249 y Fv(MPI)p 160 1249 V 16 w(TYPE)p 293 1249 V 17 w(EXTENT\(datat)o(yp)q(e,)g(extent\)) 117 1326 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 1401 y(OUT)108 b Fv(extent)471 b Fk(datat)o(yp)q(e)14 b(exten)o(t)h(\(in)o(teger\))75 1526 y Fu(int)23 b(MPI)p 245 1526 15 2 v 17 w(Type)p 358 1526 V 17 w(extent\(MPI)p 615 1526 V 16 w(Datatype)f(datatype,)h(MPI)p 1156 1526 V 17 w(Aint)g(*extent\))75 1612 y(MPI)p 150 1612 V 17 w(TYPE)p 263 1612 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h(IERROR\))170 1668 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)166 1755 y Fx(Returns)16 b(the)f(exten)o(t)g(of)g(a)f(datat)o(yp)q(e,)h(where)g (exten)o(t)g(is)h(as)f(de\014ned)h(in)g(Eq.)f(3.1)f(on)h(page)g(60.)75 1906 y Fv(MPI)p 160 1906 14 2 v 16 w(TYPE)p 293 1906 V 17 w(SIZE\(datat)o(yp)q(e,)g(size\))117 1983 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 2058 y(OUT)108 b Fv(size)519 b Fk(datat)o(yp)q(e)14 b(size)h(\(in)o (teger\))75 2183 y Fu(int)23 b(MPI)p 245 2183 15 2 v 17 w(Type)p 358 2183 V 17 w(size\(MPI)p 567 2183 V 16 w(Datatype)g(datatype,)f(MPI)p 1108 2183 V 17 w(Aint)h(*size\))75 2269 y(MPI)p 150 2269 V 17 w(TYPE)p 263 2269 V 16 w(SIZE\(DATATYPE,)f (SIZE,)i(IERROR\))170 2326 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)166 2412 y Fx(Returns)17 b(the)g(n)o(um)o(b)q(er)h(of)e(b)o(ytes)h(o)q (ccupied)i(b)o(y)d(en)o(tries)i(in)g(the)f(datat)o(yp)q(e)f(|)h(that)f (is,)i(the)f(sum)75 2468 y(of)e(the)g(sizes)h(of)f(the)g(datat)o(yp)q (e)g(elemen)o(ts.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 69 77 69 76 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(69)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(COUNT\(datat)o(yp)q(e,)17 b(count\))117 122 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 197 y(OUT)108 b Fv(count)482 b Fk(datat)o(yp)q(e)14 b(coun)o(t)g(\(in)o(teger\))75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Type)p 358 322 V 17 w(count\(MPI)p 591 322 V 16 w(Datatype)g(datatype,)f(int)i(*count\))75 408 y(MPI)p 150 408 V 17 w(TYPE)p 263 408 V 16 w(COUNT\(DATATYPE,)e (COUNT,)h(IERROR\))170 465 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)166 551 y Fx(Returns)16 b(the)f(n)o(um)o(b)q(er)g(of)g(\\top-lev)o(el")h (en)o(tries)g(in)g(the)f(datat)o(yp)q(e.)75 673 y Fj(3.12.3)49 b(Lo)o(w)o(er-b)q(ound)17 b(and)g(upp)q(er-b)q(ound)g(ma)o(rk)n(ers)75 759 y Fx(It)e(is)g(often)g(con)o(v)o(enien)o(t)g(to)g(de\014ne)h (explicitly)h(the)e(lo)o(w)o(er)g(b)q(ound)h(and)f(upp)q(er)h(b)q(ound) f(of)g(a)g(t)o(yp)q(e)g(map,)75 815 y(and)i(o)o(v)o(erride)g(the)f (de\014nition)j(giv)o(en)e(b)o(y)g(Equation)g(3.1)e(on)i(page)g(60.)23 b(This)18 b(allo)o(ws)f(one)f(to)g(de\014ne)i(a)75 872 y(datat)o(yp)q(e)12 b(that)g(has)h(\\holes")g(at)f(its)h(b)q(eginning)i (or)d(its)h(end,)g(or)g(a)f(datat)o(yp)q(e)g(with)h(en)o(tries)g(that)f (extend)75 928 y(ab)q(o)o(v)o(e)18 b(the)g(upp)q(er)i(b)q(ound)f(or)f (b)q(elo)o(w)h(the)f(lo)o(w)o(er)g(b)q(ound.)31 b(Examples)18 b(of)g(suc)o(h)h(usage)f(are)g(pro)o(vided)75 985 y(in)i(Sec.)31 b(3.12.7.)d(T)l(o)19 b(ac)o(hiev)o(e)g(this,)h(w)o(e)e(add)h(t)o(w)o(o) e(additional)k(\\pseudo-datat)o(yp)q(es,")d Fv(MPI)p 1715 985 14 2 v 16 w(LB)h Fx(and)75 1041 y Fv(MPI)p 160 1041 V 16 w(UB)p Fx(,)d(that)e(can)i(b)q(e)h(used,)e(resp)q(ectiv)o (ely)l(,)j(to)d(mark)g(the)g(lo)o(w)o(er)h(b)q(ound)g(or)f(the)h(upp)q (er)h(b)q(ound)f(of)f(a)75 1098 y(datat)o(yp)q(e.)k(These)c (pseudo-datat)o(yp)q(es)g(o)q(ccup)o(y)g(no)f(space)h(\()p Fp(extent)p Fx(\()p Fv(MPI)p 1357 1098 V 17 w(LB)p Fx(\))d(=)h Fp(extent)p Fx(\()p Fv(MPI)p 1734 1098 V 17 w(UB)p Fx(\))g(=)75 1154 y(0\).)18 b(They)c(do)f(not)f(a\013ect)g(the)h(size)h(or)f(coun)o (t)g(of)f(a)h(datat)o(yp)q(e,)f(and)h(do)g(not)g(a\013ect)f(the)h(the)g (con)o(ten)o(t)g(of)f(a)75 1210 y(message)i(created)h(with)h(this)f (datat)o(yp)q(e.)k(Ho)o(w)o(ev)o(er,)14 b(they)h(do)g(a\013ect)f(the)h (de\014nition)h(of)f(the)g(exten)o(t)f(of)75 1267 y(a)e(datat)o(yp)q(e) g(and,)h(therefore,)g(a\013ect)f(the)g(outcome)h(of)f(a)g(replication)i (of)f(this)g(datat)o(yp)q(e)f(b)o(y)g(a)g(datat)o(yp)q(e)75 1323 y(constructor.)75 1430 y Fq(Example)18 b(3.25)23 b Fx(Let)17 b Fv(D)f(=)h(\(-3,)g(0,)g(6\))p Fx(;)f Fv(T)h(=)g(\(MPI)p 1003 1430 V 16 w(LB,)f(MPI)p 1185 1430 V 16 w(INT,)g(MPI)p 1388 1430 V 16 w(UB\))p Fx(,)h(and)g Fv(B)g(=)g(\(1,)f(1,)h(1\))p Fx(.)75 1486 y(Then)g(a)e(call)i(to)f Fv(MPI)p 459 1486 V 16 w(TYPE)p 592 1486 V 16 w(STRUCT\(3,)g(B,)g(D,)g(T,)f(t)o(yp)q (e1\))i Fx(creates)e(a)h(new)g(datat)o(yp)q(e)g(that)f(has)h(an)75 1543 y(exten)o(t)h(of)g(9)g(\(from)f(-3)h(to)g(5,)g(5)g(included\),)i (and)f(con)o(tains)f(an)g(in)o(teger)h(at)e(displacemen)o(t)j(0.)26 b(This)17 b(is)75 1599 y(the)f(datat)o(yp)q(e)f(de\014ned)j(b)o(y)d (the)h(sequence)i Ft(f)p Fv(\(lb,)d(-3\),)g(\(int,)h(0\),)f(\(ub,)i (6\))p Ft(g)e Fx(.)22 b(If)16 b(this)g(t)o(yp)q(e)g(is)g(replicated)75 1655 y(t)o(wice)i(b)o(y)f(a)g(call)h(to)f Fv(MPI)p 528 1655 V 16 w(TYPE)p 661 1655 V 17 w(CONTIGUOUS\(2,)h(t)o(yp)q(e1,)g(t)o (yp)q(e2\))g Fx(then)g(the)f(newly)i(created)e(t)o(yp)q(e)75 1712 y(can)d(b)q(e)h(describ)q(ed)h(b)o(y)e(the)g(sequence)h Ft(f)p Fv(\(lb,)e(-3\),)h(\(int,)g(0\),)f(\(int,9\),)g(\(ub,)i(15\))p Ft(g)d Fx(.)20 b(\(En)o(tries)14 b(of)f(t)o(yp)q(e)h Fv(lb)g Fx(or)75 1768 y Fv(ub)i Fx(can)f(b)q(e)h(deleted)h(if)f(they)f (are)g(not)g(at)f(the)h(end-p)q(oin)o(ts)i(of)e(the)g(datat)o(yp)q (e.\))166 1875 y(In)h(general,)f(if)189 1968 y Fp(T)6 b(y)r(pemap)12 b Fx(=)h Ft(f)p Fx(\()p Fp(ty)r(pe)562 1975 y Fg(0)581 1968 y Fp(;)8 b(disp)686 1975 y Fg(0)705 1968 y Fx(\))p Fp(;)g(:::;)g Fx(\()o Fp(ty)r(pe)905 1975 y Ff(n)p Fh(\000)p Fg(1)971 1968 y Fp(;)g(disp)1076 1975 y Ff(n)p Fh(\000)p Fg(1)1144 1968 y Fx(\))p Ft(g)p Fp(;)75 2062 y Fx(then)16 b(the)f Fq(lo)o(w)o(er)h(b)q(ound)g Fx(of)f Fp(T)6 b(y)r(pemap)15 b Fx(is)g(de\014ned)i(to)e(b)q(e)189 2186 y Fp(l)q(b)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))k(=)506 2114 y Fe(\()560 2158 y Fx(min)636 2165 y Ff(j)662 2158 y Fp(disp)746 2165 y Ff(j)1257 2158 y Fx(if)16 b(no)f(en)o(try)f(has)i (basic)f(t)o(yp)q(e)h Fv(lb)560 2214 y Fx(min)636 2221 y Ff(j)654 2214 y Ft(f)p Fp(disp)761 2221 y Ff(j)794 2214 y Fx(suc)o(h)g(that)e Fp(ty)r(pe)1079 2221 y Ff(j)1110 2214 y Fx(=)f Fv(lb)q Ft(g)41 b Fx(otherwise)166 2310 y(Similarly)l(,)17 b(the)e Fq(upp)q(er)j(b)q(ound)e Fx(of)e Fp(T)6 b(y)r(pemap)15 b Fx(is)h(de\014ned)h(to)d(b)q(e)189 2434 y Fp(ub)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))11 b(=)518 2362 y Fe(\()572 2406 y Fx(max)657 2413 y Ff(j)682 2406 y Fp(disp)766 2413 y Ff(j)794 2406 y Fx(+)g Fp(siz)r(eof)5 b Fx(\()p Fp(ty)r(pe)1072 2413 y Ff(j)1090 2406 y Fx(\))182 b(if)16 b(no)f(en)o(try)f(has)i(basic)f(t)o(yp)q(e)h Fv(ub)572 2462 y Fx(max)657 2469 y Ff(j)675 2462 y Ft(f)p Fp(disp)782 2469 y Ff(j)814 2462 y Fx(suc)o(h)g(that)e Fp(ty)r(pe)1099 2469 y Ff(j)1131 2462 y Fx(=)f Fv(ub)q Ft(g)41 b Fx(otherwise)166 2558 y(Then)189 2652 y Fp(extent)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))13 b(=)g Fp(ub)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))i Ft(\000)j Fp(l)q(b)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))i(+)i Fp(\017)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 70 78 70 77 bop 75 -100 a Fx(70)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(If)i Fp(ty)r(pe)208 52 y Ff(i)242 45 y Fx(requires)h(alignmen)o(t)g(to)e(a)h (b)o(yte)g(address)g(that)g(is)h(a)e(m)o(ultiple)k(of)c Fp(k)1460 52 y Ff(i)1474 45 y Fx(,)i(then)f Fp(\017)h Fx(is)g(the)f(least)75 102 y(nonnegativ)o(e)d(incremen)o(t)g(needed)g (to)f(round)g Fp(extent)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))16 b(to)e(the)h(next)h(m)o(ultiple)h(of)d(max)1794 109 y Ff(i)1816 102 y Fp(k)1840 109 y Ff(i)1854 102 y Fx(.)166 158 y(The)g(formal)f(de\014nitions)j(giv)o(en)e(for)f(the)h(v) m(arious)g(datat)o(yp)q(e)f(constructors)g(apply)i(no)o(w,)e(with)h (the)75 214 y(amended)i(de\014nition)h(of)e Fq(exten)o(t)p Fx(.)166 271 y(The)f(t)o(w)o(o)e(functions)i(b)q(elo)o(w)h(can)e(b)q(e) i(used)f(for)f(\014nding)i(the)e(lo)o(w)o(er)g(b)q(ound)i(and)f(the)f (upp)q(er)i(b)q(ound)75 327 y(of)g(a)g(datat)o(yp)q(e.)75 478 y Fv(MPI)p 160 478 14 2 v 16 w(TYPE)p 293 478 V 17 w(LB\()f(datat)o(yp)q(e,)j(displacement\))117 556 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 627 y(OUT)108 b Fv(displacement)342 b Fk(displacemen)o(t)12 b(of)g(lo)o(w)o(er)g(b)q(ound)g(from)f(origin,)g(in)h(b)o(ytes)h(\(in-) 905 684 y(teger\))75 808 y Fu(int)23 b(MPI)p 245 808 15 2 v 17 w(Type)p 358 808 V 17 w(lb\(MPI)p 519 808 V 16 w(Datatype)g(datatype,)g(int*)g(displacement\))75 895 y(MPI)p 150 895 V 17 w(TYPE)p 263 895 V 16 w(LB\()h(DATATYPE,)e (DISPLACEMENT,)h(IERROR\))170 951 y(INTEGER)g(DATATYPE,)g (DISPLACEMENT,)f(IERROR)75 1132 y Fv(MPI)p 160 1132 14 2 v 16 w(TYPE)p 293 1132 V 17 w(UB\()15 b(datat)o(yp)q(e,)h (displacement\))117 1209 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 1281 y(OUT)108 b Fv(displacement)342 b Fk(displacemen)o(t)11 b(of)f(upp)q(er)i(b)q (ound)f(from)f(origin,)g(in)h(b)o(ytes)g(\(in-)905 1338 y(teger\))75 1462 y Fu(int)23 b(MPI)p 245 1462 15 2 v 17 w(Type)p 358 1462 V 17 w(ub\(MPI)p 519 1462 V 16 w(Datatype)g (datatype,)g(int*)g(displacement\))75 1548 y(MPI)p 150 1548 V 17 w(TYPE)p 263 1548 V 16 w(UB\()h(DATATYPE,)e(DISPLACEMENT,)h (IERROR\))170 1605 y(INTEGER)g(DATATYPE,)g(DISPLACEMENT,)f(IERROR)75 1754 y Fj(3.12.4)49 b(Comm)o(it)13 b(and)k(free)75 1839 y Fx(A)j(datat)o(yp)q(e)f(ob)s(ject)g(has)g(to)g(b)q(e)h Fq(committed)g Fx(b)q(efore)g(it)g(can)f(b)q(e)i(used)f(in)g(a)f(comm)o (unication.)34 b(A)75 1896 y(committed)15 b(datat)o(yp)q(e)g(can)h (still)h(b)q(e)f(used)g(as)e(a)i(argumen)o(t)e(in)i(datat)o(yp)q(e)f (constructors.)20 b(There)15 b(is)h(no)75 1952 y(need)g(to)f(commit)g (basic)h(datat)o(yp)q(es.)j(They)c(are)g(\\pre-committed.")75 2103 y Fv(MPI)p 160 2103 14 2 v 16 w(TYPE)p 293 2103 V 17 w(COMMIT\(datat)o(yp)q(e\))117 2180 y Fk(INOUT)62 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(that)g(is)g(committed) e(\(handle\))75 2305 y Fu(int)23 b(MPI)p 245 2305 15 2 v 17 w(Type)p 358 2305 V 17 w(commit\(MPI)p 615 2305 V 16 w(Datatype)f(*datatype\))75 2391 y(MPI)p 150 2391 V 17 w(TYPE)p 263 2391 V 16 w(COMMIT\(DATATYPE,)g(IERROR\))170 2448 y(INTEGER)h(DATATYPE,)g(IERROR)166 2534 y Fx(The)14 b(commit)h(op)q(eration)f(commits)g(the)h(datat)o(yp)q(e,)e(that)h(is,) g(the)h(formal)e(description)j(of)e(a)g(com-)75 2591 y(m)o(unication)h(bu\013er,)f(not)g(the)g(con)o(ten)o(t)f(of)h(that)g (bu\013er.)19 b(Th)o(us,)14 b(after)f(a)h(datat)o(yp)q(e)g(has)g(b)q (een)h(commit-)75 2647 y(ted,)e(it)g(can)f(b)q(e)i(rep)q(eatedly)f (reused)h(to)e(comm)o(unicate)h(the)f(c)o(hanging)h(con)o(ten)o(t)f(of) g(a)h(bu\013er)f(or,)h(indeed,)75 2704 y(the)i(con)o(ten)o(t)g(of)g (di\013eren)o(t)g(bu\013ers,)g(with)g(di\013eren)o(t)h(starting)e (addresses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 71 79 71 78 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(71)189 45 y Fl(A)n(dvic)n(e)19 b(to)i(implementors.)67 b Fx(The)20 b(system)f(ma)o(y)g(\\compile")i(at)e(commit)g(time)h(an)g (in)o(ternal)189 102 y(represen)o(tation)f(for)f(the)h(datat)o(yp)q(e)g (that)f(facilitates)i(comm)o(unication,)h(e.g.)31 b(c)o(hange)19 b(from)f(a)189 158 y(compacted)12 b(represen)o(tation)h(to)f(a)g (\015at)h(represen)o(tation)f(of)h(the)f(datat)o(yp)q(e,)h(and)g (select)g(the)g(most)189 214 y(con)o(v)o(enien)o(t)i(transfer)g(mec)o (hanism.)20 b(\()p Fl(End)c(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 414 y Fv(MPI)p 160 414 14 2 v 16 w(TYPE)p 293 414 V 17 w(FREE\(datat)o(yp)q(e\))117 491 y Fk(INOUT)62 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(that)g(is)g(freed)h (\(handle\))75 616 y Fu(int)23 b(MPI)p 245 616 15 2 v 17 w(Type)p 358 616 V 17 w(free\(MPI)p 567 616 V 16 w(Datatype)g (*datatype\))75 702 y(MPI)p 150 702 V 17 w(TYPE)p 263 702 V 16 w(FREE\(DATATYPE,)f(IERROR\))170 759 y(INTEGER)h(DATATYPE,)g (IERROR)166 845 y Fx(Marks)14 b(the)h(datat)o(yp)q(e)g(ob)s(ject)f (asso)q(ciated)h(with)h Fv(datat)o(yp)q(e)g Fx(for)f(deallo)q(cation)h (and)g(sets)e Fv(datat)o(yp)q(e)75 902 y Fx(to)e Fi(MPI)p 205 902 13 2 v 14 w(D)o(A)m(T)m(A)m(TYPE)p 431 902 V 12 w(NULL)p Fx(.)h(An)o(y)f(comm)o(unication)h(that)e(is)i(curren)o (tly)g(using)g(this)g(datat)o(yp)q(e)e(will)j(com-)75 958 y(plete)19 b(normally)l(.)31 b(Deriv)o(ed)19 b(datat)o(yp)q(es)f (that)g(w)o(ere)g(de\014ned)i(from)e(the)g(freed)h(datat)o(yp)q(e)f (are)g(not)g(af-)75 1015 y(fected.)75 1120 y Fq(Example)g(3.26)23 b Fx(The)10 b(follo)o(wing)h(co)q(de)g(fragmen)o(t)d(giv)o(es)j(some)f (examples)h(of)e(using)i Fv(MPI)p 1620 1120 14 2 v 16 w(TYPE)p 1753 1120 V 17 w(COMMIT)p Fx(.)75 1213 y Fu(INTEGER)23 b(type1,)g(type2)75 1269 y(CALL)g(MPI_TYPE_CONTIGUOUS\(5,)e(MPI_REAL,)i (type1,)g(ierr\))409 1326 y(!)h(new)f(type)h(object)f(created)75 1382 y(CALL)g(MPI_TYPE_COMMIT\(type1,)e(ierr\))409 1439 y(!)j(now)f(type1)g(can)h(be)f(used)h(for)f(communication)75 1495 y(type2)g(=)h(type1)409 1552 y(!)g(type2)f(can)g(be)h(used)f(for)h (communication)409 1608 y(!)g(\(it)f(is)h(a)g(handle)f(to)g(same)h (object)f(as)g(type1\))75 1665 y(CALL)g(MPI_TYPE_VECTOR\(3,)f(5,)h(4,)h (MPI_REAL,)f(type1,)g(ierr\))409 1721 y(!)h(new)f(uncommitted)g(type)g (object)g(created)75 1778 y(CALL)g(MPI_TYPE_COMMIT\(type1,)e(ierr\))409 1834 y(!)j(now)f(type1)g(can)h(be)f(used)h(anew)f(for)h(communication) 166 1939 y Fx(F)l(reeing)19 b(a)f(datat)o(yp)q(e)f(do)q(es)i(not)f (a\013ect)f(an)o(y)h(other)g(datat)o(yp)q(e)f(that)h(w)o(as)f(built)j (from)d(the)i(freed)75 1996 y(datat)o(yp)q(e.)32 b(The)20 b(system)e(b)q(eha)o(v)o(es)i(as)f(if)h(input)g(datat)o(yp)q(e)f (argumen)o(ts)g(to)f(deriv)o(ed)j(datat)o(yp)q(e)e(con-)75 2052 y(structors)14 b(are)h(passed)g(b)o(y)h(v)m(alue.)189 2157 y Fl(A)n(dvic)n(e)g(to)i(implementors.)49 b Fx(The)17 b(implemen)o(tation)h(ma)o(y)e(k)o(eep)h(a)g(reference)g(coun)o(t)g(of) f(activ)o(e)189 2214 y(comm)o(unications)h(that)e(use)i(the)f(datat)o (yp)q(e,)g(in)h(order)f(to)g(decide)i(when)f(to)e(free)i(it.)23 b(Also,)17 b(one)189 2270 y(ma)o(y)d(implemen)o(t)j(constructors)e(of)g (deriv)o(ed)h(datat)o(yp)q(es)f(so)g(that)g(they)g(k)o(eep)h(p)q(oin)o (ters)g(to)e(their)189 2327 y(datat)o(yp)q(e)g(argumen)o(ts,)g(rather)h (then)g(cop)o(ying)h(them.)k(In)15 b(this)h(case,)f(one)g(needs)h(to)e (k)o(eep)i(trac)o(k)189 2383 y(of)f(activ)o(e)g(datat)o(yp)q(e)g (de\014nition)i(references)f(in)g(order)g(to)e(kno)o(w)h(when)h(a)f (datat)o(yp)q(e)g(ob)s(ject)g(can)189 2440 y(b)q(e)h(freed.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 2561 y Fj(3.12.5)49 b(Use)16 b(of)h(general)f(datat)o(yp)q(es)g(in)h (comm)n(unication)75 2647 y Fx(Handles)g(to)f(deriv)o(ed)h(datat)o(yp)q (es)f(can)h(b)q(e)f(passed)h(to)f(a)g(comm)o(unication)h(call)g(wherev) o(er)f(a)g(datat)o(yp)q(e)75 2704 y(argumen)o(t)11 b(is)h(required.)19 b(A)12 b(call)g(of)f(the)h(form)e Fv(MPI)p 947 2704 V 16 w(SEND\(buf,)j(count,)g(datat)o(yp)q(e)g(,)f(...\))p Fx(,)e(where)i Fv(count)i Fp(>)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 72 80 72 79 bop 75 -100 a Fx(72)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(1)p Fx(,)12 b(is)h(in)o(terpreted)g(as)g(if)g(the)f(call)i(w)o(as)d(passed) i(a)f(new)h(datat)o(yp)q(e)f(whic)o(h)h(is)g(the)g(concatenation)g(of)f Fv(count)75 102 y Fx(copies)j(of)e Fv(datat)o(yp)q(e)p Fx(.)21 b(Th)o(us,)13 b Fv(MPI)p 666 102 14 2 v 16 w(SEND\(buf,)h (count,)h(datat)o(yp)q(e,)h(dest,)f(tag,)e(comm)m(\))d Fx(is)15 b(equiv)m(alen)o(t)g(to,)75 193 y Fu (MPI_TYPE_CONTIGUOUS\(count,)20 b(datatype,)j(newtype\))75 249 y(MPI_TYPE_COMMIT\(newtype\))75 305 y(MPI_SEND\(buf,)f(1,)i (newtype,)e(dest,)i(tag,)f(comm\).)75 396 y Fx(Similar)i(statemen)o(ts) e(apply)h(to)g(all)g(other)g(comm)o(unication)g(functions)h(that)e(ha)o (v)o(e)g(a)h Fv(count)h Fx(and)75 453 y Fv(datat)o(yp)q(e)17 b Fx(argumen)o(t.)166 509 y(Supp)q(ose)k(that)d(a)i(send)g(op)q (eration)g Fv(MPI)p 895 509 V 16 w(SEND\(buf,)h(count,)g(datat)o(yp)q (e.)35 b(dest,)22 b(tag,)e(comm)m(\))c Fx(is)75 566 y(executed,)g (where)f Fv(datat)o(yp)q(e)i Fx(has)e(t)o(yp)q(e)g(map,)189 657 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 664 y Fg(0)333 657 y Fp(;)8 b(disp)438 664 y Fg(0)457 657 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 664 y Ff(n)p Fh(\000)p Fg(1)723 657 y Fp(;)h(disp)828 664 y Ff(n)p Fh(\000)p Fg(1)895 657 y Fx(\))p Ft(g)p Fp(;)75 748 y Fx(and)19 b(exten)o(t)f Fp(extent)p Fx(.)31 b(\(Empt)o(y)18 b(en)o(tries)h(of)f(\\pseudo-t)o (yp)q(e")36 b Fi(MPI)p 1252 748 13 2 v 14 w(UB)18 b Fx(and)35 b Fi(MPI)p 1524 748 V 15 w(LB)18 b Fx(are)h(not)f(listed)75 804 y(in)g(the)e(t)o(yp)q(e)h(map,)g(but)g(they)g(a\013ect)f(the)h(v)m (alue)h(of)e Fp(extent)p Fx(.\))26 b(The)17 b(send)g(op)q(eration)g (sends)g Fp(n)12 b Ft(\001)f Fv(count)75 861 y Fx(en)o(tries,)17 b(where)g(en)o(try)g Fp(i)10 b Ft(\001)h Fp(n)g Fx(+)h Fp(j)19 b Fx(is)e(at)f(lo)q(cation)i Fp(addr)1034 868 y Ff(i;j)1088 861 y Fx(=)e Fv(buf)f Fx(+)c Fp(extent)i Ft(\001)e Fp(i)f Fx(+)i Fp(disp)1581 868 y Ff(j)1615 861 y Fx(and)17 b(has)g(t)o(yp)q(e)75 917 y Fp(ty)r(pe)159 924 y Ff(j)178 917 y Fx(,)d(for)f Fp(i)g Fx(=)g(0)p Fp(;)8 b(:::;)g Fv(co)o(unt)f Ft(\000)i Fx(1)k(and)i Fp(j)g Fx(=)e(0)p Fp(;)8 b(:::;)g(n)d Ft(\000)k Fx(1.)19 b(These)c(en)o(tries) f(need)h(not)f(b)q(e)h(con)o(tiguous,)f(nor)75 974 y(distinct;)i(their) g(order)f(can)g(b)q(e)h(arbitrary)l(.)166 1030 y(The)g(v)m(ariable)h (stored)e(at)g(address)g Fp(addr)879 1037 y Ff(i;j)933 1030 y Fx(in)i(the)e(calling)j(program)c(should)j(b)q(e)f(of)f(a)g(t)o (yp)q(e)h(that)75 1086 y(matc)o(hes)11 b Fp(ty)r(pe)331 1093 y Ff(j)349 1086 y Fx(,)h(where)f(t)o(yp)q(e)g(matc)o(hing)h(is)f (de\014ned)i(as)d(in)i(section)g(3.3.1.)17 b(The)11 b(message)g(sen)o (t)g(con)o(tains)75 1143 y Fp(n)f Ft(\001)g Fv(count)17 b Fx(en)o(tries,)e(where)g(en)o(try)g Fp(i)10 b Ft(\001)g Fp(n)g Fx(+)g Fp(j)18 b Fx(has)d(t)o(yp)q(e)g Fp(ty)r(pe)1100 1150 y Ff(j)1119 1143 y Fx(.)166 1199 y(Similarly)l(,)g(supp)q(ose)f (that)e(a)h(receiv)o(e)h(op)q(eration)f Fv(MPI)p 1100 1199 14 2 v 16 w(RECV\(buf,)h(count,)g(datat)o(yp)q(e,)h(source,)e (tag,)75 1256 y(comm)m(,)f(status\))17 b Fx(is)e(executed,)h(where)g Fv(datat)o(yp)q(e)g Fx(has)g(t)o(yp)q(e)f(map,)189 1347 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1354 y Fg(0)333 1347 y Fp(;)8 b(disp)438 1354 y Fg(0)457 1347 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1354 y Ff(n)p Fh(\000)p Fg(1)723 1347 y Fp(;)h(disp)828 1354 y Ff(n)p Fh(\000)p Fg(1)895 1347 y Fx(\))p Ft(g)p Fp(;)75 1438 y Fx(with)17 b(exten)o(t)g Fp(extent)p Fx(.)25 b(\(Again,)17 b(empt)o(y)g(en)o(tries)g(of)f (\\pseudo-t)o(yp)q(e")32 b Fi(MPI)p 1385 1438 13 2 v 15 w(UB)16 b Fx(and)32 b Fi(MPI)p 1653 1438 V 14 w(LB)17 b Fx(are)f(not)75 1494 y(listed)g(in)f(the)f(t)o(yp)q(e)h(map,)f(but)g (they)h(a\013ect)e(the)i(v)m(alue)h(of)e Fp(extent)p Fx(.\))20 b(This)15 b(receiv)o(e)h(op)q(eration)e(receiv)o(es)75 1551 y Fp(n)e Ft(\001)f Fv(count)19 b Fx(en)o(tries,)f(where)f(en)o (try)g Fp(i)12 b Ft(\001)f Fp(n)h Fx(+)g Fp(j)19 b Fx(is)f(at)f(lo)q (cation)h Fv(buf)e Fx(+)c Fp(extent)h Ft(\001)e Fp(i)g Fx(+)h Fp(disp)1579 1558 y Ff(j)1614 1551 y Fx(and)18 b(has)f(t)o(yp)q(e)75 1607 y Fp(ty)r(pe)159 1614 y Ff(j)178 1607 y Fx(.)i(If)c(the)f(incoming)h(message)f(consists)g(of)g Fp(k)h Fx(elemen)o(ts,)g(then)f(w)o(e)g(m)o(ust)g(ha)o(v)o(e)g Fp(k)f Ft(\024)g Fp(n)c Ft(\001)e Fv(count)p Fx(;)16 b(the)75 1664 y Fp(i)10 b Ft(\001)f Fp(n)i Fx(+)f Fp(j)s Fx(-th)15 b(elemen)o(t)h(of)f(the)g(message)g(should)h(ha)o(v)o(e)f(a)f (t)o(yp)q(e)i(that)e(matc)o(hes)h Fp(ty)r(pe)1521 1671 y Ff(j)1540 1664 y Fx(.)166 1720 y(T)o(yp)q(e)10 b(matc)o(hing)g(is)h (de\014ned)h(according)e(to)g(the)g(t)o(yp)q(e)g(signature)h(of)e(the)h (corresp)q(onding)i(datat)o(yp)q(es,)75 1776 y(that)k(is,)h(the)f (sequence)i(of)e(basic)i(t)o(yp)q(e)e(comp)q(onen)o(ts.)24 b(T)o(yp)q(e)17 b(matc)o(hing)g(do)q(es)f(not)h(dep)q(end)h(on)e(some) 75 1833 y(asp)q(ects)k(of)f(the)h(datat)o(yp)q(e)f(de\014nition,)j(suc) o(h)e(as)g(the)f(displacemen)o(ts)j(\(la)o(y)o(out)c(in)j(memory\))e (or)g(the)75 1889 y(in)o(termediate)d(t)o(yp)q(es)f(used.)75 1992 y Fq(Example)j(3.27)23 b Fx(This)c(example)h(sho)o(ws)e(that)h(t)o (yp)q(e)g(matc)o(hing)g(is)h(de\014ned)g(in)g(terms)e(of)h(the)g(basic) 75 2048 y(t)o(yp)q(es)c(that)g(a)g(deriv)o(ed)h(t)o(yp)q(e)f(consists)g (of.)75 2139 y Fu(...)75 2195 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\()e(2,)j (MPI_REAL,)f(type2,)g(...\))75 2252 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e (4,)j(MPI_REAL,)f(type4,)g(...\))75 2308 y(CALL)g (MPI_TYPE_CONTIGUOUS\()e(2,)j(type2,)f(type22,)g(...\))75 2365 y(...)75 2421 y(CALL)g(MPI_SEND\()g(a,)h(4,)f(MPI_REAL,)g(...\))75 2478 y(CALL)g(MPI_SEND\()g(a,)h(2,)f(type2,)g(...\))75 2534 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type22,)g(...\))75 2591 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type4,)g(...\))75 2647 y(...)75 2704 y(CALL)g(MPI_RECV\()g(a,)h(4,)f(MPI_REAL,)g(...\)) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 73 81 73 80 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(73)75 45 y Fu(CALL)23 b(MPI_RECV\()g(a,)h(2,)f(type2,)g (...\))75 102 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type22,)g(...\))75 158 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type4,)g(...\))75 250 y Fx(Eac)o(h)15 b(of)g(the)g(sends)h(matc)o(hes)f(an)o(y)f(of)h (the)g(receiv)o(es.)166 353 y(A)k(datat)o(yp)q(e)g(ma)o(y)f(sp)q(ecify) i(o)o(v)o(erlapping)g(en)o(tries.)32 b(If)19 b(suc)o(h)h(a)f(datat)o (yp)q(e)f(is)i(used)f(in)h(a)f(receiv)o(e)75 409 y(op)q(eration,)13 b(that)f(is,)i(if)f(some)f(part)h(of)f(the)h(receiv)o(e)g(bu\013er)g (is)h(written)e(more)h(than)f(once)h(b)o(y)g(the)g(receiv)o(e)75 466 y(op)q(eration,)i(then)h(the)f(call)h(is)g(erroneous.)166 522 y(Supp)q(ose)22 b(that)f Fv(MPI)p 540 522 14 2 v 16 w(RECV\(buf,)j(count,)g(datat)o(yp)q(e,)g(dest,)g(tag,)f(comm)m(,)c (status\))k Fx(is)f(executed,)75 578 y(where)15 b Fv(datat)o(yp)q(e)i Fx(has)e(t)o(yp)q(e)h(map,)189 670 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 677 y Fg(0)333 670 y Fp(;)8 b(disp)438 677 y Fg(0)457 670 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 677 y Ff(n)p Fh(\000)p Fg(1)723 670 y Fp(;)h(disp)828 677 y Ff(n)p Fh(\000)p Fg(1)895 670 y Fx(\))p Ft(g)p Fp(:)75 761 y Fx(The)14 b(receiv)o(ed)h(message)e(need)i(not)f(\014ll)h(all)g (the)e(receiv)o(e)i(bu\013er,)f(nor)f(do)q(es)i(it)f(need)g(to)f (\014ll)j(a)d(n)o(um)o(b)q(er)h(of)75 818 y(lo)q(cations)g(whic)o(h)h (is)f(a)f(m)o(ultiple)j(of)d Fp(n)p Fx(.)20 b(An)o(y)13 b(n)o(um)o(b)q(er,)h Fp(k)q Fx(,)g(of)f(basic)i(elemen)o(ts)f(can)g(b)q (e)g(receiv)o(ed,)h(where)75 874 y(0)d Ft(\024)h Fp(k)h Ft(\024)f Fv(count)8 b Ft(\001)f Fp(n)p Fx(.)19 b(The)13 b(n)o(um)o(b)q(er)h(of)f(basic)h(elemen)o(ts)g(receiv)o(ed)h(can)e(b)q (e)h(retriev)o(ed)g(from)f Fv(status)i Fx(using)75 931 y(the)g(query)h(function)g Fv(MPI)p 543 931 V 16 w(GET)p 647 931 V 16 w(ELEMENTS)p Fx(.)75 1082 y Fv(MPI)p 160 1082 V 16 w(GET)p 264 1082 V 17 w(ELEMENTS\()f(status,)h(datat)o(yp)q (e,)h(count\))117 1159 y Fk(IN)155 b Fv(status)476 b Fk(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 1234 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(used)h(b)o(y)f(receiv)o(e)h(op)q(eration)f(\(handle\))117 1308 y(OUT)108 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(basic)f(elemen)o(ts)g(\(in)o(teger\))75 1433 y Fu(int)23 b(MPI)p 245 1433 15 2 v 17 w(Get)p 334 1433 V 17 w(elements\(MPI)p 639 1433 V 16 w(Status)g(status,)f(MPI)p 1084 1433 V 17 w(Datatype)h(datatype,)g(int)g(*count\))75 1519 y(MPI)p 150 1519 V 17 w(GET)p 239 1519 V 17 w(ELEMENTS\(STATUS,)e (DATATYPE,)i(COUNT,)g(IERROR\))170 1576 y(INTEGER)g(STATUS\(MPI)p 604 1576 V 16 w(STATUS)p 764 1576 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 1662 y Fx(The)d(previously)i(de\014ned)f(function,)h Fv(MPI)p 930 1662 14 2 v 16 w(GET)p 1034 1662 V 16 w(COUNT)f Fx(\(Sec.)35 b(3.2.5\),)19 b(has)h(a)g(di\013eren)o(t)g(b)q(e-)75 1719 y(ha)o(vior.)28 b(It)17 b(returns)h(the)g(n)o(um)o(b)q(er)g(of)g (\\top-lev)o(el)g(elemen)o(ts")h(receiv)o(ed.)29 b(In)18 b(the)g(previous)g(example,)75 1775 y Fv(MPI)p 160 1775 V 16 w(GET)p 264 1775 V 17 w(COUNT)11 b Fx(ma)o(y)e(return)h(an)o(y)g (in)o(teger)g(v)m(alue)i Fp(k)q Fx(,)e(where)h(0)h Ft(\024)h Fp(k)h Ft(\024)f Fv(count)q Fx(.)18 b(If)11 b Fv(MPI)p 1618 1775 V 15 w(GET)p 1721 1775 V 17 w(COUNT)75 1832 y Fx(returns)29 b Fp(k)q Fx(,)j(then)d(the)g(n)o(um)o(b)q(er)h(of)e (basic)i(elemen)o(ts)f(receiv)o(ed)i(\(and)d(the)h(v)m(alue)i(returned) e(b)o(y)75 1888 y Fv(MPI)p 160 1888 V 16 w(GET)p 264 1888 V 17 w(ELEMENTS)p Fx(\))19 b(is)h Fp(n)13 b Ft(\001)f Fp(k)q Fx(.)32 b(If)20 b(the)f(n)o(um)o(b)q(er)h(of)f(basic)h(elemen)o (ts)g(receiv)o(ed)g(is)g(not)f(a)g(m)o(ulti-)75 1944 y(ple)g(of)e Fp(n)p Fx(,)i(that)e(is,)h(if)h(the)f(receiv)o(e)h(op)q (eration)f(has)f(not)h(receiv)o(ed)h(an)f(in)o(tegral)g(n)o(um)o(b)q (er)g(of)f Fv(datat)o(yp)q(e)75 2001 y Fx(\\copies,")e(then)h Fv(MPI)p 456 2001 V 16 w(GET)p 560 2001 V 16 w(COUNT)g Fx(returns)f(the)h(v)m(alue)g Fi(MPI)p 1175 2001 13 2 v 14 w(UNDEFINED)p Fx(.)75 2104 y Fq(Example)i(3.28)23 b Fx(Usage)14 b(of)h Fv(MPI)p 676 2104 14 2 v 16 w(GET)p 780 2104 V 17 w(COUNT)h Fx(and)f Fv(MPI)p 1139 2104 V 16 w(GET)p 1243 2104 V 17 w(ELEMENT)p Fx(.)75 2195 y Fu(...)75 2252 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\(2,)e(MPI_REAL,)i (Type2,)g(ierr\))75 2308 y(CALL)g(MPI_TYPE_COMMIT\(Type2,)e(ierr\))75 2365 y(...)75 2421 y(CALL)i(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2478 y(IF\(rank.EQ.0\))147 2534 y(THEN)218 2591 y(CALL)g(MPI_SEND\(a,)g (2,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 2647 y(CALL)g (MPI_SEND\(a,)g(3,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))147 2704 y(ELSE)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 74 82 74 81 bop 75 -100 a Fx(74)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)218 45 y Fu(CALL)23 b(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g(ierr\))218 102 y(CALL)g(MPI_GET_COUNT\(stat,)f(Type2,)h(i,)g(ierr\))119 b(!)24 b(returns)f(i=1)218 158 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e (Type2,)i(i,)h(ierr\))47 b(!)24 b(returns)f(i=2)218 214 y(CALL)g(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g(ierr\)) 218 271 y(CALL)g(MPI_GET_COUNT\(stat,)f(Type2,)h(i,)g(ierr\))119 b(!)24 b(returns)f(i=MPI_UNDEFINED)218 327 y(CALL)g (MPI_GET_ELEMENTS\(stat,)e(Type2,)i(i,)h(ierr\))47 b(!)24 b(returns)f(i=3)75 384 y(END)g(IF)166 485 y Fx(The)15 b(function)h Fv(MPI)p 522 485 14 2 v 16 w(GET)p 626 485 V 17 w(ELEMENTS)f Fx(can)g(also)g(b)q(e)g(used)h(after)e(a)h(prob)q(e)g (to)g(\014nd)h(the)f(n)o(um)o(b)q(er)75 541 y(of)20 b(elemen)o(ts)h(in) g(the)f(prob)q(ed)h(message.)34 b(Note)20 b(that)f(the)h(t)o(w)o(o)f (functions)i Fv(MPI)p 1507 541 V 16 w(GET)p 1611 541 V 17 w(COUNT)g Fx(and)75 598 y Fv(MPI)p 160 598 V 16 w(GET)p 264 598 V 17 w(ELEMENTS)15 b Fx(return)g(the)g(same)g(v)m (alues)i(when)e(they)h(are)e(used)i(with)g(basic)g(datat)o(yp)q(es.)189 699 y Fl(R)n(ationale.)38 b Fx(The)14 b(extension)g(giv)o(en)g(to)f (the)h(de\014nition)h(of)e Fv(MPI)p 1303 699 V 16 w(GET)p 1407 699 V 16 w(COUNT)i Fx(seems)e(natural:)189 755 y(one)22 b(w)o(ould)h(exp)q(ect)h(this)f(function)g(to)f(return)h(the)g(v)m (alue)g(of)g(the)f Fv(count)i Fx(argumen)o(t,)f(when)189 812 y(the)c(receiv)o(e)h(bu\013er)g(is)f(\014lled.)34 b(Sometimes)20 b Fv(datat)o(yp)q(e)h Fx(represen)o(ts)e(a)g(basic)h (unit)g(of)f(data)g(one)189 868 y(w)o(an)o(ts)h(to)g(transfer,)i(for)e (example,)j(a)e(record)g(in)h(an)g(arra)o(y)e(of)g(records)h (\(structures\).)37 b(One)189 925 y(should)15 b(b)q(e)h(able)f(to)f (\014nd)i(out)e(ho)o(w)g(man)o(y)g(comp)q(onen)o(ts)h(w)o(ere)g(receiv) o(ed)g(without)g(b)q(othering)h(to)189 981 y(divide)j(b)o(y)e(the)g(n)o (um)o(b)q(er)g(of)g(elemen)o(ts)h(in)g(eac)o(h)f(comp)q(onen)o(t.)26 b(Ho)o(w)o(ev)o(er,)16 b(on)h(other)g(o)q(ccasions,)189 1037 y Fv(datat)o(yp)q(e)g Fx(is)f(used)g(to)e(de\014ne)j(a)e(complex)h (la)o(y)o(out)f(of)g(data)g(in)h(the)f(receiv)o(er)h(memory)l(,)f(and)h (do)q(es)189 1094 y(not)h(represen)o(t)g(a)g(basic)i(unit)f(of)f(data)g (for)g(transfers.)26 b(In)18 b(suc)o(h)g(cases,)f(one)h(needs)g(to)f (use)h(the)189 1150 y(function)e Fv(MPI)p 452 1150 V 15 w(GET)p 555 1150 V 17 w(ELEMENTS)p Fx(.)f(\()p Fl(End)h(of)g(r)n (ationale.)p Fx(\))189 1251 y Fl(A)n(dvic)n(e)j(to)h(implementors.)66 b Fx(The)20 b(de\014nition)h(implies)h(that)c(a)i(receiv)o(e)g(cannot)f (c)o(hange)h(the)189 1308 y(v)m(alue)c(of)f(storage)f(outside)h(the)h (en)o(tries)f(de\014ned)i(to)d(comp)q(ose)h(the)h(comm)o(unication)f (bu\013er.)20 b(In)189 1364 y(particular,)13 b(the)f(de\014nition)i (implies)g(that)d(padding)i(space)g(in)g(a)e(structure)h(should)h(not)f (b)q(e)h(mo)q(d-)189 1421 y(i\014ed)h(when)g(suc)o(h)f(a)g(structure)g (is)h(copied)g(from)f(one)g(pro)q(cess)g(to)g(another.)19 b(This)14 b(w)o(ould)f(prev)o(en)o(t)189 1477 y(the)i(ob)o(vious)h (optimization)g(of)f(cop)o(ying)h(the)f(structure,)g(together)g(with)g (the)h(padding,)g(as)f(one)189 1534 y(con)o(tiguous)d(blo)q(c)o(k.)20 b(The)13 b(implemen)o(tation)g(is)g(free)g(to)f(do)g(this)h (optimization)h(when)f(it)g(do)q(es)f(not)189 1590 y(impact)17 b(the)h(outcome)f(of)g(the)h(computation.)26 b(The)18 b(user)f(can)h(\\force")f(this)h(optimization)g(b)o(y)189 1647 y(explicitly)e(including)h(padding)e(as)e(part)g(of)h(the)f (message.)19 b(\()p Fl(End)c(of)g(advic)n(e)g(to)g(implementors.)p Fx(\))75 1768 y Fj(3.12.6)49 b(Co)o(rrect)15 b(use)h(of)g(addresses)75 1853 y Fx(Successiv)o(ely)23 b(declared)f(v)m(ariables)h(in)e(C)g(or)g (F)l(ortran)e(are)i(not)g(necessarily)h(stored)f(at)f(con)o(tiguous)75 1910 y(lo)q(cations.)i(Th)o(us,)16 b(care)g(m)o(ust)f(b)q(e)h (exercised)i(that)d(displacemen)o(ts)i(do)f(not)f(cross)h(from)f(one)h (v)m(ariable)75 1966 y(to)i(another.)28 b(Also,)19 b(in)g(mac)o(hines)g (with)g(a)f(segmen)o(ted)g(address)g(space,)h(addresses)g(are)f(not)f (unique)75 2023 y(and)f(address)g(arithmetic)g(has)g(some)f(p)q (eculiar)j(prop)q(erties.)k(Th)o(us,)16 b(the)g(use)g(of)f Fq(addresses)p Fx(,)g(that)g(is,)75 2079 y(displacemen)o(ts)i(relativ)o (e)e(to)g(the)g(start)f(address)h Fi(MPI)p 1008 2079 13 2 v 15 w(BOTTOM)p Fx(,)f(has)h(to)g(b)q(e)h(restricted.)166 2136 y(V)l(ariables)g(b)q(elong)f(to)f(the)g(same)g Fq(sequen)o(tial)k (storage)c Fx(if)h(they)f(b)q(elong)i(to)e(the)g(same)g(arra)o(y)l(,)g (to)75 2192 y(the)k(same)g Fv(COMMON)g Fx(blo)q(c)o(k)h(in)g(F)l (ortran,)e(or)h(to)f(the)h(same)g(structure)f(in)i(C.)f(V)l(alid)h (addresses)f(are)75 2249 y(de\014ned)f(recursiv)o(ely)f(as)f(follo)o (ws:)131 2350 y(1.)22 b(The)g(function)g Fv(MPI)p 558 2350 14 2 v 16 w(ADDRESS)h Fx(returns)e(a)h(v)m(alid)h(address,)h(when) e(passed)g(as)f(argumen)o(t)g(a)189 2406 y(v)m(ariable)16 b(of)f(the)g(calling)i(program.)131 2498 y(2.)22 b(The)d Fv(buf)h Fx(argumen)o(t)f(of)g(a)g(comm)o(unication)h(function)g(ev)m (aluates)h(to)d(a)h(v)m(alid)i(address,)f(when)189 2555 y(passed)15 b(as)g(argumen)o(t)f(a)h(v)m(ariable)i(of)e(the)g(calling)i (program.)131 2647 y(3.)22 b(If)15 b Fv(v)g Fx(is)h(a)f(v)m(alid)i (address,)d(and)i Fv(i)f Fx(is)g(an)h(in)o(teger,)f(then)g Fv(v+i)g Fx(is)h(a)f(v)m(alid)i(address,)e(pro)o(vided)h Fv(v)f Fx(and)189 2704 y Fv(v+i)g Fx(are)g(in)h(the)f(same)g(sequen)o (tial)h(storage.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 75 83 75 82 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(75)131 45 y(4.)22 b(If)15 b Fv(v)g Fx(is)h(a)f(v)m(alid)i (address)e(then)g Fi(MPI)p 806 45 13 2 v 15 w(BOTTOM)e(+)h(v)i Fx(is)f(a)g(v)m(alid)i(address.)166 151 y(A)22 b(correct)g(program)g (uses)g(only)h(v)m(alid)i(addresses)d(to)g(iden)o(tify)i(the)e(lo)q (cations)h(of)f(en)o(tries)h(in)75 208 y(comm)o(unication)16 b(bu\013ers.)k(F)l(urthermore,)14 b(if)h Fv(u)h Fx(and)f Fv(v)g Fx(are)g(t)o(w)o(o)f(v)m(alid)j(addresses,)e(then)g(the)g(\(in)o (teger\))75 264 y(di\013erence)j Fv(u)g(-)g(v)f Fx(can)g(b)q(e)h (computed)g(only)f(if)h(b)q(oth)g Fv(u)f Fx(and)h Fv(v)f Fx(are)g(in)h(the)f(same)g(sequen)o(tial)i(storage.)75 321 y(No)c(other)g(arithmetic)h(op)q(erations)f(can)g(b)q(e)h (meaningfully)h(executed)f(on)g(addresses.)166 377 y(The)21 b(rules)h(ab)q(o)o(v)o(e)f(imp)q(ose)g(no)g(constrain)o(ts)g(on)g(the)g (use)g(of)g(deriv)o(ed)h(datat)o(yp)q(es,)f(as)g(long)g(as)75 434 y(they)d(are)f(used)h(to)f(de\014ne)h(a)g(comm)o(unication)g (bu\013er)f(that)g(is)h(wholly)h(con)o(tained)f(within)h(the)e(same)75 490 y(sequen)o(tial)23 b(storage.)38 b(Ho)o(w)o(ev)o(er,)22 b(the)g(construction)g(of)f(a)h(comm)o(unication)g(bu\013er)g(that)f (con)o(tains)75 547 y(v)m(ariables)f(that)f(are)g(not)g(within)h(the)f (same)g(sequen)o(tial)h(storage)e(m)o(ust)h(ob)q(ey)g(certain)h (restrictions.)75 603 y(Basically)l(,)g(a)e(comm)o(unication)h (bu\013er)f(with)g(v)m(ariables)i(that)d(are)h(not)g(within)h(the)f (same)g(sequen)o(tial)75 659 y(storage)g(can)h(b)q(e)h(used)g(only)g(b) o(y)f(sp)q(ecifying)i(in)f(the)g(comm)o(unication)g(call)g Fv(buf)g(=)g(MPI)p 1651 659 14 2 v 16 w(BOTTOM)p Fx(,)75 716 y Fv(count)i(=)g(1)p Fx(,)g(and)f(using)h(a)f Fv(datat)o(yp)q(e)i Fx(argumen)o(t)d(where)i(all)g(displacemen)o(ts)h(are)d(v)m(alid)j (\(absolute\))75 772 y(addresses.)189 879 y Fl(A)n(dvic)n(e)15 b(to)i(users.)43 b Fx(It)15 b(is)h(not)g(exp)q(ected)g(that)f Fv(MPI)g Fx(implemen)o(tations)i(will)g(b)q(e)f(able)h(to)e(detect)189 935 y(erroneous,)20 b(\\out)f(of)g(b)q(ound")h(displacemen)o(ts)h(|)f (unless)g(those)f(o)o(v)o(er\015o)o(w)g(the)g(user)h(address)189 992 y(space)15 b(|)g(since)g(the)g Fv(MPI)f Fx(call)i(ma)o(y)e(not)g (kno)o(w)h(the)f(exten)o(t)h(of)f(the)h(arra)o(ys)e(and)i(records)f(in) i(the)189 1048 y(host)e(program.)19 b(\()p Fl(End)d(of)g(advic)n(e)g (to)h(users.)p Fx(\))189 1154 y Fl(A)n(dvic)n(e)h(to)h(implementors.)57 b Fx(There)18 b(is)g(no)g(need)h(to)f(distinguish)i(\(absolute\))e (addresses)g(and)189 1211 y(\(relativ)o(e\))12 b(displacemen)o(ts)i(on) e(a)g(mac)o(hine)h(with)g(con)o(tiguous)f(address)h(space:)18 b Fi(MPI)p 1638 1211 13 2 v 15 w(BOTTOM)11 b Fx(is)189 1267 y(zero,)j(and)g(b)q(oth)g(addresses)h(and)f(displacemen)o(ts)i (are)e(in)o(tegers.)19 b(On)c(mac)o(hines)g(where)g(the)f(dis-)189 1324 y(tinction)d(is)g(required,)h(addresses)f(are)f(recognized)i(as)e (expressions)i(that)d(in)o(v)o(olv)o(e)i Fi(MPI)p 1668 1324 V 15 w(BOTTOM)p Fx(.)189 1380 y(\()p Fl(End)k(of)i(advic)n(e)f(to) g(implementors.)p Fx(\))75 1502 y Fj(3.12.7)49 b(Examples)75 1588 y Fx(The)15 b(follo)o(wing)h(examples)g(illustrate)h(the)e(use)g (of)g(deriv)o(ed)h(datat)o(yp)q(es.)75 1694 y Fq(Example)i(3.29)23 b Fx(Send)16 b(and)f(receiv)o(e)h(a)f(section)h(of)f(a)f(3D)h(arra)o(y) l(.)218 1800 y Fu(REAL)23 b(a\(100,100,100\),)f(e\(9,9,9\))218 1857 y(INTEGER)h(oneslice,)g(twoslice,)f(threeslice,)h(sizeofreal,)f (myrank,)h(ierr)218 1913 y(MPI_STATUS)g(status)75 2026 y(C)143 b(extract)23 b(the)g(section)g(a\(1:17:2,)g(3:11,)g(2:10\))75 2082 y(C)143 b(and)23 b(store)h(it)f(in)h(e\(:,:,:\).)218 2195 y(CALL)f(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))218 2308 y(CALL)j(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 2421 y(C)119 b(create)23 b(datatype)g(for)g(a)h(1D)g(section)218 2478 y(CALL)f(MPI_TYPE_VECTOR\()f(9,)i(1,)f(2,)h(MPI_REAL,)f(oneslice,) f(ierr\))75 2591 y(C)119 b(create)23 b(datatype)g(for)g(a)h(2D)g (section)218 2647 y(CALL)f(MPI_TYPE_HVECTOR\(9,)f(1,)h(100*sizeofreal,) f(oneslice,)h(twoslice,)f(ierr\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 76 84 76 83 bop 75 -100 a Fx(76)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fu(C)119 b(create)23 b(datatype)g(for)g(the)h(entire)f(section)218 102 y(CALL)g(MPI_TYPE_HVECTOR\()f(9,)i(1,)f(100*100*sizeofreal,)e (twoslice,)i(1,)767 158 y(threeslice,)f(ierr\))218 271 y(CALL)h(MPI_TYPE_COMMIT\()f(threeslice,)h(ierr\))218 327 y(CALL)g(MPI_SENDRECV\(a\(1,3,2\),)e(1,)j(threeslice,)e(myrank,)h (0,)h(e,)f(9*9*9,)648 384 y(MPI_REAL,)f(myrank,)h(0,)h(MPI_COMM_WORLD,) e(status,)h(ierr\))75 484 y Fq(Example)18 b(3.30)23 b Fx(Cop)o(y)14 b(the)h(\(strictly\))h(lo)o(w)o(er)e(triangular)i(part)e (of)h(a)g(matrix.)218 584 y Fu(REAL)23 b(a\(100,100\),)g(b\(100,100\)) 218 640 y(INTEGER)47 b(disp\(100\),)22 b(blocklen\(100\),)g(ltype,)h (myrank,)g(ierr)218 697 y(MPI_STATUS)g(status)75 810 y(C)119 b(copy)23 b(lower)h(triangular)e(part)h(of)h(array)f(a)75 866 y(C)119 b(onto)23 b(lower)h(triangular)e(part)h(of)h(array)f(b)218 979 y(CALL)g(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))75 1092 y(C)119 b(compute)23 b(start)g(and)h(size)f(of)h(each)f(column)218 1149 y(DO)h(i=1,)f(100)266 1205 y(disp\(i\))g(=)h(100*\(i-1\))e(+)i(i) 266 1261 y(block\(i\))f(=)g(100-i)218 1318 y(END)h(DO)75 1431 y(C)119 b(create)23 b(datatype)g(for)g(lower)h(triangular)e(part) 218 1487 y(CALL)h(MPI_TYPE_INDEXED\()f(100,)h(block,)g(disp,)h (MPI_REAL,)e(ltype,)h(ierr\))218 1600 y(CALL)g(MPI_TYPE_COMMIT\(ltype,) e(ierr\))218 1657 y(CALL)i(MPI_SENDRECV\()g(a,)g(1,)h(ltype,)f(myrank,) g(0,)g(b,)h(1,)552 1713 y(ltype,)f(myrank,)g(0,)h(MPI_COMM_WORLD,)e (status,)h(ierr\))75 1813 y Fq(Example)18 b(3.31)23 b Fx(T)l(ransp)q(ose)15 b(a)g(matrix.)218 1913 y Fu(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 1970 y(INTEGER)g(row,)g(xpose,)g (sizeofreal,)g(myrank,)g(ierr)218 2026 y(MPI_STATUS)g(status)75 2139 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 2252 y(CALL)g(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))218 2365 y(CALL)j(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 2478 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 2534 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f(MPI_REAL,)g(row,)g (ierr\))75 2647 y(C)119 b(create)23 b(datatype)g(for)g(matrix)g(in)h (row-major)f(order)218 2704 y(CALL)g(MPI_TYPE_HVECTOR\()f(100,)h(1,)h (sizeofreal,)e(row,)i(xpose,)f(ierr\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 77 85 77 84 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(77)218 102 y Fu(CALL)23 b(MPI_TYPE_COMMIT\()f(xpose,)h (ierr\))75 214 y(C)119 b(send)23 b(matrix)h(in)f(row-major)g(order)g (and)g(receive)g(in)h(column)f(major)g(order)218 271 y(CALL)g(MPI_SENDRECV\()g(a,)g(1,)h(xpose,)f(myrank,)g(0,)g(b,)h (100*100,)457 327 y(MPI_REAL,)e(myrank,)h(0,)h(MPI_COMM_WORLD,)e (status,)h(ierr\))75 430 y Fq(Example)18 b(3.32)23 b Fx(Another)15 b(approac)o(h)f(to)h(the)g(transp)q(ose)g(problem:)218 522 y Fu(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 578 y(INTEGER)47 b(disp\(2\),)23 b(blocklen\(2\),)f(type\(2\),)h(row,)g (row1,)g(sizeofreal)218 635 y(INTEGER)47 b(myrank,)23 b(ierr)218 691 y(MPI_STATUS)g(status)218 804 y(CALL)g (MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))75 917 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 1030 y(CALL)g (MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 1143 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 1199 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f(MPI_REAL,)g(row,)g (ierr\))75 1312 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row,)f (with)h(the)f(extent)g(of)h(one)f(real)g(number)218 1369 y(disp\(1\))g(=)h(0)218 1425 y(disp\(2\))f(=)h(sizeofreal)218 1482 y(type\(1\))47 b(=)24 b(row)218 1538 y(type\(2\))47 b(=)24 b(MPI_UB)218 1594 y(blocklen\(1\))46 b(=)24 b(1)218 1651 y(blocklen\(2\))46 b(=)24 b(1)218 1707 y(CALL)f(MPI_TYPE_STRUCT\() f(2,)i(blocklen,)e(disp,)i(type,)f(row1,)g(ierr\))218 1820 y(CALL)g(MPI_TYPE_COMMIT\()f(row1,)h(ierr\))75 1933 y(C)119 b(send)23 b(100)h(rows)f(and)h(receive)f(in)g(column)g(major)g (order)218 1990 y(CALL)g(MPI_SENDRECV\()g(a,)g(100,)g(row1,)h(myrank,)f (0,)g(b,)h(100*100,)457 2046 y(MPI_REAL,)e(myrank,)h(0,)h (MPI_COMM_WORLD,)e(status,)h(ierr\))75 2149 y Fq(Example)18 b(3.33)23 b Fx(W)l(e)15 b(manipulate)h(an)f(arra)o(y)f(of)h (structures.)75 2252 y Fu(struct)23 b(Partstruct)147 2308 y({)147 2365 y(int)95 b(class;)47 b(/*)23 b(particle)g(class)g(*/) 147 2421 y(double)g(d[6];)71 b(/*)23 b(particle)g(coordinates)f(*/)147 2478 y(char)71 b(b[7];)g(/*)23 b(some)g(additional)g(information)f(*/) 147 2534 y(};)75 2647 y(struct)h(Partstruct)94 b(particle[1000];)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 78 86 78 85 bop 75 -100 a Fx(78)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fu(int)429 b(i,)24 b(dest,)f(rank;)75 102 y(MPI_Comm)118 b(comm;)75 271 y(/*)24 b(build)f(datatype)g(describing)f(structure)h(*/)75 384 y(MPI_Datatype)f(Particletype;)75 440 y(MPI_Datatype)g(type[3])h(=) h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 497 y(int)238 b(blocklen[3])23 b(=)g({1,)h(6,)f(7};)75 553 y(MPI_Aint)118 b(disp[3];)75 723 y(/*)24 b(compute)e(displacements)h(of)g(structure)g (components)f(*/)75 835 y(MPI_Address\()g(particle,)h(disp\);)75 892 y(MPI_Address\()f(particle[0].d,)g(disp+1\);)75 948 y(MPI_Address\()g(particle[0].b,)g(disp+2\);)75 1005 y(for)h(\(i=0;)h(i)f(<3;)h(i++\))f(disp[i])g(-=)h(disp[0];)75 1118 y(MPI_Type_struct\()e(3,)h(blocklen,)g(disp,)g(type,)g (&Particletype\);)147 1231 y(/*)g(If)h(compiler)f(does)g(padding)g(in)g (mysterious)g(ways,)147 1287 y(the)g(following)g(may)g(be)h(safer)f(*/) 75 1400 y(MPI_Datatype)f(type1[4])h(=)h({MPI_INT,)e(MPI_DOUBLE,)h (MPI_CHAR,)f(MPI_UB};)75 1456 y(int)238 b(blocklen1[4])22 b(=)i({1,)g(6,)f(7,)h(1};)75 1513 y(MPI_Aint)118 b(disp1[4];)75 1626 y(/*)24 b(compute)e(displacements)h(of)g(structure)g(components)f (*/)75 1739 y(MPI_Address\()g(particle,)h(disp1\);)75 1795 y(MPI_Address\()f(particle[0].d,)g(disp1+1\);)75 1852 y(MPI_Address\()g(particle[0].b,)g(disp1+2\);)75 1908 y(MPI_Address\()g(particle+1,)h(disp1+3\);)75 1965 y(for)g(\(i=0;)h(i)f(<4;)h(i++\))f(disp1[i])g(-=)g(disp1[0];)75 2077 y(/*)h(build)f(datatype)g(describing)f(structure)h(*/)75 2190 y(MPI_Type_struct\()f(4,)h(blocklen1,)g(disp1,)g(type1,)g (&Particletype\);)409 2360 y(/*)h(4.1:)266 2416 y(send)f(the)h(entire)f (array)g(*/)75 2529 y(MPI_Type_commit\()f(&Particletype\);)75 2586 y(MPI_Send\()h(particle,)f(1000,)h(Particletype,)g(dest,)g(tag,)g (comm\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 79 87 79 86 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(79)409 45 y Fu(/*)24 b(4.2:)266 102 y(send)f(only)h(the)f (entries)g(of)g(class)h(zero)f(particles,)266 158 y(preceded)g(by)g (the)h(number)f(of)g(such)h(entries)f(*/)75 271 y(MPI_Datatype)f (Zparticles;)70 b(/*)24 b(datatype)f(describing)f(all)i(particles)791 327 y(with)f(class)g(zero)h(\(needs)f(to)g(be)h(recomputed)791 384 y(if)f(classes)g(change\))g(*/)75 440 y(MPI_Datatype)f(Ztype;)75 553 y(MPI_Aint)118 b(zdisp[1000];)75 610 y(int)23 b(zblock[1000],)g(j,) g(k;)75 666 y(int)g(zzblock[2])g(=)h({1,1};)75 723 y(MPI_Aint)118 b(zzdisp[2];)75 779 y(MPI_Datatype)22 b(zztype[2];)75 892 y(/*)i(compute)e(displacements)h(of)g(class)g(zero)h(particles)e (*/)75 948 y(j)i(=)f(0;)75 1005 y(for\(i=0;)g(i)g(<)h(1000;)f(i++\))123 1061 y(if)g(\(particle[i].class==0\))194 1118 y({)194 1174 y(zdisp[j])g(=)h(i;)194 1231 y(zblock[j])f(=)h(1;)194 1287 y(j++;)194 1344 y(})75 1456 y(/*)g(create)f(datatype)f(for)i (class)f(zero)g(particles)47 b(*/)75 1513 y(MPI_Type_indexed\()22 b(j,)h(zblock,)g(zdisp,)g(Particletype,)f(&Zparticles\);)75 1626 y(/*)i(prepend)e(particle)h(count)g(*/)75 1682 y(MPI_Address\(&j,) f(zzdisp\);)75 1739 y(MPI_Address\(particle,)f(zzdisp+1\);)75 1795 y(zztype[0])i(=)g(MPI_INT;)75 1852 y(zztype[1])g(=)g(Zparticles;) 75 1908 y(MPI_Type_struct\(2,)e(zzblock,)i(zzdisp,)g(zztype,)g (&Ztype\);)75 2021 y(MPI_Type_commit\()f(&Ztype\);)75 2077 y(MPI_Send\()h(MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g(comm\);) 242 2247 y(/*)h(A)f(probably)g(more)g(efficient)g(way)h(of)f(defining)g (Zparticles)f(*/)75 2360 y(/*)i(consecutive)e(particles)h(with)g(index) g(zero)g(are)h(handled)f(as)g(one)h(block)f(*/)75 2416 y(j=0;)75 2473 y(for)g(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 2529 y(if)g(\(particle[i].index==0\))170 2586 y({)170 2642 y(for)h(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d(==)i (0\))h(;)f(k++\);)170 2698 y(zdisp[j])g(=)h(i;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 80 88 80 87 bop 75 -100 a Fx(80)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Fu(zblock[j])23 b(=)h(k-i;)170 102 y(j++;)170 158 y(i)g(=)g(k;)170 214 y(})75 271 y(MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g (Particletype,)f(&Zparticles\);)457 440 y(/*)h(4.3:)314 497 y(send)g(the)g(first)h(two)f(coordinates)f(of)i(all)f(entries)g(*/) 75 610 y(MPI_Datatype)f(Allpairs;)142 b(/*)24 b(datatype)f(for)g(all)g (pairs)h(of)f(coordinates)g(*/)75 723 y(MPI_Aint)g(sizeofentry;)75 835 y(MPI_Type_extent\()f(Particletype,)g(sizeofentry\);)194 948 y(/*)i(sizeofentry)e(can)i(also)f(be)h(computed)e(by)i(subtracting) e(the)i(address)266 1005 y(of)f(particle[0])g(from)g(the)h(address)f (of)g(particle[1])f(*/)75 1118 y(MPI_Type_hvector\()g(1000,)h(2,)g (sizeofentry,)g(MPI_REAL,)f(&Allpairs\);)75 1174 y(MPI_Type_commit\()g (&Allpairs\);)75 1231 y(MPI_Send\()h(particle.d,)f(1,)i(Allpairs,)e (dest,)h(tag,)h(comm\);)218 1344 y(/*)g(an)f(alternative)g(solution)f (to)i(4.3)g(*/)75 1456 y(MPI_Datatype)e(Onepair;)71 b(/*)23 b(datatype)g(for)h(one)f(pair)g(of)h(coordinates,)e(with)695 1513 y(the)i(extent)f(of)h(one)f(particle)g(entry)g(*/)75 1569 y(MPI_Aint)g(disp2[3];)75 1626 y(MPI_Datatype)f(type2[3])h(=)h ({MPI_LB,)e(MPI_DOUBLE,)h(MPI_UB};)75 1682 y(int)g(blocklen2[3])g(=)g ({1,)h(2,)f(1};)75 1795 y(MPI_Address\()f(particle,)h(disp2\);)75 1852 y(MPI_Address\()f(particle[0].d,)g(disp2+1\);)75 1908 y(MPI_Address\()g(particle+1,)h(disp2+2\);)75 1965 y(for)g(\(i=0;)h(i<2;)f(i++\))g(disp2[i])g(-=)h(disp2[0];)75 2077 y(MPI_Type_struct\()e(3,)h(blocklen2,)g(disp2,)g(type2,)g (&Onepair\);)75 2134 y(MPI_Type_commit\()f(&Onepair\);)75 2190 y(MPI_Send\()h(particle[0].d,)f(1000,)h(Onepair,)g(dest,)g(tag,)g (comm\);)75 2353 y Fq(Example)18 b(3.34)23 b Fx(The)15 b(same)g(manipulations)i(as)e(in)h(the)g(previous)g(example,)g(but)f (use)h(absolute)g(ad-)75 2410 y(dresses)f(in)h(datat)o(yp)q(es.)75 2516 y Fu(struct)23 b(Partstruct)147 2572 y({)147 2629 y(int)g(class;)147 2685 y(double)g(d[6];)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 81 89 81 88 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(81)147 45 y Fu(char)23 b(b[7];)147 102 y(};)75 214 y(struct)g(Partstruct)g(particle[1000];)337 327 y(/*)h(build)f (datatype)g(describing)g(first)g(array)g(entry)g(*/)75 440 y(MPI_Datatype)f(Particletype;)75 497 y(MPI_Datatype)g(type[3])h(=) h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 553 y(int)238 b(block[3])23 b(=)h({1,)f(6,)h(7};)75 610 y(MPI_Aint)118 b(disp[3];)75 723 y(MPI_Address\()22 b(particle,)h(disp\);)75 779 y(MPI_Address\()f(particle[0].d,)g(disp+1\);)75 835 y(MPI_Address\()g(particle[0].b,)g(disp+2\);)75 892 y (MPI_Type_struct\()g(3,)h(block,)g(disp,)g(type,)h(&Particletype\);)75 1005 y(/*)g(Particletype)e(describes)g(first)i(array)f(entry)g(--)h (using)f(absolute)147 1061 y(addresses)f(*/)505 1174 y(/*)h(5.1:)361 1231 y(send)h(the)f(entire)g(array)g(*/)75 1344 y(MPI_Type_commit\()f(&Particletype\);)75 1400 y(MPI_Send\()h (MPI_BOTTOM,)f(1000,)h(Particletype,)f(dest,)h(tag,)h(comm\);)481 1569 y(/*)f(5.2:)290 1626 y(send)g(the)h(entries)e(of)i(class)f(zero,) 290 1682 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)e(*/)75 1795 y(MPI_Datatype)g(Zparticles,)h(Ztype;)75 1908 y(MPI_Aint)g (zdisp[1000])75 1965 y(int)g(zblock[1000],)g(i,)g(j,)h(k;)75 2021 y(int)f(zzblock[2])g(=)h({1,1};)75 2077 y(MPI_Datatype)e (zztype[2];)75 2134 y(MPI_Aint)118 b(zzdisp[2];)75 2247 y(j=0;)75 2303 y(for)23 b(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 2360 y(if)g(\(particle[i].index==0\))170 2416 y({)170 2473 y(for)h(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d(=)i (0\))h(;)g(k++\);)170 2529 y(zdisp[j])f(=)h(i;)170 2586 y(zblock[j])f(=)h(k-i;)170 2642 y(j++;)170 2698 y(i)g(=)g(k;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 82 90 82 89 bop 75 -100 a Fx(82)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Fu(})75 102 y(MPI_Type_indexed\()22 b(j,)h(zblock,)g(zdisp,)g(Particletype,)f (&Zparticles\);)75 158 y(/*)i(Zparticles)e(describe)h(particles)f(with) i(class)f(zero,)g(using)147 214 y(their)g(absolute)g(addresses*/)75 327 y(/*)h(prepend)e(particle)h(count)g(*/)75 384 y(MPI_Address\(&j,)f (zzdisp\);)75 440 y(zzdisp[1])h(=)g(MPI_BOTTOM;)75 497 y(zztype[0])g(=)g(MPI_INT;)75 553 y(zztype[1])g(=)g(Zparticles;)75 610 y(MPI_Type_struct\(2,)e(zzblock,)i(zzdisp,)g(zztype,)g(&Ztype\);)75 723 y(MPI_Type_commit\()f(&Ztype\);)75 779 y(MPI_Send\()h(MPI_BOTTOM,)f (1,)i(Ztype,)f(dest,)g(tag,)g(comm\);)75 942 y Fq(Example)18 b(3.35)23 b Fx(Handling)16 b(of)f(unions.)75 1048 y Fu(union)23 b({)147 1104 y(int)118 b(ival;)147 1161 y(float)70 b(fval;)218 1217 y(})24 b(u[1000])75 1330 y(int)119 b(utype;)75 1443 y(/*)24 b(All)f(entries)g(of)h(u)f(have)h(identical)e(type;)h(variable) 147 1500 y(utype)g(keeps)g(track)g(of)h(their)f(current)g(type)g(*/)75 1613 y(MPI_Datatype)70 b(type[2];)75 1669 y(int)286 b(blocklen[2])22 b(=)i({1,1};)75 1725 y(MPI_Aint)166 b(disp[2];)75 1782 y(MPI_Datatype)70 b(mpi_utype[2];)75 1838 y(MPI_Aint)166 b(i,j;)75 1951 y(/*)24 b(compute)e(an)i(MPI)g(datatype)e(for)i(each)f (possible)g(union)g(type;)147 2008 y(assume)g(values)g(are)g (left-aligned)f(in)i(union)f(storage.)g(*/)75 2121 y(MPI_Address\()f (u,)i(&i\);)75 2177 y(MPI_Address\()e(u+1,)i(&j\);)75 2234 y(disp[0])f(=)h(0;)f(disp[1])g(=)h(j-i)75 2290 y(type[1])f(=)h (MPI_UB;)75 2403 y(type[0])f(=)h(MPI_INT;)75 2459 y (MPI_Type_struct\(2,)d(blocklen,)i(disp,)g(type,)g(&mpi_utype[0]\);)75 2572 y(type[0])g(=)h(MPI_FLOAT;)75 2629 y(MPI_Type_struct\(2,)d (blocklen,)i(disp,)g(type,)g(&mpi_utype[1]\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 83 91 83 90 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fx(83)75 45 y Fu(for\(i=0;)23 b(i<2;)g(i++\))g (MPI_Type_commit\(&mpi_utype[i])o(\);)75 158 y(/*)h(actual)f (communication)f(*/)75 271 y(MPI_Send\(u,)g(1000,)i(mpi_utype[utype],)d (dest,)i(tag,)h(comm\);)75 414 y Fn(3.13)59 b(P)n(ack)20 b(and)f(unpack)75 516 y Fx(Some)c(existing)i(comm)o(unication)e (libraries)i(pro)o(vide)f(pac)o(k/unpac)o(k)f(functions)h(for)f (sending)h(noncon-)75 572 y(tiguous)c(data.)19 b(In)13 b(these,)f(the)h(user)f(explicitly)j(pac)o(ks)d(data)g(in)o(to)g(a)g (con)o(tiguous)h(bu\013er)f(b)q(efore)h(sending)75 629 y(it,)h(and)h(unpac)o(ks)g(it)f(from)g(a)g(con)o(tiguous)h(bu\013er)f (after)g(receiving)i(it.)k(Deriv)o(ed)14 b(datat)o(yp)q(es,)g(whic)o(h) h(are)75 685 y(describ)q(ed)g(in)e(Section)h(3.12,)e(allo)o(w)h(one,)g (in)h(most)e(cases,)g(to)h(a)o(v)o(oid)f(explicit)j(pac)o(king)e(and)g (unpac)o(king.)75 741 y(The)19 b(user)g(sp)q(eci\014es)h(the)f(la)o(y)o (out)f(of)g(the)h(data)f(to)g(b)q(e)h(send)g(or)f(receiv)o(ed,)j(and)e (the)f(comm)o(unication)75 798 y(library)i(directly)g(accesses)f(a)f (noncon)o(tiguous)h(bu\013er.)31 b(The)19 b(pac)o(k/unpac)o(k)g (routines)g(are)f(pro)o(vided)75 854 y(for)d(compatibilit)o(y)i(with)f (previous)h(libraries.)22 b(Also,)16 b(they)g(pro)o(vide)g(some)f (functionalit)o(y)i(that)e(is)h(not)75 911 y(otherwise)f(a)o(v)m (ailable)i(in)f Fv(MPI)p Fx(.)e(F)l(or)g(instance,)i(a)e(message)h(can) g(b)q(e)h(receiv)o(ed)g(in)g(sev)o(eral)f(parts,)f(where)75 967 y(the)19 b(receiv)o(e)h(op)q(eration)g(done)f(on)h(a)f(later)g (part)f(ma)o(y)h(dep)q(end)i(on)e(the)g(con)o(ten)o(t)g(of)g(a)f (former)h(part.)75 1024 y(Another)f(use)h(is)f(that)g(outgoing)g (messages)f(ma)o(y)h(b)q(e)h(explicitly)i(bu\013ered)d(in)h(user)g (supplied)h(space,)75 1080 y(th)o(us)f(o)o(v)o(erriding)h(the)f(system) g(bu\013ering)g(p)q(olicy)l(.)34 b(Finally)l(,)22 b(the)d(a)o(v)m (ailabilit)o(y)i(of)e(pac)o(k)g(and)h(unpac)o(k)75 1137 y(op)q(erations)c(facilitate)h(the)f(dev)o(elopmen)o(t)g(of)f (additional)j(comm)o(unication)e(libraries)i(la)o(y)o(ered)e(on)f(top) 75 1193 y(of)g Fv(MPI)p Fx(.)75 1344 y Fv(MPI)p 160 1344 14 2 v 16 w(P)l(A)o(CK\(inbuf,)g(incount,)i(datat)o(yp)q(e,)f(outbuf,)g (outcount,)h(p)q(osition,)f(comm)m(\))117 1421 y Fk(IN)155 b Fv(inbuf)493 b Fk(input)14 b(bu\013er)h(start)f(\(c)o(hoice\))117 1496 y(IN)155 b Fv(incount)448 b Fk(n)o(um)o(b)q(er)13 b(of)h(input)f(data)h(items)f(\(in)o(teger\))117 1571 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(input)g(data)f(item)g(\(handle\))117 1647 y(OUT)108 b Fv(outbuf)465 b Fk(output)14 b(bu\013er)h(start)g(\(c) o(hoice\))117 1722 y(IN)155 b Fv(outcount)420 b Fk(output)14 b(bu\013er)h(size,)f(in)g(b)o(ytes)g(\(in)o(teger\))117 1797 y(INOUT)62 b Fv(p)q(osition)439 b Fk(curren)o(t)16 b(p)q(osition)d(in)g(bu\013er,)i(in)e(b)o(ytes)i(\(in)o(teger\))117 1872 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(for)j(pac)o(k)o (ed)g(message)g(\(handle\))75 1996 y Fu(int)23 b(MPI)p 245 1996 15 2 v 17 w(Pack\(void*)g(inbuf,)g(int)g(incount,)g(MPI)p 1074 1996 V 17 w(Datatype)g(datatype,)f(void)h(*outbuf,)393 2053 y(int)h(outcount,)e(int)i(*position,)e(MPI)p 1160 2053 V 17 w(Comm)h(comm\))75 2139 y(MPI)p 150 2139 V 17 w(PACK\(INBUF,)f(INCOUNT,)h(DATATYPE,)g(OUTBUF,)f(OUTCOUNT,)h (POSITION,)g(COMM,)393 2196 y(IERROR\))170 2252 y()g(INBUF\(*\),) g(OUTBUF\(*\))170 2309 y(INTEGER)g(INCOUNT,)g(DATATYPE,)g(OUTCOUNT,)f (POSITION,)h(COMM,)g(IERROR)166 2395 y Fx(P)o(ac)o(ks)11 b(the)i(message)e(in)i(the)g(send)f(bu\013er)h(sp)q(eci\014ed)h(b)o(y)e Fv(inbuf,)i(incount,)g(datat)o(yp)q(e)g Fx(in)o(to)e(the)g(bu\013er)75 2452 y(space)j(sp)q(eci\014ed)i(b)o(y)e Fv(outbuf)h Fx(and)f Fv(outcount)p Fx(.)22 b(The)15 b(input)h(bu\013er)e(can)h(b)q(e)h(an)o (y)e(comm)o(unication)i(bu\013er)75 2508 y(allo)o(w)o(ed)h(in)g Fv(MPI)p 377 2508 14 2 v 16 w(SEND)p Fx(.)f(The)h(output)f(bu\013er)g (is)h(a)f(con)o(tiguous)h(storage)e(area)h(con)o(taining)h Fv(outcount)75 2565 y Fx(b)o(ytes,)e(starting)f(at)h(the)g(address)h Fv(outbuf)g Fx(\(length)g(is)g(coun)o(ted)f(in)h Fv(b)o(ytes)p Fx(,)h(not)d(elemen)o(ts,)i(as)f(if)h(it)f(w)o(ere)75 2621 y(a)g(comm)o(unication)h(bu\013er)f(for)g(a)f(message)h(of)g(t)o (yp)q(e)g Fv(MPI)p 1081 2621 V 16 w(P)l(A)o(CKED)p Fx(\).)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 84 92 84 91 bop 75 -100 a Fx(84)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fx(The)k(input)g(v)m (alue)h(of)e Fv(p)q(osition)h Fx(is)g(the)g(\014rst)f(lo)q(cation)h(in) g(the)g(output)f(bu\013er)h(to)e(b)q(e)j(used)f(for)75 102 y(pac)o(king.)27 b Fv(p)q(osition)19 b Fx(is)f(incremen)o(ted)g(b)o (y)g(the)f(size)h(of)f(the)h(pac)o(k)o(ed)f(message,)g(and)h(the)g (output)f(v)m(alue)75 158 y(of)e Fv(p)q(osition)i Fx(is)f(the)g (\014rst)f(lo)q(cation)i(in)f(the)g(output)f(bu\013er)h(follo)o(wing)h (the)e(lo)q(cations)i(o)q(ccupied)g(b)o(y)f(the)75 214 y(pac)o(k)o(ed)g(message.)k(The)c Fv(comm)10 b Fx(argumen)o(t)k(is)i (the)g(comm)o(unicator)f(that)g(will)i(b)q(e)g(subsequen)o(tly)f(used) 75 271 y(for)f(sending)h(the)f(pac)o(k)o(ed)h(message.)75 422 y Fv(MPI)p 160 422 14 2 v 16 w(UNP)l(A)o(CK\(inbuf,)g(insize,)f(p)q (osition,)h(outbuf,)g(outcount,)h(datat)o(yp)q(e,)g(comm)m(\))117 499 y Fk(IN)155 b Fv(inbuf)493 b Fk(input)14 b(bu\013er)h(start)f(\(c)o (hoice\))117 574 y(IN)155 b Fv(insize)485 b Fk(size)15 b(of)e(input)h(bu\013er,)g(in)g(b)o(ytes)g(\(in)o(teger\))117 649 y(INOUT)62 b Fv(p)q(osition)439 b Fk(curren)o(t)16 b(p)q(osition)d(in)g(b)o(ytes)i(\(in)o(teger\))117 724 y(OUT)108 b Fv(outbuf)465 b Fk(output)14 b(bu\013er)h(start)g(\(c)o (hoice\))117 800 y(IN)155 b Fv(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(items)f(to)g(b)q(e)i(unpac)o(k)o(ed)f(\(in)o(teger\))117 875 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(output)g(data)g(item)e(\(handle\))117 950 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(for)j(pac)o(k)o (ed)g(message)g(\(handle\))75 1074 y Fu(int)23 b(MPI)p 245 1074 15 2 v 17 w(Unpack\(void*)f(inbuf,)h(int)h(insize,)f(int)g (*position,)g(void)g(*outbuf,)393 1131 y(int)h(outcount,)e(MPI)p 802 1131 V 17 w(Datatype)h(datatype,)f(MPI)p 1344 1131 V 17 w(Comm)i(comm\))75 1217 y(MPI)p 150 1217 V 17 w(UNPACK\(INBUF,)e (INSIZE,)h(POSITION,)f(OUTBUF,)h(OUTCOUNT,)g(DATATYPE,)g(SOURCE,)393 1274 y(COMM,)g(IERROR\))170 1330 y()g(INBUF\(*\),)g(OUTBUF\(*\)) 170 1387 y(INTEGER)g(INSIZE,)g(POSITION,)g(OUTCOUNT,)g(DATATYPE,)f (COMM,)h(IERROR)166 1473 y Fx(Unpac)o(ks)15 b(a)g(message)f(in)o(to)h (the)g(receiv)o(e)h(bu\013er)f(sp)q(eci\014ed)i(b)o(y)e Fv(outbuf,)h(outcount,)g(datat)o(yp)q(e)h Fx(from)75 1530 y(the)d(bu\013er)g(space)g(sp)q(eci\014ed)i(b)o(y)e Fv(inbuf)i Fx(and)e Fv(insize)p Fx(.)20 b(The)14 b(output)g(bu\013er)g (can)g(b)q(e)h(an)o(y)e(comm)o(unication)75 1586 y(bu\013er)h(allo)o(w) o(ed)g(in)g Fv(MPI)p 500 1586 14 2 v 16 w(RECV)p Fx(.)g(The)g(input)h (bu\013er)e(is)h(a)g(con)o(tiguous)g(storage)e(area)h(con)o(taining)h Fv(insize)75 1642 y Fx(b)o(ytes,)e(starting)f(at)g(address)h Fv(inbuf)p Fx(.)20 b(The)12 b(input)h(v)m(alue)g(of)e Fv(p)q(osition)i Fx(is)g(the)f(\014rst)f(lo)q(cation)i(in)f(the)g (output)75 1699 y(bu\013er)18 b(o)q(ccupied)h(b)o(y)f(the)f(pac)o(k)o (ed)h(message.)27 b Fv(p)q(osition)18 b Fx(is)g(incremen)o(ted)h(b)o(y) f(the)g(size)g(of)f(the)h(pac)o(k)o(ed)75 1755 y(message,)d(so)g(that)f (the)i(output)f(v)m(alue)i(of)e Fv(p)q(osition)i Fx(is)e(the)h(\014rst) f(lo)q(cation)h(in)h(the)e(output)g(bu\013er)h(after)75 1812 y(the)e(lo)q(cations)g(o)q(ccupied)i(b)o(y)e(the)g(message)f(that) g(w)o(as)g(unpac)o(k)o(ed.)20 b Fv(comm)8 b Fx(is)14 b(the)g(comm)o(unicator)f(used)75 1868 y(to)i(receiv)o(e)h(the)f(pac)o (k)o(ed)g(message.)189 1975 y Fl(A)n(dvic)n(e)21 b(to)i(users.)83 b Fx(Note)22 b(the)g(di\013erence)h(b)q(et)o(w)o(een)g Fv(MPI)p 1280 1975 V 16 w(RECV)g Fx(and)f Fv(MPI)p 1611 1975 V 16 w(UNP)l(A)o(CK)p Fx(:)g(in)189 2031 y Fv(MPI)p 274 2031 V 15 w(RECV)p Fx(,)g(the)g Fv(count)h Fx(argumen)o(t)e(sp)q (eci\014es)i(the)f(maxim)o(um)f(n)o(um)o(b)q(er)h(of)f(items)h(that)f (can)189 2087 y(b)q(e)g(receiv)o(ed.)39 b(The)22 b(actual)f(n)o(um)o(b) q(er)h(of)e(items)i(receiv)o(ed)g(is)g(determined)g(b)o(y)f(the)h (length)f(of)189 2144 y(the)h(incoming)h(message.)40 b(In)23 b Fv(MPI)p 837 2144 V 15 w(UNP)l(A)o(CK)p Fx(,)f(the)g Fv(count)i Fx(argumen)o(t)d(sp)q(eci\014es)j(the)e(actual)189 2200 y(n)o(um)o(b)q(er)c(of)g(items)h(that)f(are)g(unpac)o(k)o(ed;)i (the)f(\\size")g(of)f(the)g(corresp)q(onding)i(message)e(is)h(the)189 2257 y(incremen)o(t)c(in)g Fv(p)q(osition)p Fx(.)21 b(The)14 b(reason)g(for)g(this)g(c)o(hange)h(is)f(that)g(the)g(\\incoming)h (message)f(size")189 2313 y(is)19 b(not)g(predetermined)i(since)g(the)e (user)g(decides)i(ho)o(w)e(m)o(uc)o(h)g(to)g(unpac)o(k;)i(nor)e(is)h (it)g(easy)f(to)189 2370 y(determine)c(the)g(\\message)f(size")i(from)e (the)h(n)o(um)o(b)q(er)g(of)f(items)h(to)g(b)q(e)g(unpac)o(k)o(ed.)20 b(In)c(fact,)e(in)h(a)189 2426 y(heterogeneous)i(system,)f(this)i(n)o (um)o(b)q(er)f(ma)o(y)f(not)h(b)q(e)g(determined)h Fl(a)g(priori)p Fx(.)26 b(\()p Fl(End)17 b(of)h(advic)n(e)189 2483 y(to)e(users.)p Fx(\))166 2589 y(T)l(o)g(understand)h(the)f(b)q(eha)o(vior)h(of)e(pac)o (k)h(and)h(unpac)o(k,)f(it)h(is)f(con)o(v)o(enien)o(t)h(to)f(think)h (of)e(the)i(data)75 2645 y(part)11 b(of)g(a)g(message)g(as)g(b)q(eing)i (the)e(sequence)i(obtained)f(b)o(y)g(concatenating)f(the)h(successiv)o (e)h(v)m(alues)f(sen)o(t)75 2702 y(in)k(that)f(message.)20 b(The)c(pac)o(k)f(op)q(eration)h(stores)f(this)g(sequence)i(in)f(the)g (bu\013er)f(space,)h(as)f(if)h(sending)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 85 93 85 92 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fx(85)75 45 y(the)13 b(message)g(to)g(that)f(bu\013er.)19 b(The)14 b(unpac)o(k)f(op)q(eration)h(retriev)o(es)f(this)h(sequence)g (from)f(bu\013er)g(space,)75 102 y(as)h(if)h(receiving)h(a)f(message)f (from)g(that)g(bu\013er.)19 b(\(It)c(is)g(helpful)h(to)e(think)i(of)e (in)o(ternal)h(F)l(ortran)f(\014les)h(or)75 158 y Fv(sscanf)h Fx(in)g(C,)f(for)g(a)g(similar)h(function.\))166 214 y(Sev)o(eral)h(messages)g(can)g(b)q(e)h(successiv)o(ely)g(pac)o(k)o(ed) f(in)o(to)g(one)h Fq(pac)o(king)i(unit)p Fx(.)26 b(This)17 b(is)h(e\013ected)75 271 y(b)o(y)c(sev)o(eral)h(successiv)o(e)h Fq(related)f Fx(calls)g(to)f Fv(MPI)p 909 271 14 2 v 16 w(P)l(A)o(CK)p Fx(,)g(where)g(the)h(\014rst)f(call)h(pro)o(vides)g Fv(p)q(osition)h(=)f(0)p Fx(,)75 327 y(and)i(eac)o(h)g(successiv)o(e)h (call)g(inputs)g(the)f(v)m(alue)i(of)d Fv(p)q(osition)i Fx(that)f(w)o(as)f(output)h(b)o(y)f(the)h(previous)h(call,)75 384 y(and)g(the)g(same)f(v)m(alues)i(for)e Fv(outbuf,)i(outcount)h Fx(and)e Fv(comm)m Fx(.)24 b(This)18 b(pac)o(king)g(unit)h(con)o(tains) f(no)o(w)f(the)75 440 y(equiv)m(alen)o(t)i(information)f(that)e(w)o (ould)i(ha)o(v)o(e)f(b)q(een)h(stored)f(in)h(a)f(message)g(b)o(y)h(one) f(send)h(call)h(with)e(a)75 497 y(send)f(bu\013er)f(that)f(is)i(the)f (\\concatenation")g(of)g(the)g(individual)k(send)d(bu\013ers.)166 553 y(A)g(pac)o(king)g(unit)g(can)g(b)q(e)h(sen)o(t)e(using)i(t)o(yp)q (e)f Fv(MPI)p 1029 553 V 15 w(P)l(A)o(CKED)p Fx(.)g(An)o(y)f(p)q(oin)o (t)i(to)e(p)q(oin)o(t)h(or)f(collectiv)o(e)75 610 y(comm)o(unication)d (function)h(can)f(b)q(e)g(used)g(to)f(mo)o(v)o(e)g(the)h(sequence)g(of) g(b)o(ytes)f(that)g(consists)h(the)f(pac)o(king)75 666 y(unit)16 b(from)e(one)h(pro)q(cess)g(to)f(another.)19 b(This)d(pac)o(king)f(unit)h(can)f(no)o(w)f(b)q(e)i(receiv)o(ed)g (using)g(an)o(y)e(receiv)o(e)75 723 y(op)q(eration,)j(with)g(an)o(y)f (datat)o(yp)q(e:)22 b(the)16 b(t)o(yp)q(e)h(matc)o(hing)g(rules)g(are)f (relaxed)h(for)f(messages)g(sen)o(t)h(with)75 779 y(t)o(yp)q(e)e Fv(MPI)p 262 779 V 16 w(P)l(A)o(CKED)p Fx(.)166 835 y(A)e(message)g (sen)o(t)g(with)h(an)o(y)e(t)o(yp)q(e)i(\(including)h Fv(MPI)p 1062 835 V 16 w(P)l(A)o(CKED)p Fx(\))e(can)g(b)q(e)h(receiv)o (ed)h(using)f(the)f(t)o(yp)q(e)75 892 y Fv(MPI)p 160 892 V 16 w(P)l(A)o(CKED)p Fx(.)h(Suc)o(h)i(message)f(can)g(then)h(b)q (e)g(unpac)o(k)o(ed)g(b)o(y)f(calls)h(to)e Fv(MPI)p 1414 892 V 16 w(UNP)l(A)o(CK)p Fx(.)166 948 y(A)19 b(pac)o(king)h(unit)g (\(or)f(a)g(message)g(created)g(b)o(y)g(a)g(regular,)h(\\t)o(yp)q(ed")f (send\))h(can)f(b)q(e)h(unpac)o(k)o(ed)75 1005 y(in)o(to)j(sev)o(eral)g (successiv)o(e)h(messages.)42 b(This)23 b(is)h(e\013ected)f(b)o(y)f (sev)o(eral)h(successiv)o(e)h(related)g(calls)g(to)75 1061 y Fv(MPI)p 160 1061 V 16 w(UNP)l(A)o(CK)p Fx(,)19 b(where)h(the)f(\014rst)g(call)i(pro)o(vides)f Fv(p)q(osition)g(=)g(0)p Fx(,)g(and)f(eac)o(h)h(successiv)o(e)g(call)h(inputs)75 1118 y(the)d(v)m(alue)h(of)e Fv(p)q(osition)i Fx(that)e(w)o(as)g (output)h(b)o(y)f(the)h(previous)h(call,)g(and)f(the)g(same)f(v)m (alues)i(for)e Fv(inbuf,)75 1174 y(insize)f Fx(and)f Fv(comm)m Fx(.)166 1231 y(The)23 b(concatenation)g(of)f(t)o(w)o(o)g (pac)o(king)h(unit)g(is)g(not)g(necessarily)h(a)e(pac)o(king)i(unit;)j (nor)22 b(is)h(a)75 1287 y(substring)17 b(of)g(a)f(pac)o(king)h(unit)h (necessarily)g(a)f(pac)o(king)g(unit.)25 b(Th)o(us,)17 b(one)g(cannot)g(concatenate)f(t)o(w)o(o)75 1344 y(pac)o(king)11 b(units)g(and)g(then)g(unpac)o(k)g(the)f(result)h(as)g(one)f(pac)o (king)h(unit;)i(nor)d(can)h(one)f(unpac)o(k)h(a)g(substring)75 1400 y(of)f(a)g(pac)o(king)h(unit)g(as)f(a)g(separate)g(pac)o(king)h (unit.)19 b(Eac)o(h)11 b(pac)o(king)f(unit,)i(that)e(w)o(as)g(created)g (b)o(y)g(a)h(related)75 1456 y(sequence)17 b(of)e(pac)o(k)h(calls,)g (or)f(b)o(y)h(a)f(regular)h(send,)g(m)o(ust)f(b)q(e)h(unpac)o(k)o(ed)h (as)e(a)g(unit,)i(b)o(y)e(a)h(sequence)g(of)75 1513 y(related)g(unpac)o (k)f(calls.)189 1609 y Fl(R)n(ationale.)74 b Fx(The)21 b(restriction)h(on)f(\\atomic")f(pac)o(king)h(and)g(unpac)o(king)h(of)f (pac)o(king)g(units)189 1665 y(allo)o(ws)13 b(the)f(implemen)o(tation)j (to)d(add)h(at)f(the)h(head)g(of)f(pac)o(king)h(units)h(additional)g (information,)189 1722 y(suc)o(h)i(as)g(a)g(description)h(of)f(the)g (sender)h(arc)o(hitecture)f(\(to)f(b)q(e)i(used)g(for)e(t)o(yp)q(e)h (con)o(v)o(ersion,)g(in)h(a)189 1778 y(heterogeneous)e(en)o(vironmen)o (t\))g(\()p Fl(End)h(of)g(r)n(ationale.)p Fx(\))166 1874 y(The)21 b(follo)o(wing)g(call)g(allo)o(ws)g(the)g(user)f(to)g(\014nd)h (out)f(ho)o(w)g(m)o(uc)o(h)h(space)f(is)h(needed)h(to)e(pac)o(k)g(a)75 1931 y(message)15 b(and,)g(th)o(us,)f(manage)h(space)h(allo)q(cation)g (for)e(bu\013ers.)75 2082 y Fv(MPI)p 160 2082 V 16 w(P)l(A)o(CK)p 291 2082 V 16 w(SIZE\(incount,)i(datat)o(yp)q(e,)g(comm)n(,)11 b(size\))117 2159 y Fk(IN)155 b Fv(incount)448 b Fk(coun)o(t)14 b(argumen)o(t)f(to)h(pac)o(king)f(call)g(\(in)o(teger\))117 2232 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(argumen)o(t)f(to)h(pac)o(king)f(call)g(\(handle\))117 2306 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(argumen)o(t)i(to) h(pac)o(king)f(call)g(\(handle\))117 2380 y(OUT)108 b Fv(size)519 b Fk(upp)q(er)16 b(b)q(ound)f(on)g(size)g(of)g(pac)o(k)o (ed)g(message,)f(in)h(b)o(ytes)g(\(in-)905 2436 y(teger\))75 2561 y Fu(int)23 b(MPI)p 245 2561 15 2 v 17 w(Pack)p 358 2561 V 17 w(size\(int)g(incount,)f(MPI)p 876 2561 V 17 w(Datatype)h(datatype,)g(MPI)p 1419 2561 V 16 w(Comm)h(comm,)393 2617 y(int)g(*size\))75 2704 y(MPI)p 150 2704 V 17 w(PACK)p 263 2704 V 16 w(SIZE\(INCOUNT,)e(DATATYPE,)h(COMM,)g(SIZE,)g(IERROR\)) -32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 86 94 86 93 bop 75 -100 a Fx(86)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Fu(INTEGER)23 b(INCOUNT,)g(DATATYPE,)g(COMM,)g(SIZE,)g(IERROR)166 132 y Fx(A)c(call)h(to)e Fv(MPI)p 450 132 14 2 v 16 w(P)l(A)o(CK)p 581 132 V 16 w(SIZE\(incount,)i(datat)o(yp)q(e,)h(size\))e Fx(returns)f(in)i Fv(size)f Fx(an)g(upp)q(er)g(b)q(ound)h(on)75 188 y(the)f(incremen)o(t)g(in)g Fv(p)q(osition)h Fx(that)e(is)h (e\013ected)f(b)o(y)h(a)f(call)i(to)e Fv(MPI)p 1253 188 V 15 w(P)l(A)o(CK\(inbuf,)i(incount,)h(datat)o(yp)q(e,)75 244 y(outbuf,)16 b(outcount,)h(p)q(osition\))p Fx(.)189 349 y Fl(R)n(ationale.)43 b Fx(The)16 b(call)h(returns)f(an)g(upp)q(er) h(b)q(ound,)f(rather)f(than)h(an)g(exact)f(b)q(ound,)i(since)g(the)189 406 y(exact)e(amoun)o(t)g(of)g(space)g(needed)i(to)e(pac)o(k)g(the)h (message)f(ma)o(y)g(dep)q(end)i(on)e(the)h(con)o(text)f(\(e.g.,)189 462 y(\014rst)f(message)h(pac)o(k)o(ed)g(in)h(a)f(pac)o(king)h(unit)g (ma)o(y)e(tak)o(e)h(more)g(space\).)20 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))75 567 y Fq(Example)h(3.36)23 b Fu(int)g(position,)g(i,)g(j,)h(a[2];)75 623 y(char)f(buff[1000];)75 736 y(....)75 849 y(MPI_Comm_rank\(MPI_COMM_WOR)o(LD,)e(myrank\);)75 905 y(if)j(\(myrank)e(==)i(0\))75 962 y({)147 1018 y(/)f(*)h(SENDER)f (CODE)g(*/)123 1131 y(position)f(=)i(0;)123 1188 y(MPI_Pack\()e(buff,)i (1000,)f(position,)f(i,)i(1,)g(MPI_INT\);)123 1244 y(MPI_Pack\()e (buff,)i(1000,)f(position,)f(j,)i(1,)g(MPI_INT\);)123 1301 y(MPI_Send\()e(buff,)i(position,)e(MPI_PACKED,)h(1,)g(0,)h (MPI_COMM_WORLD\);)75 1357 y(})75 1413 y(else)47 b(/*)24 b(RECEIVER)f(CODE)g(*/)123 1470 y(MPI_Recv\()f(a,)i(2,)g(MPI_INTEGER,)e (0,)h(0,)h(MPI_COMM_WORLD\))75 1583 y(})75 1687 y Fq(Example)18 b(3.37)23 b Fu(int)g(position,)g(i;)75 1744 y(float)g(a[1000];)75 1800 y(char)g(buff[1000])75 1913 y(....)75 2026 y (MPI_Comm_rank\(MPI_Comm_wor)o(ld,)e(myrank\);)75 2083 y(if)j(\(myrank)e(==)i(0\))75 2139 y({)123 2195 y(/)f(*)h(SENDER)f (CODE)h(*/)123 2308 y(int)f(len[2];)123 2365 y(MPI_Aint)f(disp[2];)123 2421 y(MPI_Datatype)g(type[2],)h(newtype;)123 2534 y(/*)g(build)g (datatype)g(for)h(i)f(followed)g(by)h(a[0]...a[i-1])e(*/)123 2647 y(len[0])h(=)g(1;)123 2704 y(len[1])g(=)g(i;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 87 95 87 94 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fx(87)123 45 y Fu(MPI_Address\()22 b(&i,)h(disp\);)123 102 y(MPI_Address\()f(a,)i(disp+1\);)123 158 y(type[0])f(=)g(MPI_INT;) 123 214 y(type[1])g(=)g(MPI_FLOAT;)123 271 y(MPI_Type_struct\()e(2,)j (len,)f(disp,)g(type,)h(&newtype\);)123 327 y(MPI_Type_commit\()d (&newtype\);)123 440 y(/*)i(Pack)h(i)f(followed)g(by)h(a[0]...a[i-1]*/) 123 553 y(position)e(=)i(0;)123 610 y(MPI_Pack\()e(MPI_BOTTOM,)h(1,)g (newtype,)g(buff,)g(position,)361 666 y(MPI_COMM_WORLD\);)123 779 y(/*)g(Send)h(*/)123 892 y(MPI_Send\()e(buff,)i(position,)e (MPI_PACKED,)h(1,)g(0,)361 948 y(MPI_COMM_WORLD\))75 1061 y(/*)h(*****)147 1118 y(One)f(can)g(replace)g(the)h(last)f(three)g (lines)h(with)147 1174 y(MPI_Send\()e(MPI_BOTTOM,)h(1,)g(newtype,)g(1,) h(0,)f(MPI_COMM_WORLD\);)147 1231 y(*****)g(*/)75 1569 y(})75 1682 y(else)g(/*)h(myrank)f(==)h(1)f(*/)75 1739 y({)147 1795 y(/*)g(RECEIVER)g(CODE)g(*/)123 1908 y(MPI_Status)f (status;)123 2021 y(/*)h(Receive)g(*/)123 2134 y(MPI_Recv\()f(buff,)i (1000,)f(MPI_PACKED,)f(0,)i(0,)f(status\);)123 2247 y(/*)g(Unpack)g(i)h (*/)99 2360 y(position)f(=)g(0;)99 2416 y(MPI_Unpack\()f(buff,)h(1000,) h(position,)e(i,)i(1,)f(MPI_INT\);)99 2529 y(/*)g(Unpack)g (a[0]...a[i-1])g(*/)99 2642 y(MPI_Unpack\()f(buff,)h(1000,)h(position,) e(a,)i(i,)f(MPI_FLOAT\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 88 96 88 95 bop 75 -100 a Fx(88)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fu(})75 136 y Fq(Example)h(3.38)23 b Fx(Eac)o(h)16 b(pro)q(cess)g(sends)h(a)f (coun)o(t,)f(follo)o(w)o(ed)i(b)o(y)f(coun)o(t)g(c)o(haracters)g(to)f (the)h(ro)q(ot;)g(the)75 193 y(ro)q(ot)e(concatenate)h(all)h(c)o (haracters)f(in)o(to)g(one)g(string.)75 276 y Fu(int)23 b(count,)g(gsize,)h(counts[64],)e(totalcount,)g(k1,)i(k2,)f(k,)170 332 y(displs[64],)g(position,)f(concat_pos;)75 389 y(char)h(chr[100],)g (*lbuf,)g(*rbuf,)g(*cbuf;)75 445 y(...)75 502 y(MPI_Comm_size\(comm,)e (gsize\);)75 558 y(MPI_Comm_rank\(comm,)g(myrank\);)218 671 y(/*)j(allocate)f(local)g(pack)g(buffer)g(*/)75 728 y(MPI_Pack_size\(1,)f(MPI_INT,)g(comm,)i(k1\);)75 784 y(MPI_Pack_size\(count,)d(MPI_CHAR,)i(k2\);)75 841 y(k)h(=)f(k1+k2;)75 897 y(lbuf)g(=)h(\(char)f(*\)malloc\(k\);)218 1010 y(/*)h(pack)f (count,)g(followed)g(by)g(count)h(characters)e(*/)75 1066 y(position)h(=)g(0;)75 1123 y(MPI_Pack\(&count,)f(1,)h(MPI_INT,)g (&lbuf,)g(k,)h(&position,)e(comm\);)75 1179 y(MPI_Pack\(chr,)g(count,)h (MPI_CHAR,)g(&lbuf,)g(k,)g(&position,)g(comm\);)75 1292 y(if)h(\(myrank)e(!=)i(root\))218 1349 y(/*)g(gather)f(at)g(root)h (sizes)f(of)g(all)h(packed)f(messages)g(*/)147 1405 y(MPI_Gather\()f (&position,)g(1,)i(MPI_INT,)f(NULL,)385 1462 y(NULL,)g(root,)h(comm\);) 218 1574 y(/*)g(gather)f(at)g(root)h(packed)f(messages)g(*/)123 1631 y(MPI_Gatherv\()f(&buf,)h(position,)g(MPI_PACKED,)f(NULL,)337 1687 y(NULL,)i(NULL,)f(NULL,)g(root,)g(comm\);)75 1800 y(else)g({)72 b(/*)23 b(root)h(code)f(*/)218 1857 y(/*)h(gather)f (sizes)g(of)h(all)f(packed)g(messages)g(*/)147 1913 y(MPI_Gather\()f (&position,)g(1,)i(MPI_INT,)f(counts,)385 1970 y(MPI_INT,)g(root,)g (comm\);)218 2083 y(/*)h(gather)f(all)g(packed)g(messages)g(*/)123 2139 y(displs[0])f(=)i(0;)123 2195 y(do)f(\(i=1;)g(i)h(<)g(gsize;)f (i++\))170 2252 y(displs[i])g(=)h(displs[i-1])e(+)i(counts[i-1];)123 2308 y(totalcount)e(=)i(dipls[gsize-1])e(+)i(counts[gsize-1];)123 2365 y(rbuf)f(=)h(\(char)f(*\)malloc\(totalcount\);)123 2421 y(cbuf)g(=)h(\(char)f(*\)malloc\(totalcount\);)123 2478 y(MPI_Gatherv\()f(lbuf,)h(position,)g(MPI_PACKED,)f(rbuf,)337 2534 y(counts,)h(displs,)g(MPI_PACKED,)g(root,)g(comm\);)218 2647 y(/*)h(unpack)f(all)g(messages)g(and)g(concatenate)g(strings)g(*/) 123 2704 y(concat_pos)f(=)i(0;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 89 97 89 96 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fx(89)123 45 y Fu(do)23 b(\(i=0;)g(i)h(<)g(gsize;)f(i++\))g({) 170 102 y(position)g(=)h(0;)170 158 y(MPI_Unpack\()f(rbuf+displs[i],)f (totalcount-displs[i],)314 214 y(position,)g(count,)h(1,)h(MPI_INT,)f (comm\);)170 271 y(MPI_Unpack\()g(rbuf+displs[i],)f (totalcount-displs[i],)314 327 y(position,)g(cbuf+concat_pos,)g(count,) h(MPI_CHAR,)g(comm\);)170 384 y(concat_pos)g(+=)h(count;)170 440 y(})123 497 y(cbuf[concat_pos])d(=)j(`\\0';)123 553 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 90 98 90 97 bop 75 356 a Fo(Chapter)34 b(4)75 564 y Fr(Collecti)q(v)m(e)42 b(Comm)m(unication)75 805 y Fn(4.1)59 b(Intro)r(duction)18 b(and)i(Overview)75 906 y Fx(Collectiv)o(e)e(comm)o(unication)g(is)f (de\014ned)h(as)e(comm)o(unication)i(that)e(in)o(v)o(olv)o(es)h(a)f (group)h(of)f(pro)q(cesses.)75 963 y(The)f(functions)h(of)f(this)h(t)o (yp)q(e)f(pro)o(vided)h(b)o(y)f Fv(MPI)g Fx(are)g(the)g(follo)o(wing.) 143 1056 y Ft(\017)23 b Fx(Barrier)15 b(sync)o(hronization)h(across)f (all)h(group)f(mem)o(b)q(ers)g(\(Sec.)20 b(4.3\).)143 1150 y Ft(\017)j Fx(Broadcast)15 b(from)h(one)h(mem)o(b)q(er)g(to)f (all)h(mem)o(b)q(ers)g(of)f(a)h(group)f(\(Sec.)25 b(4.4\).)d(This)c(is) f(sho)o(wn)f(in)189 1207 y(\014gure)f(4.1.)143 1301 y Ft(\017)23 b Fx(Gather)18 b(data)h(from)f(all)j(group)e(mem)o(b)q(ers)g (to)g(one)g(mem)o(b)q(er)g(\(Sec.)33 b(4.5\).)e(This)19 b(is)h(sho)o(wn)f(in)189 1357 y(\014gure)c(4.1.)143 1451 y Ft(\017)23 b Fx(Scatter)16 b(data)g(from)h(one)g(mem)o(b)q(er)g(to)f (all)i(mem)o(b)q(ers)g(of)e(a)h(group)g(\(Sec.)25 b(4.6\).)f(This)18 b(is)f(sho)o(wn)189 1507 y(in)f(\014gure)f(4.1.)143 1601 y Ft(\017)23 b Fx(A)16 b(v)m(ariation)i(on)e(Gather)g(where)h(all)h (mem)o(b)q(ers)f(of)f(the)h(group)f(receiv)o(e)i(the)f(result)g(\(Sec.) 25 b(4.7\).)189 1658 y(This)15 b(is)h(sho)o(wn)f(as)g(\\allgather")g (in)h(\014gure)f(4.1.)143 1751 y Ft(\017)23 b Fx(Scatter/Gather)10 b(data)h(from)h(all)g(mem)o(b)q(ers)g(to)g(all)g(mem)o(b)q(ers)g(of)g (a)g(group)f(\(also)h(called)h(complete)189 1808 y(exc)o(hange)i(or)g (all-to-all\))h(\(Sec.)k(4.8\).)f(This)d(is)g(sho)o(wn)e(as)h (\\alltoall")h(in)g(\014gure)g(4.1.)143 1902 y Ft(\017)23 b Fx(Global)15 b(reduction)h(op)q(erations)g(suc)o(h)f(as)g(sum,)g (max,)f(min,)i(or)e(user-de\014ned)j(functions,)f(where)189 1958 y(the)11 b(result)h(is)g(returned)g(to)e(all)j(group)e(mem)o(b)q (ers)g(and)h(a)f(v)m(ariation)h(where)f(the)h(result)g(is)g(returned) 189 2015 y(to)i(only)i(one)f(mem)o(b)q(er)h(\(Sec.)k(4.9\).)143 2108 y Ft(\017)j Fx(A)15 b(com)o(bined)h(reduction)g(and)g(scatter)e (op)q(eration)h(\(Sec.)21 b(4.10\).)143 2202 y Ft(\017)i Fx(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g (called)h(pre\014x\))g(\(Sec.)k(4.11\).)166 2296 y(A)e(collectiv)o(e)i (op)q(eration)e(is)g(executed)h(b)o(y)f(ha)o(ving)g(all)h(pro)q(cesses) f(in)h(the)f(group)g(call)h(the)f(com-)75 2352 y(m)o(unication)f (routine,)f(with)g(matc)o(hing)g(argumen)o(ts.)21 b(The)16 b(syn)o(tax)f(and)h(seman)o(tics)g(of)f(the)h(collectiv)o(e)75 2409 y(op)q(erations)e(are)g(de\014ned)i(to)e(b)q(e)g(consisten)o(t)h (with)f(the)h(syn)o(tax)e(and)i(seman)o(tics)f(of)g(the)g(p)q(oin)o (t-to-p)q(oin)o(t)75 2465 y(op)q(erations.)22 b(Th)o(us,)16 b(general)g(datat)o(yp)q(es)g(are)f(allo)o(w)o(ed)i(and)f(m)o(ust)f (matc)o(h)g(b)q(et)o(w)o(een)i(sending)g(and)f(re-)75 2522 y(ceiving)i(pro)q(cesses)e(as)g(sp)q(eci\014ed)i(in)f(Chapter)32 b(3.)23 b(One)17 b(of)e(the)i(k)o(ey)f(argumen)o(ts)f(is)h(a)g(comm)o (unicator)75 2578 y(that)f(de\014nes)i(the)f(group)g(of)g (participating)h(pro)q(cesses)f(and)g(pro)o(vides)h(a)e(con)o(text)h (for)f(the)h(op)q(eration.)75 2635 y(Sev)o(eral)f(collectiv)o(e)h (routines)f(suc)o(h)g(as)f(broadcast)g(and)h(gather)e(ha)o(v)o(e)i(a)f (single)i(originating)f(or)f(receiv-)75 2691 y(ing)i(pro)q(cess.)k(Suc) o(h)15 b(pro)q(cesses)h(are)f(called)h(the)f Fl(r)n(o)n(ot)p Fx(.)20 b(Some)15 b(argumen)o(ts)f(in)i(the)f(collectiv)o(e)i (functions)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 91 99 91 98 bop 75 -100 a Fm(4.1.)34 b(INTR)o(ODUCTION)16 b(AND)f(O)o(VER)-5 b(VIEW)890 b Fx(91)75 2444 y @beginspecial @setspecial %%BeginDocument: coll-fig1.ps /arrowdict 13 dict def % Local storage for the procedure % ``arrow.'' /arrow % The procedure ``arrow'' adds an { arrowdict begin % arrow shape to the current path. /headlength exch def % It takes seven arguments: the x /halfheadthickness exch 2 div def % and y coordinates of the tail /halfthickness exch 2 div def % (imagine that a line has been /tipy exch def /tipx exch def % drawn down the center of the /taily exch def /tailx exch def % arrow from the tip to the tail, % then x and y lie on this line), % the x and y coordinates of the % tip of the arrow, the thickness % of the arrow in the tail % portion, the thickness of the % arrow at the widest part of the % arrowhead and the length of the % arrowhead. /dx tipx tailx sub def % Compute the differences in x and /dy tipy taily sub def % y for the tip and tail. These /arrowlength dx dx mul dy dy mul add % will be used to compute the sqrt def % length of the arrow and to /angle dy dx atan def % compute the angle of direction % that the arrow is facing with % respect to the current user % coordinate system origin. /base arrowlength headlength sub def % Compute where the base of the % arrowhead will be. /savematrix matrix currentmatrix def % Save the current user coordinate % system. We are using the same % strategy to localize the effect % of transformations as was used % in the program to draw an % ellipse. tailx taily translate % Translate to the starting point % of the tail. angle rotate % Rotate the x-axis to correspond % with the center line of the % arrow. 0 halfthickness neg moveto % Add the arrow shape to the % current path. base halfthickness neg lineto base halfheadthickness neg lineto arrowlength 0 lineto base halfheadthickness lineto base halfthickness lineto 0 halfthickness lineto closepath savematrix setmatrix % Restore the current user % coordinate system. end } def /Box { /height exch def /length exch def length 0 rlineto 0 height rlineto length neg 0 rlineto closepath } def /Gdict 200 dict def /Grid { Gdict begin /ny exch def /nx exch def /dely exch def /delx exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def 0 1 ny { pop x y moveto lenx 0 rlineto stroke /y y dely add def} for /y ypos def /x xpos def 0 1 nx { pop x y moveto 0 leny rlineto stroke /x x delx add def} for end } def /GLdict 300 dict def /GridLabels { GLdict begin /shift exch def /raise exch def /yoff exch def /xoff exch def /p1 exch def /p2 exch def /ny exch def /nx exch def /dely exch def /delx exch def /Darray exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def /dx3 delx 3 div def /dy3 dely 3 div def /ix -1 def /iy ny 1 sub def Darray{ aload pop /Subc exch def /Text exch def /ix ix 1 add def ix nx ge { /ix 0 def /iy iy 1 sub def} if /x xpos delx ix 0.5 add mul add /Helvetica findfont p1 scalefont setfont Text stringwidth pop /Helvetica findfont p2 scalefont setfont Subc stringwidth pop add xoff add 2 div sub shift add def /y ypos dely iy 0.5 add mul add raise add def x y moveto /Helvetica findfont p1 scalefont Text show xoff yoff rmoveto /Helvetica findfont p2 scalefont Subc show } forall end clear } def 2 setlinecap 6.5 72 mul 320 sub 2 div 0 translate 0 300 moveto 20 20 6 6 Grid 0 300 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 380 200 380 12 24 18 arrow stroke 200 330 140 330 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (scatter) dup stringwidth pop 170 exch 2 div sub 400 moveto show (gather) dup stringwidth pop 170 exch 2 div sub 350 moveto show 220 300 moveto 20 20 6 6 Grid 220 300 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(1)] [()()] [()()] [()()] [()()] [()()] [(A)(2)] [()()] [()()] [()()] [()()] [()()] [(A)(3)] [()()] [()()] [()()] [()()] [()()] [(A)(4)] [()()] [()()] [()()] [()()] [()()] [(A)(5)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 0 moveto 20 20 6 6 Grid 0 0 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [(B)(0)] [(B)(1)] [(B)(2)] [(B)(3)] [(B)(4)] [(B)(5)] [(C)(0)] [(C)(1)] [(C)(2)] [(C)(3)] [(C)(4)] [(C)(5)] [(D)(0)] [(D)(1)] [(D)(2)] [(D)(3)] [(D)(4)] [(D)(5)] [(E)(0)] [(E)(1)] [(E)(2)] [(E)(3)] [(E)(4)] [(E)(5)] [(F)(0)] [(F)(1)] [(F)(2)] [(F)(3)] [(F)(4)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 220 0 moveto 20 20 6 6 Grid 220 0 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(1)] [(B)(1)] [(C)(1)] [(D)(1)] [(E)(1)] [(F)(1)] [(A)(2)] [(B)(2)] [(C)(2)] [(D)(2)] [(E)(2)] [(F)(2)] [(A)(3)] [(B)(3)] [(C)(3)] [(D)(3)] [(E)(3)] [(F)(3)] [(A)(4)] [(B)(4)] [(C)(4)] [(D)(4)] [(E)(4)] [(F)(4)] [(A)(5)] [(B)(5)] [(C)(5)] [(D)(5)] [(E)(5)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 60 200 60 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (alltoall) dup stringwidth pop 170 exch 2 div sub 77 moveto show 0 150 moveto 20 20 6 6 Grid 0 150 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(B)(0)] [()()] [()()] [()()] [()()] [()()] [(C)(0)] [()()] [()()] [()()] [()()] [()()] [(D)(0)] [()()] [()()] [()()] [()()] [()()] [(E)(0)] [()()] [()()] [()()] [()()] [()()] [(F)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 200 200 200 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (allgather) dup stringwidth pop 170 exch 2 div sub 220 moveto show 220 150 moveto 20 20 6 6 Grid 220 150 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 450 moveto 20 20 6 6 Grid 0 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 510 200 510 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont 0 575 moveto (data) show (data) stringwidth pop 4 add 575 4 add (data) stringwidth pop 33 add 575 4 add 1 4 5 arrow fill (broadcast) dup stringwidth pop 170 exch 2 div sub 527 moveto show gsave 0 570 (processes) stringwidth pop sub translate 90 rotate 0 5 moveto (processes) show -4 8 -33 8 1 4 5 arrow fill grestore 220 450 moveto 20 20 6 6 Grid 220 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels showpage %%EndDocument @endspecial 98 x(Figure)15 b(4.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2598 y(eac)o(h)f(ro)o(w)e(of)i(b)q(o)o(xes)f (represen)o(ts)h(data)f(lo)q(cations)i(in)f(one)g(pro)q(cess.)20 b(Th)o(us,)13 b(in)i(the)e(broadcast,)g(initially)75 2655 y(just)i(the)g(\014rst)g(pro)q(cess)g(con)o(tains)g(the)g(data)g Fp(A)889 2662 y Fg(0)909 2655 y Fx(,)f(but)h(after)g(the)g(broadcast)f (all)i(pro)q(cesses)g(con)o(tain)f(it.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 92 100 92 99 bop 75 -100 a Fx(92)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fx(are)i(sp)q(eci\014ed)i (as)e(\\signi\014can)o(t)h(only)f(at)g(ro)q(ot,")f(and)i(are)e(ignored) i(for)f(all)h(participan)o(ts)f(except)h(the)75 102 y(ro)q(ot.)f(The)12 b(reader)g(is)g(referred)g(to)f(Chapter)h(3)g(for)f(information)h (concerning)h(comm)o(unication)f(bu\013ers,)75 158 y(general)19 b(datat)o(yp)q(es)f(and)h(t)o(yp)q(e)g(matc)o(hing)g(rules,)h(and)f(to) f(Chapter)h(5)f(for)h(information)g(on)f(ho)o(w)h(to)75 214 y(de\014ne)d(groups)f(and)h(create)f(comm)o(unicators.)166 271 y(The)d(t)o(yp)q(e-matc)o(hing)h(conditions)g(for)e(the)i (collectiv)o(e)g(op)q(erations)g(are)e(more)h(strict)g(than)g(the)g (cor-)75 327 y(resp)q(onding)j(conditions)g(b)q(et)o(w)o(een)f(sender)h (and)f(receiv)o(er)h(in)f(p)q(oin)o(t-to-p)q(oin)o(t.)20 b(Namely)l(,)15 b(for)e(collectiv)o(e)75 384 y(op)q(erations,)18 b(the)g(amoun)o(t)f(of)g(data)g(sen)o(t)g(m)o(ust)g(exactly)h(matc)o(h) f(the)h(amoun)o(t)f(of)g(data)g(sp)q(eci\014ed)j(b)o(y)75 440 y(the)15 b(receiv)o(er.)20 b(Distinct)c(t)o(yp)q(e)f(maps)f(\(the)h (la)o(y)o(out)f(in)i(memory)l(,)e(see)h(Sec.)20 b(3.12\))13 b(b)q(et)o(w)o(een)i(sender)h(and)75 497 y(receiv)o(er)g(are)f(still)h (allo)o(w)o(ed.)166 553 y(Collectiv)o(e)g(routine)f(calls)h(can)f (\(but)f(are)g(not)h(required)g(to\))f(return)h(as)f(so)q(on)h(as)f (their)h(participa-)75 610 y(tion)d(in)h(the)f(collectiv)o(e)i(comm)o (unication)e(is)h(complete.)19 b(The)12 b(completion)h(of)f(a)f(call)i (indicates)h(that)d(the)75 666 y(caller)k(is)g(no)o(w)f(free)g(to)g (access)g(lo)q(cations)h(in)g(the)g(comm)o(unication)g(bu\013er.)k(It) 14 b(do)q(es)h(not)f(indicate)i(that)75 723 y(other)c(pro)q(cesses)h (in)h(the)f(group)f(ha)o(v)o(e)h(completed)g(or)f(ev)o(en)i(started)d (the)i(op)q(eration)g(\(unless)h(otherwise)75 779 y(indicated)k(in)f (the)f(description)i(of)e(the)g(op)q(eration\).)23 b(Th)o(us,)16 b(a)g(collectiv)o(e)i(comm)o(unication)f(call)h(ma)o(y)l(,)75 835 y(or)c(ma)o(y)h(not,)f(ha)o(v)o(e)h(the)g(e\013ect)f(of)h(sync)o (hronizing)h(all)g(calling)h(pro)q(cesses.)j(This)c(statemen)o(t)e (excludes,)75 892 y(of)h(course,)g(the)g(barrier)g(function.)166 948 y(Collectiv)o(e)24 b(comm)o(unication)f(calls)g(ma)o(y)f(use)g(the) g(same)g(comm)o(unicators)g(as)g(p)q(oin)o(t-to-p)q(oin)o(t)75 1005 y(comm)o(unication;)16 b Fv(MPI)f Fx(guaran)o(tees)g(that)g (messages)g(generated)h(on)g(b)q(ehalf)h(of)e(collectiv)o(e)i(comm)o (uni-)75 1061 y(cation)c(calls)h(will)g(not)f(b)q(e)g(confused)h(with)f (messages)f(generated)h(b)o(y)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication.)75 1118 y(A)h(more)g(detailed)i(discussion)g(of)d(correct)h (use)h(of)e(collectiv)o(e)j(routines)f(is)g(found)f(in)h(Sec.)21 b(4.12.)189 1195 y Fl(R)n(ationale.)50 b Fx(The)18 b(equal-data)f (restriction)g(\(on)g(t)o(yp)q(e)g(matc)o(hing\))g(w)o(as)f(made)h(so)f (as)h(to)f(a)o(v)o(oid)189 1252 y(the)e(complexit)o(y)h(of)f(pro)o (viding)h(a)f(facilit)o(y)h(analogous)f(to)g(the)g(status)f(argumen)o (t)h(of)f Fv(MPI)p 1746 1252 14 2 v 16 w(RECV)189 1308 y Fx(for)f(disco)o(v)o(ering)j(the)e(amoun)o(t)g(of)g(data)f(sen)o(t.) 19 b(Some)14 b(of)f(the)g(collectiv)o(e)j(routines)d(w)o(ould)h (require)189 1365 y(an)h(arra)o(y)f(of)g(status)h(v)m(alues.)189 1436 y(The)e(statemen)o(ts)g(ab)q(out)g(sync)o(hronization)i(are)e (made)h(so)f(as)g(to)g(allo)o(w)h(a)f(v)m(ariet)o(y)h(of)f(implemen-) 189 1492 y(tations)h(of)h(the)g(collectiv)o(e)i(functions.)189 1563 y(The)c(collectiv)o(e)i(op)q(erations)e(do)g(not)g(accept)g(a)g (message)f(tag)g(argumen)o(t.)19 b(If)13 b(future)g(revisions)h(of)189 1619 y Fv(MPI)g Fx(de\014ne)j(non-blo)q(c)o(king)g(collectiv)o(e)g (functions,)f(then)f(tags)g(\(or)f(a)h(similar)i(mec)o(hanism\))e(will) 189 1676 y(need)i(to)g(b)q(e)g(added)h(so)f(as)f(to)h(allo)o(w)g(the)g (dis-am)o(biguation)h(of)e(m)o(ultiple,)j(p)q(ending,)g(collectiv)o(e) 189 1732 y(op)q(erations.)h(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 1810 y Fl(A)n(dvic)n(e)h(to)h(users.)59 b Fx(It)19 b(is)f(dangerous)h(to)e(rely)i(on)f(sync)o(hronization)i (side-e\013ects)f(of)f(the)g(col-)189 1866 y(lectiv)o(e)j(op)q (erations)f(for)f(program)g(correctness.)34 b(F)l(or)20 b(example,)h(ev)o(en)g(though)f(a)f(particular)189 1923 y(implemen)o(tation)i(ma)o(y)e(pro)o(vide)i(a)e(broadcast)g(routine)i (with)f(a)g(side-e\013ect)h(of)e(sync)o(hroniza-)189 1979 y(tion,)14 b(the)g(standard)g(do)q(es)h(not)f(require)h(this,)g (and)f(a)g(program)f(that)h(relies)h(on)g(this)f(will)i(not)e(b)q(e)189 2036 y(p)q(ortable.)189 2107 y(On)d(the)g(other)f(hand,)i(a)f(correct,) g(p)q(ortable)g(program)f(m)o(ust)g(allo)o(w)h(for)f(the)h(fact)g(that) f(a)g(collectiv)o(e)189 2163 y(call)j Fl(may)f Fx(b)q(e)g(sync)o (hronizing.)21 b(Though)11 b(one)h(cannot)g(rely)g(on)g(an)o(y)g(sync)o (hronization)g(side-e\013ect,)189 2220 y(one)17 b(m)o(ust)f(program)f (so)i(as)f(to)g(allo)o(w)h(it.)25 b(These)18 b(issues)f(are)g (discussed)h(further)f(in)g(Sec.)26 b(4.12.)189 2276 y(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))189 2353 y Fl(A)n(dvic)n(e)21 b(to)h(implementors.)77 b Fx(While)22 b(v)o(endors)g(ma)o(y)e(write)i(optimized)g(collectiv)o(e)i(routines) 189 2410 y(matc)o(hed)c(to)f(their)i(arc)o(hitectures,)g(a)f(complete)h (library)f(of)g(the)g(collectiv)o(e)i(comm)o(unication)189 2466 y(routines)d(can)h(b)q(e)f(written)h(en)o(tirely)g(using)g(the)f Fv(MPI)f Fx(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)g(func-)189 2523 y(tions)11 b(and)g(a)f(few)h(auxiliary)i(functions.)19 b(If)11 b(implemen)o(ting)i(on)e(top)f(of)h(p)q(oin)o(t-to-p)q(oin)o (t,)h(a)f(hidden,)189 2579 y(sp)q(ecial)j(comm)o(unicator)e(m)o(ust)g (b)q(e)h(created)g(for)f(the)g(collectiv)o(e)i(op)q(eration)f(so)f(as)g (to)g(a)o(v)o(oid)h(in)o(ter-)189 2636 y(ference)j(with)f(an)o(y)g (on-going)h(p)q(oin)o(t-to-p)q(oin)o(t)f(comm)o(unication)h(at)f(the)g (time)h(of)f(the)g(collectiv)o(e)189 2692 y(call.)21 b(This)15 b(is)h(discussed)h(further)e(in)h(Sec.)k(4.12.)f(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 93 101 93 100 bop 75 -100 a Fm(4.2.)34 b(COMMUNICA)l(TOR)16 b(AR)o(GUMENT)957 b Fx(93)75 45 y Fn(4.2)59 b(Communicato)n(r)19 b(a)n(rgument)75 147 y Fx(The)c(k)o(ey)h(concept)f(of)g(the)g (collectiv)o(e)i(functions)f(is)g(to)f(ha)o(v)o(e)f(a)h(\\group")g(of)g (participating)h(pro)q(cesses.)75 203 y(The)d(routines)f(do)h(not)e(ha) o(v)o(e)h(a)g(group)g(iden)o(ti\014er)i(as)e(an)g(explicit)j(argumen)o (t.)j(Instead,)13 b(there)f(is)h(a)f(com-)75 259 y(m)o(unicator)k (argumen)o(t.)21 b(F)l(or)15 b(the)h(purp)q(oses)h(of)e(this)i(c)o (hapter,)e(a)h(comm)o(unicator)g(can)g(b)q(e)g(though)o(t)f(of)75 316 y(as)d(a)g(group)g(iden)o(ti\014er)h(link)o(ed)h(with)f(a)f(con)o (text.)18 b(An)12 b(in)o(ter-comm)o(unicator,)h(that)e(is,)i(a)f(comm)o (unicator)75 372 y(that)i(spans)i(t)o(w)o(o)e(groups,)g(is)i Fl(not)f Fx(allo)o(w)o(ed)g(as)g(an)g(argumen)o(t)g(to)f(a)h(collectiv) o(e)i(function.)75 516 y Fn(4.3)59 b(Ba)n(rrier)21 b(synchronization)75 712 y Fv(MPI)p 160 712 14 2 v 16 w(BARRIER\()16 b(comm)9 b(\))117 789 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 913 y Fu(int)23 b(MPI)p 245 913 15 2 v 17 w(Barrier\(MPI)p 526 913 V 16 w(Comm)g(comm)h(\))75 1000 y(MPI)p 150 1000 V 17 w(BARRIER\(COMM,)e(IERROR\))170 1056 y(INTEGER)h(COMM,)h(IERROR)166 1143 y Fv(MPI)p 251 1143 14 2 v 16 w(BARRIER)13 b Fx(blo)q(c)o(ks)g(the)f(caller)i(un)o (til)f(all)g(group)f(mem)o(b)q(ers)h(ha)o(v)o(e)e(called)j(it.)19 b(The)13 b(call)g(returns)75 1199 y(at)i(an)o(y)f(pro)q(cess)i(only)g (after)e(all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered)h(the)f (call.)75 1342 y Fn(4.4)59 b(Broadcast)90 1538 y Fv(MPI)p 175 1538 V 16 w(BCAST\()16 b(bu\013er,)f(count,)h(datat)o(yp)q(e,)g(ro) q(ot,)f(comm)9 b(\))117 1615 y Fk(INOUT)78 b Fv(bu\013er)462 b Fk(starting)14 b(address)h(of)f(bu\013er)g(\(c)o(hoice\))117 1691 y(IN)171 b Fv(count)466 b Fk(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(bu\013er)i(\(in)o(teger\))117 1766 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)i(\(handle\))117 1841 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(broadcast)i(ro)q(ot)f(\(in)o(teger\))117 1916 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 2040 y Fu(int)23 b(MPI)p 245 2040 15 2 v 17 w(Bcast\(void*)g(buffer,)f (int)i(count,)f(MPI)p 1074 2040 V 17 w(Datatype)g(datatype,)f(int)i (root,)393 2097 y(MPI)p 468 2097 V 17 w(Comm)f(comm)h(\))75 2183 y(MPI)p 150 2183 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g (ROOT,)g(COMM,)g(IERROR\))170 2240 y()g(BUFFER\(*\))170 2296 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)166 2383 y Fv(MPI)p 251 2383 14 2 v 16 w(BCAST)17 b Fx(broadcasts)e(a)h (message)g(from)g(the)g(pro)q(cess)g(with)h(rank)f Fv(ro)q(ot)g Fx(to)g(all)h(pro)q(cesses)f(of)75 2439 y(the)h(group,)g(itself)h (included.)27 b(It)17 b(is)h(called)g(b)o(y)f(all)h(mem)o(b)q(ers)f(of) g(group)f(using)i(the)f(same)g(argumen)o(ts)75 2496 y(for)d Fv(comm)n(,)d(ro)q(ot)p Fx(.)20 b(On)15 b(return,)g(the)g(con)o(ten)o (ts)f(of)h Fv(ro)q(ot)p Fx('s)f(comm)o(unication)h(bu\013er)g(has)g(b)q (een)h(copied)h(to)75 2552 y(all)f(pro)q(cesses.)166 2609 y(General,)24 b(deriv)o(ed)f(datat)o(yp)q(es)e(are)h(allo)o(w)o (ed)g(for)g Fv(datat)o(yp)q(e)p Fx(.)42 b(The)22 b(t)o(yp)q(e)g (signature)g(of)g Fv(count,)75 2665 y(datat)o(yp)q(e)16 b Fx(on)f(an)o(y)f(pro)q(cess)h(m)o(ust)f(b)q(e)h(equal)g(to)f(the)h(t) o(yp)q(e)g(signature)g(of)f Fv(count,)h(datat)o(yp)q(e)i Fx(at)c(the)i(ro)q(ot.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 94 102 94 101 bop 75 -100 a Fx(94)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fx(This)g(implies)i(that)d (the)h(amoun)o(t)e(of)i(data)f(sen)o(t)g(m)o(ust)g(b)q(e)h(equal)h(to)e (the)h(amoun)o(t)e(receiv)o(ed,)j(pairwise)75 102 y(b)q(et)o(w)o(een)i (eac)o(h)h(pro)q(cess)f(and)g(the)g(ro)q(ot.)31 b Fv(MPI)p 909 102 14 2 v 16 w(BCAST)20 b Fx(and)f(all)h(other)f(data-mo)o(v)o (emen)o(t)e(collectiv)o(e)75 158 y(routines)j(mak)o(e)g(this)g (restriction.)34 b(Distinct)21 b(t)o(yp)q(e)f(maps)g(b)q(et)o(w)o(een)g (sender)g(and)g(receiv)o(er)h(are)f(still)75 214 y(allo)o(w)o(ed.)75 333 y Fj(4.4.1)49 b(Example)15 b(using)i(MPI)p 622 333 15 2 v 18 w(BCAST)75 419 y Fq(Example)h(4.1)k Fx(Broadcast)15 b(100)f(in)o(ts)h(from)g(pro)q(cess)g Fu(0)g Fx(to)g(ev)o(ery)g(pro)q (cess)g(in)i(the)e(group.)170 503 y Fu(MPI_Comm)23 b(comm;)170 559 y(int)h(array[100];)170 616 y(int)g(root=0;)170 672 y(...)170 728 y(MPI_Bcast\()f(array,)g(100,)g(MPI_INT,)g(root,)g (comm\);)75 812 y Fx(As)15 b(in)h(man)o(y)f(of)g(our)g(example)h(co)q (de)g(fragmen)o(ts,)e(w)o(e)h(assume)g(that)g(some)g(of)g(the)g(v)m (ariables)i(\(suc)o(h)e(as)75 869 y Fu(comm)g Fx(in)h(the)f(ab)q(o)o(v) o(e\))f(ha)o(v)o(e)h(b)q(een)i(assigned)e(appropriate)h(v)m(alues.)75 1009 y Fn(4.5)59 b(Gather)75 1205 y Fv(MPI)p 160 1205 14 2 v 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcount,)h(sendt)o(yp)q(e,)f (recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)m(\))117 1282 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 1354 y(IN)171 b Fv(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (send)h(bu\013er)f(\(in)o(teger\))117 1426 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er) g(elemen)o(ts)f(\(handle\))117 1498 y(OUT)124 b Fv(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 1554 y(ro)q(ot\))117 1626 y(IN)171 b Fv(recvcount)389 b Fk(n)o(um)o(b)q(er)11 b(of)g(elemen)o(ts)g(for)g(an)o(y)g(single)g(receiv)o(e)i(\(in)o (teger,)f(sig-)905 1682 y(ni\014can)o(t)i(only)f(at)h(ro)q(ot\))117 1754 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)16 b(t)o(yp)q(e)g(of)f (recv)i(bu\013er)g(elemen)o(ts)f(\(signi\014can)o(t)f(only)g(at)905 1811 y(ro)q(ot\))f(\(handle\))117 1883 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 1955 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 2079 y Fu(int)23 b(MPI)p 245 2079 15 2 v 17 w(Gather\(void*)f(sendbuf,) h(int)h(sendcount,)e(MPI)p 1217 2079 V 17 w(Datatype)h(sendtype,)393 2135 y(void*)g(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 2135 V 17 w(Datatype)h(recvtype,)f(int)i(root,)393 2192 y(MPI)p 468 2192 V 17 w(Comm)f(comm\))75 2278 y(MPI)p 150 2278 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 2335 y(ROOT,)g(COMM,)h(IERROR\))170 2391 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2448 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)166 2534 y Fx(Eac)o(h)16 b(pro)q(cess)h(\(ro)q(ot)f (pro)q(cess)h(included\))h(sends)g(the)e(con)o(ten)o(ts)g(of)g(its)h (send)h(bu\013er)e(to)g(the)h(ro)q(ot)75 2591 y(pro)q(cess.)i(The)11 b(ro)q(ot)f(pro)q(cess)i(receiv)o(es)g(the)f(messages)f(and)i(stores)e (them)h(in)h(rank)f(order.)18 b(The)11 b(outcome)75 2647 y(is)j Fl(as)h(if)f Fx(eac)o(h)g(of)f(the)h Fu(n)g Fx(pro)q(cesses)g (in)h(the)f(group)f(\(including)k(the)d(ro)q(ot)f(pro)q(cess\))g(had)h (executed)h(a)f(call)75 2704 y(to)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 95 103 95 102 bop 75 -100 a Fm(4.5.)34 b(GA)l(THER)1452 b Fx(95)189 45 y Fu(MPI)p 264 45 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)p Fp(;)8 b Fu(sendcount)p Fp(;)f Fu(se)o(ndtype)p Fp(;)g Fu(roo)o(t)p Fp(;)h(:)o(::)o Fx(\))p Fp(;)75 129 y Fx(and)15 b(the)h(ro)q(ot)e(had)h(executed)i Fu(n)e Fx(calls)h(to)189 213 y Fu(MPI)p 264 213 V 16 w(Recv)p Fx(\()p Fu(recvbuf)9 b Fx(+)h Fu(i)g Ft(\001)g Fu(recvcount)f Ft(\001)g Fu(extent)p Fx(\()p Fu(recvtype)p Fx(\))p Fp(;)f Fu(recvc)o(ount)p Fp(;)f Fu(recv)o(type)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 298 y Fx(where)15 b Fu(extent\(recvtype\))e Fx(is)j(the)f(t)o(yp)q(e)h(exten)o(t)f(obtained)g(from)g(a)g(call)h(to) f Fu(MPI)p 1515 298 V 16 w(Type)p 1627 298 V 17 w(extent\(\))p Fx(.)166 354 y(An)j(alternativ)o(e)h(description)h(is)e(that)g(the)g Fu(n)g Fx(messages)g(sen)o(t)g(b)o(y)g(the)g(pro)q(cesses)h(in)g(the)f (group)75 410 y(are)c(concatenated)h(in)g(rank)f(order,)g(and)h(the)g (resulting)g(message)f(is)h(receiv)o(ed)h(b)o(y)e(the)h(ro)q(ot)f(as)g (if)h(b)o(y)f(a)75 467 y(call)i(to)f Fv(MPI)p 299 467 14 2 v 16 w(RECV\(recvbuf,)h(recvcount)p Ft(\001)p Fv(n,)g(recvt)o(yp)q (e,)g(...\))p Fx(.)166 523 y(The)f(receiv)o(e)h(bu\013er)g(is)f (ignored)h(for)f(all)h(non-ro)q(ot)f(pro)q(cesses.)166 580 y(General,)i(deriv)o(ed)h(datat)o(yp)q(es)e(are)g(allo)o(w)o(ed)h (for)f(b)q(oth)h Fv(sendt)o(yp)q(e)i Fx(and)e Fv(recvt)o(yp)q(e)p Fx(.)25 b(The)17 b(t)o(yp)q(e)g(sig-)75 636 y(nature)c(of)g Fv(sendcount,)j(sendt)o(yp)q(e)f Fx(on)e(pro)q(cess)h Fu(i)e Fx(m)o(ust)h(b)q(e)h(equal)g(to)e(the)h(t)o(yp)q(e)g(signature)h (of)e Fv(recvcount,)75 693 y(recvt)o(yp)q(e)h Fx(at)e(the)i(ro)q(ot.)18 b(This)12 b(implies)i(that)e(the)g(amoun)o(t)f(of)h(data)f(sen)o(t)h(m) o(ust)g(b)q(e)g(equal)h(to)f(the)g(amoun)o(t)75 749 y(of)j(data)g (receiv)o(ed,)i(pairwise)f(b)q(et)o(w)o(een)g(eac)o(h)g(pro)q(cess)g (and)g(the)g(ro)q(ot.)k(Distinct)c(t)o(yp)q(e)g(maps)g(b)q(et)o(w)o (een)75 806 y(sender)g(and)f(receiv)o(er)h(are)f(still)i(allo)o(w)o (ed.)166 862 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p Fx(,)e(while)j(on)e (other)g(pro)q(cesses,)75 919 y(only)19 b(argumen)o(ts)f Fv(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,)d(comm)13 b Fx(are)19 b(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 975 y Fv(ro)q(ot)c Fx(and)g Fv(comm)10 b Fx(m)o(ust)k(ha)o(v)o(e)h (iden)o(tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)166 1031 y(The)f(sp)q(eci\014cation)h(of)f(coun)o(ts)f(and)h(t)o(yp)q(es)f (should)i(not)e(cause)h(an)o(y)g(lo)q(cation)g(on)g(the)f(ro)q(ot)g(to) g(b)q(e)75 1088 y(written)g(more)g(than)g(once.)20 b(Suc)o(h)c(a)f (call)h(is)g(erroneous.)166 1144 y(Note)e(that)f(the)h Fv(recvcount)i Fx(argumen)o(t)d(at)h(the)g(ro)q(ot)f(indicates)j(the)e (n)o(um)o(b)q(er)h(of)e(items)i(it)f(receiv)o(es)75 1201 y(from)g Fl(e)n(ach)i Fx(pro)q(cess,)f(not)f(the)i(total)e(n)o(um)o(b)q (er)i(of)f(items)g(it)h(receiv)o(es.)75 1352 y Fv(MPI)p 160 1352 V 16 w(GA)l(THERV\()24 b(sendbuf,)j(sendcount,)g(sendt)o(yp)q (e,)g(recvbuf,)e(recvcounts,)h(displs,)g(recvt)o(yp)q(e,)g(ro)q(ot,)75 1408 y(comm)m(\))117 1485 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1559 y(IN)171 b Fv(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1632 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 1705 y(OUT)124 b Fv(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j (bu\013er)e(\(c)o(hoice,)i(signi\014can)o(t)d(only)g(at)905 1761 y(ro)q(ot\))117 1835 y(IN)171 b Fv(recvcounts)372 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(con)o (taining)e(the)i(n)o(um-)905 1891 y(b)q(er)20 b(of)e(elemen)o(ts)h (that)f(are)i(receiv)o(ed)g(from)d(eac)o(h)i(pro)q(cess)905 1948 y(\(signi\014can)o(t)14 b(only)f(at)h(ro)q(ot\))117 2021 y(IN)171 b Fv(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es) 905 2077 y(the)h(displacemen)o(t)e(relativ)o(e)g(to)h Fi(recvbuf)i Fk(at)d(whic)o(h)h(to)g(place)905 2134 y(the)h(incoming)e (data)h(from)f(pro)q(cess)k Fb(i)d Fk(\(signi\014can)o(t)g(only)g(at) 905 2190 y(ro)q(ot\))117 2263 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)16 b(t)o(yp)q(e)g(of)f(recv)i(bu\013er)g(elemen)o(ts)f (\(signi\014can)o(t)f(only)g(at)905 2320 y(ro)q(ot\))f(\(handle\))117 2393 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(receiving)i(pro)q (cess)g(\(in)o(teger\))117 2466 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 2591 y Fu(int)23 b(MPI)p 245 2591 15 2 v 17 w(Gatherv\(void*)f(sendbuf,)h(int)h (sendcount,)e(MPI)p 1241 2591 V 17 w(Datatype)h(sendtype,)393 2647 y(void*)g(recvbuf,)g(int)h(*recvcounts,)e(int)h(*displs,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)g(recvtype,)f(int)i(root,)f(MPI)p 1249 2704 V 17 w(Comm)g(comm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 96 104 96 103 bop 75 -100 a Fx(96)708 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(GATHERV\(SENDBUF,)21 b(SENDCOUNT,)i(SENDTYPE,)g (RECVBUF,)f(RECVCOUNTS,)h(DISPLS,)393 102 y(RECVTYPE,)g(ROOT,)g(COMM,)g (IERROR\))170 158 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 214 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g (DISPLS\(*\),)h(RECVTYPE,)g(ROOT,)170 271 y(COMM,)h(IERROR)166 357 y Fv(MPI)p 251 357 14 2 v 16 w(GA)l(THERV)15 b Fx(extends)g(the)f (functionalit)o(y)h(of)e Fv(MPI)p 1126 357 V 16 w(GA)l(THER)i Fx(b)o(y)f(allo)o(wing)h(a)f(v)m(arying)g(coun)o(t)75 414 y(of)i(data)g(from)g(eac)o(h)h(pro)q(cess,)g(since)h Fv(recvcounts)g Fx(is)f(no)o(w)f(an)h(arra)o(y)l(.)23 b(It)17 b(also)f(allo)o(ws)h(more)f(\015exibili)q(t)o(y)75 470 y(as)f(to)f(where)i(the)f(data)g(is)g(placed)i(on)e(the)g(ro)q(ot,) f(b)o(y)h(pro)o(viding)h(the)g(new)f(argumen)o(t,)f Fv(displs)p Fx(.)166 527 y(The)i(outcome)g(is)h Fl(as)g(if)f Fx(eac)o(h)h(pro)q (cess,)f(including)j(the)e(ro)q(ot)e(pro)q(cess,)i(sends)f(a)g(message) g(to)g(the)75 583 y(ro)q(ot,)189 677 y Fu(MPI)p 264 677 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)p Fp(;)8 b Fu(sendcount)p Fp(;)f Fu(se)o(ndtype)p Fp(;)g Fu(roo)o(t)p Fp(;)h(:)o(::)o Fx(\))p Fp(;)75 771 y Fx(and)15 b(the)h(ro)q(ot)e(executes)i Fu(n)f Fx(receiv)o(es,)189 865 y Fu(MPI)p 264 865 V 16 w(Recv)p Fx(\()p Fu(recvbuf)9 b Fx(+)h Fu(disp)p Fx([)p Fu(i)p Fx(])f Ft(\001)h Fu(extent)p Fx(\()p Fu(recvtype)p Fx(\))p Fp(;)d Fu(recvc)o(ounts)p Fx([)o Fu(i)p Fx(])p Fp(;)g Fu(recvt)o(ype)p Fp(;)g Fu(i)p Fp(;)g(:::)o Fx(\))p Fp(:)166 958 y Fx(Messages)14 b(are)g(placed)i(in)f(the)g(receiv)o(e)g (bu\013er)g(of)f(the)g(ro)q(ot)g(pro)q(cess)h(in)g(rank)f(order,)g (that)g(is,)h(the)75 1015 y(data)e(sen)o(t)g(from)f(pro)q(cess)i Fu(j)f Fx(is)g(placed)i(in)f(the)f Fu(j)p Fx(th)g(p)q(ortion)h(of)f (the)g(receiv)o(e)h(bu\013er)g Fv(recvbuf)f Fx(on)h(pro)q(cess)75 1071 y Fv(ro)q(ot)p Fx(.)19 b(The)13 b Fu(j)p Fx(th)g(p)q(ortion)h(of)f Fv(recvbuf)h Fx(b)q(egins)g(at)f(o\013set)f Fv(displs[j])j Fx(elemen)o(ts)f(\(in)g(terms)e(of)h Fv(recvt)o(yp)q(e)p Fx(\))h(in)o(to)75 1128 y Fv(recvbuf)p Fx(.)166 1184 y(The)h(receiv)o(e)h(bu\013er)g(is)f(ignored)h(for)f(all)h(non-ro)q(ot) f(pro)q(cesses.)166 1241 y(The)i(t)o(yp)q(e)f(signature)h(implied)i(b)o (y)d Fv(sendcount,)j(sendt)o(yp)q(e)g Fx(on)e(pro)q(cess)g Fu(i)f Fx(m)o(ust)g(b)q(e)h(equal)g(to)f(the)75 1297 y(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fv(recvcounts[i],)i(recvt) o(yp)q(e)f Fx(at)f(the)g(ro)q(ot.)20 b(This)c(implies)i(that)c(the)i (amoun)o(t)75 1354 y(of)g(data)f(sen)o(t)h(m)o(ust)g(b)q(e)h(equal)g (to)e(the)i(amoun)o(t)e(of)h(data)f(receiv)o(ed,)j(pairwise)f(b)q(et)o (w)o(een)f(eac)o(h)h(pro)q(cess)75 1410 y(and)12 b(the)g(ro)q(ot.)17 b(Distinct)c(t)o(yp)q(e)f(maps)f(b)q(et)o(w)o(een)h(sender)g(and)g (receiv)o(er)g(are)g(still)h(allo)o(w)o(ed,)f(as)g(illustrated)75 1467 y(in)k(Example)g(4.6.)166 1523 y(All)f(argumen)o(ts)f(to)f(the)h (function)h(are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p Fx(,)e(while)j(on)e(other)g(pro)q(cesses,)75 1579 y(only)19 b(argumen)o(ts)f Fv(sendbuf,)k(sendcount,)g(sendt)o(yp)q (e,)g(ro)q(ot,)d(comm)13 b Fx(are)19 b(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1636 y Fv(ro)q(ot)c Fx(and)g Fv(comm)10 b Fx(m)o(ust)k(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)f(on)f (all)i(pro)q(cesses.)166 1692 y(The)d(sp)q(eci\014cation)i(of)d(coun)o (ts,)h(t)o(yp)q(es,)g(and)g(displacemen)o(ts)h(should)g(not)f(cause)g (an)o(y)f(lo)q(cation)i(on)75 1749 y(the)g(ro)q(ot)g(to)f(b)q(e)i (written)f(more)g(than)g(once.)20 b(Suc)o(h)c(a)f(call)h(is)g (erroneous.)75 1871 y Fj(4.5.1)49 b(Examples)14 b(using)j(MPI)p 640 1871 V 18 w(GA)l(THER,)g(MPI)p 963 1871 V 18 w(GA)l(THERV)75 1956 y Fq(Example)h(4.2)k Fx(Gather)15 b(100)f(in)o(ts)i(from)e(ev)o (ery)h(pro)q(cess)h(in)g(group)e(to)h(ro)q(ot.)k(See)d(\014gure)f(4.2.) 170 2063 y Fu(MPI_Comm)23 b(comm;)170 2119 y(int)h (gsize,sendarray[100];)170 2176 y(int)g(root,)f(*rbuf;)170 2232 y(...)170 2288 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2345 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 2401 y(MPI_Gather\()f(sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(100,)f (MPI_INT,)g(root,)g(comm\);)75 2508 y Fq(Example)18 b(4.3)k Fx(Previous)15 b(example)g(mo)q(di\014ed)g({)f(only)h(the)f(ro)q(ot)f (allo)q(cates)i(memory)e(for)h(the)g(receiv)o(e)75 2564 y(bu\013er.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 97 105 97 104 bop 75 -100 a Fm(4.5.)29 b(GA)l(THER)1457 b Fx(97)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 189 244 m 189 294 l gs 0.50 setgray fill gr gs col-1 s gr n 191.000 286.000 m 189.000 294.000 l 187.000 286.000 l gs 2 setlinejoin col-1 s gr n 289 244 m 259 294 l gs 0.50 setgray fill gr gs col-1 s gr n 264.831 288.169 m 259.000 294.000 l 261.401 286.111 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr $F2psEnd %%EndDocument endTexFig 186 511 a Fx(Figure)16 b(4.2:)j(The)c(ro)q(ot)g(pro)q(cess)g(gathers)f (100)h Fu(int)p Fx(s)f(from)h(eac)o(h)g(pro)q(cess)h(in)g(the)f(group.) 170 644 y Fu(MPI_Comm)23 b(comm;)170 701 y(int)h(gsize,sendarray[100];) 170 757 y(int)g(root,)f(myrank,)g(*rbuf;)170 814 y(...)170 870 y(MPI_Comm_rank\()f(comm,)i(myrank\);)170 927 y(if)g(\()g(myrank)f (==)g(root\))h({)242 983 y(MPI_Comm_size\()e(comm,)h(&gsize\);)242 1040 y(rbuf)g(=)h(\(int)f(*\)malloc\(gsize*100*sizeof\(int)o(\)\);)242 1096 y(})170 1153 y(MPI_Gather\()g(sendarray,)f(100,)i(MPI_INT,)e (rbuf,)i(100,)f(MPI_INT,)g(root,)g(comm\);)75 1309 y Fq(Example)18 b(4.4)k Fx(Do)d(the)h(same)f(as)g(the)h(previous)g (example,)h(but)f(use)g(a)f(deriv)o(ed)i(datat)o(yp)q(e.)32 b(Note)75 1365 y(that)17 b(the)h(t)o(yp)q(e)g(cannot)g(b)q(e)h(the)f (en)o(tire)g(set)g(of)g Fu(gsize*100)k(int)p Fx(s)c(since)h(t)o(yp)q(e) f(matc)o(hing)g(is)h(de\014ned)75 1422 y(pairwise)d(b)q(et)o(w)o(een)g (the)f(ro)q(ot)f(and)i(eac)o(h)f(pro)q(cess)g(in)h(the)g(gather.)170 1528 y Fu(MPI_Comm)23 b(comm;)170 1584 y(int)h(gsize,sendarray[100];) 170 1641 y(int)g(root,)f(*rbuf;)170 1697 y(MPI_Datatype)g(rtype;)170 1754 y(...)170 1810 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 1867 y(MPI_Type_contiguous\()e(100,)h(MPI_INT,)g(&rtype)g(\);)170 1923 y(MPI_Type_commit\()f(&rtype)h(\);)170 1979 y(rbuf)h(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 2036 y(MPI_Gather\()f (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(1,)f(rtype,)g(root,)g(comm\);) 75 2142 y Fq(Example)18 b(4.5)k Fx(No)o(w)d(ha)o(v)o(e)g(eac)o(h)g(pro) q(cess)h(send)f(100)g(in)o(ts)g(to)g(ro)q(ot,)g(but)g(place)h(eac)o(h)g (set)f(\(of)f(100\))75 2199 y Fl(stride)f Fx(in)o(ts)f(apart)g(at)g (receiving)i(end.)25 b(Use)17 b Fv(MPI)p 945 2199 14 2 v 16 w(GA)l(THERV)h Fx(and)f(the)f Fv(displs)j Fx(argumen)o(t)c(to)h (ac)o(hiev)o(e)75 2255 y(this)g(e\013ect.)j(Assume)d Fp(str)q(ide)c Ft(\025)h Fx(100.)19 b(See)d(\014gure)f(4.3.)170 2361 y Fu(MPI_Comm)23 b(comm;)170 2418 y(int)h(gsize,sendarray[100];) 170 2474 y(int)g(root,)f(*rbuf,)g(stride;)170 2531 y(int)h (*displs,i,*rcounts;)170 2644 y(...)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 98 106 98 105 bop 75 -100 a Fx(98)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig3.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 189 244 m 204 294 l gs col-1 s gr n 203.617 285.763 m 204.000 294.000 l 199.786 286.912 l gs 2 setlinejoin col-1 s gr n 289 244 m 289 294 l gs col-1 s gr n 291.000 286.000 m 289.000 294.000 l 287.000 286.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 511 a Fx(Figure)h(4.3:)22 b(The)16 b(ro)q(ot)g(pro)q(cess)h (gathers)f(100)g Fu(int)p Fx(s)g(from)f(eac)o(h)i(pro)q(cess)g(in)g (the)g(group,)f(eac)o(h)h(set)f(is)75 568 y(placed)g Fu(stride)f Fx(in)o(ts)g(apart.)170 700 y Fu(MPI_Comm_size\()22 b(comm,)i(&gsize\);)170 757 y(rbuf)g(=)f(\(int)h (*\)malloc\(gsize*stride*siz)o(eof\(int)o(\)\);)170 813 y(displs)f(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o(nt\)\);)170 870 y(rcounts)f(=)h(\(int)f(*\)malloc\(gsize*sizeof\(int\)\);)170 926 y(for)h(\(i=0;)f(i)f(SENDBUF\(*\),)g (RECVBUF\(*\))170 1450 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,) h(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR)166 1537 y Fv(MPI)p 251 1537 14 2 v 16 w(SCA)l(TTER)16 b Fx(is)g(the)f(in)o(v)o(erse)h(op)q (eration)f(to)g Fv(MPI)p 1097 1537 V 15 w(GA)l(THER)p Fx(.)166 1593 y(The)g(outcome)g(is)h Fl(as)g(if)f Fx(the)g(ro)q(ot)g (executed)h Fu(n)f Fx(send)h(op)q(erations,)189 1687 y Fu(MPI)p 264 1687 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)9 b Fx(+)h Fu(i)g Ft(\001)g Fu(sendcount)f Ft(\001)g Fu(extent)p Fx(\()p Fu(sendtype)p Fx(\))p Fp(;)f Fu(sendc)o(ount)p Fp(;)f Fu(send)o(type)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 1781 y Fx(and)15 b(eac)o(h)h(pro)q(cess)f(executed)h(a) f(receiv)o(e,)189 1875 y Fu(MPI)p 264 1875 V 16 w(Recv)p Fx(\()p Fu(recvbuf)p Fp(;)8 b Fu(recvcount)p Fp(;)f Fu(re)o(cvtype)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))o Fp(:)166 1968 y Fx(An)20 b(alternativ)o(e)g(description)h(is)g(that)e(the)h(ro)q(ot)f (sends)h(a)f(message)h(with)g Fv(MPI)p 1594 1968 14 2 v 16 w(Send\(sendbuf,)75 2025 y(sendcount)p Ft(\001)p Fv(n,)f(sendt)o(yp)q(e,)h(...\))p Fx(.)i(This)17 b(message)f(is)h (split)h(in)o(to)e Fu(n)g Fx(equal)i(segmen)o(ts,)e(the)g Fp(i)p Fx(th)g(segmen)o(t)g(is)75 2081 y(sen)o(t)f(to)g(the)g Fp(i)p Fx(th)g(pro)q(cess)g(in)h(the)f(group,)g(and)g(eac)o(h)g(pro)q (cess)h(receiv)o(es)g(this)g(message)e(as)h(ab)q(o)o(v)o(e.)166 2138 y(The)g(send)h(bu\013er)f(is)h(ignored)g(for)e(all)i(non-ro)q(ot)f (pro)q(cesses.)166 2194 y(The)j(t)o(yp)q(e)f(signature)h(asso)q(ciated) g(with)f Fv(sendcount,)k(sendt)o(yp)q(e)f Fx(at)d(the)h(ro)q(ot)e(m)o (ust)h(b)q(e)h(equal)h(to)75 2251 y(the)f(t)o(yp)q(e)g(signature)g (asso)q(ciated)g(with)g Fv(recvcount,)h(recvt)o(yp)q(e)g Fx(at)e(all)i(pro)q(cesses)f(\(ho)o(w)o(ev)o(er,)f(the)h(t)o(yp)q(e)75 2307 y(maps)f(ma)o(y)g(b)q(e)h(di\013eren)o(t\).)27 b(This)18 b(implies)i(that)d(the)g(amoun)o(t)g(of)g(data)g(sen)o(t)g(m)o(ust)g(b) q(e)i(equal)f(to)f(the)75 2364 y(amoun)o(t)f(of)f(data)h(receiv)o(ed,)i (pairwise)f(b)q(et)o(w)o(een)f(eac)o(h)h(pro)q(cess)f(and)h(the)f(ro)q (ot.)23 b(Distinct)17 b(t)o(yp)q(e)f(maps)75 2420 y(b)q(et)o(w)o(een)g (sender)f(and)h(receiv)o(er)g(are)e(still)j(allo)o(w)o(ed.)166 2477 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f(signi\014can)o (t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p Fx(,)e(while)j(on)e(other)g(pro)q (cesses,)75 2533 y(only)f(argumen)o(ts)f Fv(recvbuf,)i(recvcount,)g (recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)7 b Fx(are)12 b(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fv(ro)q(ot)75 2589 y Fx(and)j Fv(comm)10 b Fx(m)o(ust)k(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e (all)h(pro)q(cesses.)166 2646 y(The)g(sp)q(eci\014cation)h(of)f(coun)o (ts)f(and)h(t)o(yp)q(es)f(should)i(not)e(cause)h(an)o(y)g(lo)q(cation)g (on)g(the)f(ro)q(ot)g(to)g(b)q(e)75 2702 y(read)g(more)g(than)g(once.) -32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 104 112 104 111 bop 75 -100 a Fx(104)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fl(R)n(ationale.)79 b Fx(Though)22 b(not)g(needed,)i(the)e(last)g(restriction)g(is)g(imp)q (osed)h(so)f(as)f(to)g(ac)o(hiev)o(e)189 102 y(symmetry)16 b(with)h Fv(MPI)p 593 102 14 2 v 16 w(GA)l(THER)p Fx(,)h(where)f(the)g (corresp)q(onding)h(restriction)g(\(a)e(m)o(ultiple-write)189 158 y(restriction\))f(is)h(necessary)l(.)k(\()p Fl(End)c(of)g(r)n (ationale.)p Fx(\))75 354 y Fv(MPI)p 160 354 V 16 w(SCA)l(TTERV\()21 b(sendbuf,)j(sendcounts)q(,)g(displs,)f(sendt)o(yp)q(e,)h(recvbuf,)f (recvcount,)f(recvt)o(yp)q(e,)h(ro)q(ot,)75 411 y(comm)m(\))117 488 y Fk(IN)171 b Fv(sendbuf)428 b Fk(address)13 b(of)d(send)i (bu\013er)h(\(c)o(hoice,)f(signi\014can)o(t)e(only)h(at)g(ro)q(ot\))117 562 y(IN)171 b Fv(sendcounts)366 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(sp)q(ecifying)g(the)g(n)o(um-)905 619 y(b)q(er)15 b(of)e(elemen)o(ts)h(to)g(send)h(to)e(eac)o(h)i(pro)q (cessor)117 693 y(IN)171 b Fv(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es)905 750 y(the)20 b(displacemen)o(t)e (\(relativ)o(e)h(to)f Fi(sendbuf)j Fk(from)c(whic)o(h)i(to)905 806 y(tak)o(e)14 b(the)h(outgoing)d(data)i(to)g(pro)q(cess)h Fb(i)117 881 y Fk(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 955 y(OUT)124 b Fv(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 1030 y(IN)171 b Fv(recvcount)389 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1104 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 1179 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(sending)h(pro)q(cess)i(\(in)o(teger\))117 1253 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 1378 y Fu(int)23 b(MPI)p 245 1378 15 2 v 17 w(Scatterv\(void*)f (sendbuf,)h(int)g(*sendcounts,)g(int)g(*displs,)393 1434 y(MPI)p 468 1434 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e(int)i (recvcount,)393 1491 y(MPI)p 468 1491 V 17 w(Datatype)f(recvtype,)f (int)i(root,)f(MPI)p 1249 1491 V 17 w(Comm)g(comm\))75 1577 y(MPI)p 150 1577 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i (DISPLS,)g(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)393 1634 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 1690 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 1747 y(INTEGER)g(SENDCOUNTS\(*\),)f (DISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 1803 y(COMM,)h(IERROR)166 1889 y Fv(MPI)p 251 1889 14 2 v 16 w(SCA)l(TTERV)16 b Fx(is)g(the)f(in)o(v)o(erse)h(op)q(eration)f (to)g Fv(MPI)p 1127 1889 V 16 w(GA)l(THERV)p Fx(.)166 1946 y Fv(MPI)p 251 1946 V 16 w(SCA)l(TTERV)24 b Fx(extends)f(the)g (functionalit)o(y)g(of)g Fv(MPI)p 1193 1946 V 15 w(SCA)l(TTER)h Fx(b)o(y)f(allo)o(wing)g(a)f(v)m(arying)75 2002 y(coun)o(t)17 b(of)h(data)f(to)g(b)q(e)h(sen)o(t)f(to)g(eac)o(h)h(pro)q(cess,)g (since)h Fv(sendcounts)i Fx(is)d(no)o(w)f(an)g(arra)o(y)l(.)27 b(It)17 b(also)h(allo)o(ws)75 2059 y(more)k(\015exibilit)o(y)i(as)e(to) f(where)h(the)g(data)g(is)g(tak)o(en)g(from)f(on)h(the)g(ro)q(ot,)h(b)o (y)f(pro)o(viding)h(the)f(new)75 2115 y(argumen)o(t,)14 b Fv(displs)p Fx(.)166 2172 y(The)h(outcome)g(is)h(as)f(if)g(the)h(ro)q (ot)e(executed)i Fu(n)f Fx(send)h(op)q(erations,)189 2262 y Fu(MPI)p 264 2262 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)9 b Fx(+)h Fu(displs)p Fx([)p Fu(i)p Fx(])f Ft(\001)g Fu(extent)p Fx(\()p Fu(sendtype)p Fx(\))p Fp(;)f Fu(sen)o(dcounts)p Fx([)o Fu(i)p Fx(])p Fp(;)f Fu(sen)o(dtype)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 2353 y Fx(and)15 b(eac)o(h)h(pro)q (cess)f(executed)h(a)f(receiv)o(e,)189 2444 y Fu(MPI)p 264 2444 V 16 w(Recv)p Fx(\()p Fu(recvbuf)p Fp(;)8 b Fu(recvcount)p Fp(;)f Fu(re)o(cvtype)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))o Fp(:)166 2534 y Fx(The)15 b(send)h(bu\013er)f (is)h(ignored)g(for)e(all)i(non-ro)q(ot)f(pro)q(cesses.)166 2591 y(The)h(t)o(yp)q(e)g(signature)g(implied)i(b)o(y)e Fv(sendcount[i],)j(sendt)o(yp)q(e)g Fx(at)c(the)h(ro)q(ot)f(m)o(ust)g (b)q(e)i(equal)g(to)e(the)75 2647 y(t)o(yp)q(e)f(signature)h(implied)i (b)o(y)d Fv(recvcount,)i(recvt)o(yp)q(e)f Fx(at)f(pro)q(cess)g Fu(i)g Fx(\(ho)o(w)o(ev)o(er,)f(the)i(t)o(yp)q(e)f(maps)h(ma)o(y)e(b)q (e)75 2704 y(di\013eren)o(t\).)20 b(This)c(implies)h(that)e(the)g (amoun)o(t)f(of)h(data)g(sen)o(t)g(m)o(ust)f(b)q(e)i(equal)g(to)f(the)g (amoun)o(t)g(of)g(data)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 105 113 105 112 bop 75 -100 a Fm(4.6.)29 b(SCA)l(TTER)1413 b Fx(105)458 4 y 16577003 6630800 0 0 33943388 13682606 startTexFig 458 4 a %%BeginDocument: mycoll-fig7.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 373.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 414 m gs 1 -1 scale (sendbuf) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 89 244 l gs col-1 s gr n 91.075 251.981 m 89.000 244.000 l 94.587 250.065 l gs 2 setlinejoin col-1 s gr n 189 299 m 189 244 l gs col-1 s gr n 187.000 252.000 m 189.000 244.000 l 191.000 252.000 l gs 2 setlinejoin col-1 s gr n 249 299 m 289 244 l gs col-1 s gr n 282.677 249.294 m 289.000 244.000 l 285.912 251.646 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 138 522 a Fx(Figure)16 b(4.7:)i(The)e(ro)q(ot)e(pro)q(cess)i(scatters) e(sets)h(of)g(100)f Fu(int)p Fx(s)h(to)f(eac)o(h)i(pro)q(cess)f(in)h (the)f(group.)75 655 y(receiv)o(ed,)j(pairwise)f(b)q(et)o(w)o(een)g (eac)o(h)g(pro)q(cess)g(and)g(the)g(ro)q(ot.)23 b(Distinct)17 b(t)o(yp)q(e)g(maps)f(b)q(et)o(w)o(een)h(sender)75 711 y(and)e(receiv)o(er)h(are)f(still)i(allo)o(w)o(ed.)166 768 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f(signi\014can)o (t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p Fx(,)e(while)j(on)e(other)g(pro)q (cesses,)75 824 y(only)f(argumen)o(ts)f Fv(recvbuf,)i(recvcount,)g (recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)7 b Fx(are)12 b(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fv(ro)q(ot)75 881 y Fx(and)j Fv(comm)10 b Fx(m)o(ust)k(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e (all)h(pro)q(cesses.)166 937 y(The)e(sp)q(eci\014cation)i(of)d(coun)o (ts,)h(t)o(yp)q(es,)g(and)g(displacemen)o(ts)h(should)g(not)f(cause)g (an)o(y)f(lo)q(cation)i(on)75 994 y(the)g(ro)q(ot)g(to)f(b)q(e)i(read)f (more)g(than)g(once.)75 1115 y Fj(4.6.1)49 b(Examples)14 b(using)j(MPI)p 640 1115 15 2 v 18 w(SCA)l(TTER,)h(MPI)p 988 1115 V 18 w(SCA)l(TTERV)75 1201 y Fq(Example)g(4.11)23 b Fx(The)15 b(rev)o(erse)g(of)f(Example)i(4.2.)j(Scatter)14 b(sets)h(of)g(100)f(in)o(ts)h(from)f(the)h(ro)q(ot)f(to)h(eac)o(h)75 1258 y(pro)q(cess)g(in)h(the)g(group.)j(See)d(\014gure)g(4.7.)170 1364 y Fu(MPI_Comm)23 b(comm;)170 1420 y(int)h(gsize,*sendbuf;)170 1477 y(int)g(root,)f(rbuf[100];)170 1533 y(...)170 1590 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 1646 y(sendbuf)f(=)h(\(int)f (*\)malloc\(gsize*100*sizeof\(int)o(\)\);)170 1703 y(...)170 1759 y(MPI_Scatter\()g(sendbuf,)f(100,)i(MPI_INT,)f(rbuf,)g(100,)g (MPI_INT,)g(root,)g(comm\);)75 1915 y Fq(Example)18 b(4.12)23 b Fx(The)16 b(rev)o(erse)h(of)f(Example)i(4.5.)23 b(The)17 b(ro)q(ot)f(pro)q(cess)h(scatters)f(sets)g(of)h(100)f(in)o(ts)h(to)75 1972 y(the)d(other)g(pro)q(cesses,)g(but)g(the)g(sets)g(of)g(100)f(are) h Fl(stride)f Fx(in)o(ts)i(apart)e(in)i(the)f(sending)h(bu\013er.)20 b(Requires)75 2028 y(use)c(of)e Fv(MPI)p 290 2028 14 2 v 16 w(SCA)l(TTERV)p Fx(.)i(Assume)g Fp(str)q(ide)c Ft(\025)h Fx(100.)19 b(See)d(\014gure)g(4.8.)170 2134 y Fu(MPI_Comm)23 b(comm;)170 2191 y(int)h(gsize,*sendbuf;)170 2247 y(int)g(root,)f(rbuf[100],)g(i,)g(*displs,)g(*scounts;)170 2360 y(...)170 2473 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2530 y(sendbuf)f(=)h(\(int)f(*\)malloc\(gsize*stride*sizeof\()o (int\)\);)170 2586 y(...)170 2642 y(displs)g(=)h(\(int)g (*\)malloc\(gsize*sizeof\(i)o(nt\)\);)170 2699 y(scounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 106 114 106 113 bop 75 -100 a Fx(106)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6796570 0 0 33943388 14011514 startTexFig 458 4 a %%BeginDocument: mycoll-fig8.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 378.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 99 419 m gs 1 -1 scale (sendbuf) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 84 244 l gs col-1 s gr n 86.608 251.823 m 84.000 244.000 l 89.982 249.676 l gs 2 setlinejoin col-1 s gr n 204 304 m 189 244 l gs col-1 s gr n 189.000 252.246 m 189.000 244.000 l 192.881 251.276 l gs 2 setlinejoin col-1 s gr n 284 304 m 289 244 l gs col-1 s gr n 286.343 251.806 m 289.000 244.000 l 290.329 252.138 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 532 a Fx(Figure)f(4.8:)j(The)d(ro)q(ot)e(pro)q(cess)i(scatters)f (sets)g(of)g(100)f Fu(int)p Fx(s,)h(mo)o(ving)g(b)o(y)h Fu(stride)e Fx(in)o(ts)i(from)f(send)h(to)75 589 y(send)h(in)g(the)f (scatter.)170 710 y Fu(for)24 b(\(i=0;)f(i)g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2504 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 2591 y Fv(MPI)p 251 2591 14 2 v 16 w(ALLGA)l(THER)16 b Fx(can)g(b)q(e)h(though)o(t)e(of)g(as)h Fv(MPI)p 1047 2591 V 15 w(GA)l(THER)p Fx(,)h(but)f(where)g(all)h(pro)q(cesses)f (receiv)o(e)75 2647 y(the)e(result,)f(instead)i(of)e(just)g(the)g(ro)q (ot.)19 b(The)13 b Fu(j)p Fx(th)h(blo)q(c)o(k)g(of)f(data)g(sen)o(t)g (from)g(eac)o(h)g(pro)q(cess)h(is)g(receiv)o(ed)75 2704 y(b)o(y)h(ev)o(ery)g(pro)q(cess)h(and)f(placed)h(in)g(the)g Fu(j)p Fx(th)f(blo)q(c)o(k)h(of)e(the)i(bu\013er)f Fv(recvbuf)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 108 116 108 115 bop 75 -100 a Fx(108)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Fx(The)f(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)h Fv(sendcount,)h(sendt)o(yp)q(e)p Fx(,)h(at)c(a)h(pro)q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 102 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fv(recvcount,)g(recvt)o(yp)q(e)g Fx(at)f(an)o(y)f(other)h(pro)q(cess.) 166 158 y(The)f(outcome)g(of)g(a)g(call)h(to)f Fv(MPI)p 749 158 14 2 v 16 w(ALLGA)l(THER\(...\))19 b Fx(is)c(as)f(if)g(all)h (pro)q(cesses)g(executed)g Fu(n)f Fx(calls)i(to)147 250 y Fu(MPI_GATHER\(sendbuf,send)o(count,s)o(endtype,)o(recvbuf)o(,recvco) o(unt,)1244 306 y(recvtype,root,comm\),)75 398 y Fx(for)11 b Fu(root)23 b(=)h(0)g(,)f(...,)h(n-1)p Fx(.)18 b(The)12 b(rules)g(for)f(correct)g(usage)g(of)h Fv(MPI)p 1289 398 V 15 w(ALLGA)l(THER)h Fx(are)e(easily)i(found)75 455 y(from)h(the)i(corresp)q(onding)g(rules)g(for)e Fv(MPI)p 816 455 V 16 w(GA)l(THER)p Fx(.)75 606 y Fv(MPI)p 160 606 V 16 w(ALLGA)l(THERV\()d(sendbuf,)i(sendcount,)h(sendt)o(yp)q(e,)f (recvbuf,)f(recvcounts,)g(displs,)g(recvt)o(yp)q(e,)g(comm)m(\))117 739 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 814 y(IN)171 b Fv(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 889 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 964 y(OUT)124 b Fv(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1038 y(IN)171 b Fv(recvcounts)372 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(con)o(taining)e(the)i(n)o(um-)905 1095 y(b)q(er)15 b(of)e(elemen)o(ts)h(that)g(are)g(receiv)o(ed)h(from)e (eac)o(h)h(pro)q(cess)117 1170 y(IN)171 b Fv(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es)905 1226 y(the)d(displacemen)o (t)e(\(relativ)o(e)g(to)h Fi(recvbuf)p Fk(\))i(at)d(whic)o(h)h(to)g (place)905 1283 y(the)15 b(incoming)c(data)j(from)e(pro)q(cess)k Fb(i)117 1357 y Fk(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\)) 117 1432 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 1556 y Fu(int)23 b(MPI)p 245 1556 15 2 v 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1556 V 16 w(Datatype)g(sendtype,)393 1613 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1669 y(MPI)p 468 1669 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1669 V 17 w(Comm)h(comm\))75 1756 y(MPI)p 150 1756 V 17 w (ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNTS,)f(DISPLS,)393 1812 y(RECVTYPE,)h(COMM,)g(IERROR\))170 1869 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1925 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h (RECVTYPE,)g(COMM,)170 1982 y(IERROR)166 2068 y Fv(MPI)p 251 2068 14 2 v 16 w(ALLGA)l(THERV)11 b Fx(can)g(b)q(e)g(though)o(t)e (of)h(as)g Fv(MPI)p 1044 2068 V 16 w(GA)l(THERV)p Fx(,)h(but)g(where)f (all)h(pro)q(cesses)g(receiv)o(e)75 2125 y(the)j(result,)f(instead)i (of)e(just)g(the)g(ro)q(ot.)19 b(The)13 b Fu(j)p Fx(th)h(blo)q(c)o(k)g (of)f(data)g(sen)o(t)g(from)g(eac)o(h)g(pro)q(cess)h(is)g(receiv)o(ed) 75 2181 y(b)o(y)i(ev)o(ery)h(pro)q(cess)g(and)f(placed)i(in)f(the)g Fu(j)p Fx(th)f(blo)q(c)o(k)h(of)f(the)h(bu\013er)f Fv(recvbuf)p Fx(.)25 b(These)17 b(blo)q(c)o(ks)g(need)g(not)75 2237 y(all)f(b)q(e)g(the)f(same)g(size.)166 2294 y(The)g(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fv(sendcount,)j(sendt)o(yp)q(e)p Fx(,)f(at)d(pro)q(cess)h Fu(j)g Fx(m)o(ust)f(b)q(e)i(equal)g(to)75 2350 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fv(recvcounts[j],)h(recvt)o(yp)q(e)f Fx(at)e(an)o(y)h(other)g(pro)q (cess.)166 2407 y(The)g(outcome)g(is)h(as)f(if)g(all)h(pro)q(cesses)g (executed)g(calls)g(to)170 2499 y Fu(MPI_GATHERV\(sendbuf,sendcount)o (,sendtyp)o(e,recvb)o(uf,disp)o(ls,recvc)o(ounts,)1292 2555 y(recvtype,root,comm\),)75 2647 y Fx(for)k Fu(root)j(=)h(0)f(,)h (...,)f(n-1)p Fx(.)35 b(The)20 b(rules)h(for)e(correct)h(usage)g(of)g Fv(MPI)p 1365 2647 V 16 w(ALLGA)l(THERV)h Fx(are)f(easily)75 2704 y(found)c(from)e(the)h(corresp)q(onding)h(rules)g(for)f Fv(MPI)p 944 2704 V 16 w(GA)l(THERV)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 109 117 109 116 bop 75 -100 a Fm(4.8.)34 b(ALL-TO-ALL)18 b(SCA)l(TTER/GA)l (THER)892 b Fx(109)75 45 y Fj(4.7.1)49 b(Examples)14 b(using)j(MPI)p 640 45 15 2 v 18 w(ALLGA)l(THER,)h(MPI)p 1048 45 V 18 w(ALLGA)l(THERV)75 131 y Fq(Example)g(4.14)23 b Fx(The)18 b(all-gather)h(v)o(ersion)g(of)f(Example)i(4.2.)29 b(Using)19 b Fv(MPI)p 1434 131 14 2 v 16 w(ALLGA)l(THER)p Fx(,)g(w)o(e)f(will)75 187 y(gather)d(100)f(in)o(ts)h(from)g(ev)o(ery)g (pro)q(cess)g(in)h(the)g(group)e(to)h(ev)o(ery)g(pro)q(cess.)170 294 y Fu(MPI_Comm)23 b(comm;)170 350 y(int)h(gsize,sendarray[100];)170 407 y(int)g(*rbuf;)170 463 y(...)170 519 y(MPI_Comm_size\()e(comm,)i (&gsize\);)170 576 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o (\(int\)\);)170 632 y(MPI_Allgather\()e(sendarray,)h(100,)g(MPI_INT,)g (rbuf,)g(100,)g(MPI_INT,)g(comm\);)166 739 y Fx(After)15 b(the)g(call,)h(ev)o(ery)f(pro)q(cess)h(has)f(the)g(group-wide)h (concatenation)f(of)g(the)g(sets)g(of)g(data.)75 882 y Fn(4.8)59 b(All-to-All)20 b(Scatter/Gather)75 1078 y Fv(MPI)p 160 1078 V 16 w(ALL)l(TO)o(ALL\(sendbuf,)c(sendcount,)i (sendt)o(yp)q(e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(comm)m(\)) 117 1155 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1230 y(IN)171 b Fv(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1305 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 1380 y(OUT)124 b Fv(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 1456 y(IN)171 b Fv(recvcount)389 b Fk(n)o(um)o(b)q(er)17 b(of)f(elemen)o(ts)h(receiv)o(ed)i(from)c(an)o (y)i(pro)q(cess)i(\(in)o(te-)905 1512 y(ger\))117 1587 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\))117 1662 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 1787 y Fu(int)23 b(MPI)p 245 1787 15 2 v 17 w(Alltoall\(void*)f (sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1787 V 17 w(Datatype)f (sendtype,)393 1843 y(void*)h(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 1843 V 17 w(Datatype)h(recvtype,)393 1900 y(MPI)p 468 1900 V 17 w(Comm)g(comm\))75 1986 y(MPI)p 150 1986 V 17 w(ALLTOALL\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNT,)g(RECVTYPE,)393 2043 y(COMM,)g(IERROR\))170 2099 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2155 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 2242 y Fv(MPI)p 251 2242 14 2 v 16 w(ALL)l(TO)o(ALL)18 b Fx(is)h(an)g(extension)h(of)e Fv(MPI)p 962 2242 V 16 w(ALLGA)l(THER)i Fx(to)e(the)h(case)g(where)g(eac)o(h)g(pro)q(cess)75 2298 y(sends)e(distinct)g(data)f(to)g(eac)o(h)g(of)g(the)g(receiv)o (ers.)24 b(The)17 b Fu(j)p Fx(th)f(blo)q(c)o(k)h(sen)o(t)f(from)f(pro)q (cess)i Fu(i)f Fx(is)h(receiv)o(ed)75 2355 y(b)o(y)e(pro)q(cess)h Fu(j)f Fx(and)g(is)h(placed)g(in)g(the)f Fu(i)p Fx(th)g(blo)q(c)o(k)h (of)f Fv(recvbuf)p Fx(.)166 2411 y(The)g(t)o(yp)q(e)g(signature)g(asso) q(ciated)g(with)h Fv(sendcount,)h(sendt)o(yp)q(e)p Fx(,)h(at)c(a)h(pro) q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 2468 y(the)i(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fv(recvcount,)i(recvt)o(yp)q(e)f Fx(at)e(an)o(y)h(other)f(pro)q(cess.)28 b(This)19 b(implies)75 2524 y(that)10 b(the)h(amoun)o(t)g(of)f(data)h(sen)o(t)f(m)o(ust)h(b)q (e)h(equal)f(to)g(the)g(amoun)o(t)f(of)h(data)f(receiv)o(ed,)j (pairwise)f(b)q(et)o(w)o(een)75 2581 y(ev)o(ery)j(pair)h(of)e(pro)q (cesses.)21 b(As)15 b(usual,)g(ho)o(w)o(ev)o(er,)f(the)i(t)o(yp)q(e)f (maps)g(ma)o(y)f(b)q(e)i(di\013eren)o(t.)166 2637 y(The)d(outcome)g(is) h(as)e(if)i(eac)o(h)f(pro)q(cess)g(executed)h(a)f(send)h(to)e(eac)o(h)h (pro)q(cess)h(\(itself)f(included\))j(with)75 2694 y(a)f(call)h(to,)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 110 118 110 117 bop 75 -100 a Fx(110)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fu(MPI)p 264 45 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)9 b Fx(+)h Fu(i)g Ft(\001)g Fu(sendcount)f Ft(\001)g Fu(extent)p Fx(\()p Fu(sendtype)p Fx(\))p Fp(;)f Fu(sendc)o(ount)p Fp(;)f Fu(send)o(type)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 139 y Fx(and)15 b(a)g(receiv)o(e)h(from)f(ev)o(ery)g (other)g(pro)q(cess)g(with)h(a)f(call)h(to,)189 233 y Fu(MPI)p 264 233 V 16 w(Recv)p Fx(\()p Fu(recvbuf)9 b Fx(+)h Fu(i)g Ft(\001)g Fu(recvcount)f Ft(\001)g Fu(extent)p Fx(\()p Fu(recvtype)p Fx(\))p Fp(;)f Fu(recvc)o(ount)p Fp(;)f Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(:)166 327 y Fx(All)14 b(argumen)o(ts)d(on)h(all)i(pro)q(cesses)f(are)f (signi\014can)o(t.)20 b(The)12 b(argumen)o(t)g Fv(comm)6 b Fx(m)o(ust)12 b(ha)o(v)o(e)g(iden)o(tical)75 383 y(v)m(alues)k(on)f (all)i(pro)q(cesses.)75 534 y Fv(MPI)p 160 534 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V\(sendbuf,)13 b(sendcounts)q(,)h(sdispls,)g (sendt)o(yp)q(e,)g(recvbuf,)e(recvcounts,)i(rdispls,)e(recvt)o(yp)q(e,) 75 590 y(comm)m(\))117 668 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 743 y(IN)171 b Fv(sendcounts)366 b Fk(in)o(teger)20 b(arra)o(y)f(equal) h(to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 799 y(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(to)h(send)h(to)f(eac)o(h)g (pro)q(cessor)117 874 y(IN)171 b Fv(sdispls)452 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(j)f Fk(sp)q(eci\014es)905 931 y(the)20 b(displacemen)o(t)e (\(relativ)o(e)h(to)f Fi(sendbuf)j Fk(from)c(whic)o(h)i(to)905 987 y(tak)o(e)14 b(the)h(outgoing)d(data)i(destined)h(for)e(pro)q(cess) j Fb(j)117 1062 y Fk(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 1138 y(OUT)124 b Fv(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1213 y(IN)171 b Fv(recvcounts)372 b Fk(in)o(teger)20 b(arra)o(y)f(equal)h (to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 1269 y(maxim)n(um)d(n)o(um)o(b)q(er)j(of)f(elemen)o(ts)i(that)g(can)f(b)q(e) i(receiv)o(ed)905 1326 y(from)12 b(eac)o(h)j(pro)q(cessor)117 1401 y(IN)171 b Fv(rdispls)453 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es) 905 1457 y(the)f(displacemen)o(t)e(\(relativ)o(e)g(to)h Fi(recvbuf)i Fk(at)d(whic)o(h)h(to)g(place)905 1514 y(the)j(incoming)c (data)j(from)e(pro)q(cess)k Fb(i)117 1589 y Fk(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j (bu\013er)f(elemen)o(ts)e(\(handle\))117 1664 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 1788 y Fu(int)23 b(MPI)p 245 1788 15 2 v 17 w(Alltoallv\(void*)f (sendbuf,)h(int)g(*sendcounts,)f(int)i(*sdispls,)393 1845 y(MPI)p 468 1845 V 17 w(Datatype)f(sendtype,)f(void*)i(recvbuf,)e (int)i(*recvcounts,)393 1901 y(int)g(*rdispls,)e(MPI)p 802 1901 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1901 V 17 w(Comm)i(comm\))75 1988 y(MPI)p 150 1988 V 17 w (ALLTOALLV\(SENDBUF,)d(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h(RECVBUF,)g (RECVCOUNTS,)393 2044 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g(IERROR\))170 2101 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2157 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 2214 y(RECVTYPE,)h(COMM,)g (IERROR)166 2300 y Fv(MPI)p 251 2300 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V)12 b Fx(adds)g(\015exibili)q(t)o(y)j(to)c Fv(MPI)p 959 2300 V 16 w(ALL)l(TO)o(ALL)h Fx(in)h(that)f(the)h(lo)q(cation)g(of) f(data)g(for)g(the)75 2356 y(send)j(is)f(sp)q(eci\014ed)j(b)o(y)d Fv(sdispls)i Fx(and)e(the)g(lo)q(cation)h(of)f(the)g(placemen)o(t)h(of) f(the)g(data)g(on)g(the)g(receiv)o(e)h(side)75 2413 y(is)h(sp)q (eci\014ed)h(b)o(y)e Fv(rdispls)p Fx(.)166 2469 y(The)k Fu(j)p Fx(th)f(blo)q(c)o(k)i(sen)o(t)e(from)g(pro)q(cess)h Fu(i)g Fx(is)g(receiv)o(ed)h(b)o(y)e(pro)q(cess)h Fu(j)g Fx(and)g(is)g(placed)h(in)f(the)g Fu(i)p Fx(th)75 2526 y(blo)q(c)o(k)d(of)f Fv(recvbuf)p Fx(.)20 b(These)c(blo)q(c)o(ks)g (need)g(not)f(all)h(ha)o(v)o(e)f(the)g(same)g(size.)166 2582 y(The)i(t)o(yp)q(e)h(signature)f(asso)q(ciated)h(with)f Fv(sendcount[)q(j],)j(sendt)o(yp)q(e)g Fx(at)d(pro)q(cess)g Fu(i)g Fx(m)o(ust)g(b)q(e)h(equal)75 2639 y(to)d(the)i(t)o(yp)q(e)f (signature)g(asso)q(ciated)g(with)h Fv(recvcount[i],)g(recvt)o(yp)q(e)g Fx(at)e(pro)q(cess)i Fu(j)p Fx(.)22 b(This)17 b(implies)h(that)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 111 119 111 118 bop 75 -100 a Fm(4.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)814 b Fx(111)75 45 y(the)17 b(amoun)o(t)g(of)g(data)g(sen)o(t)g (m)o(ust)f(b)q(e)i(equal)g(to)f(the)h(amoun)o(t)e(of)h(data)g(receiv)o (ed,)h(pairwise)h(b)q(et)o(w)o(een)75 102 y(ev)o(ery)c(pair)h(of)e(pro) q(cesses.)21 b(Distinct)16 b(t)o(yp)q(e)f(maps)g(b)q(et)o(w)o(een)g (sender)h(and)f(receiv)o(er)h(are)f(still)i(allo)o(w)o(ed.)166 158 y(The)e(outcome)g(is)h(as)f(if)g(eac)o(h)h(pro)q(cess)f(sen)o(t)g (a)g(message)g(to)f(ev)o(ery)h(other)g(pro)q(cess)h(with,)189 235 y Fu(MPI)p 264 235 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)9 b Fx(+)h Fu(displs)p Fx([)p Fu(i)p Fx(])p Fp(;)e Fu(sendcount)o(s)p Fx([)p Fu(i)p Fx(])o Fp(;)g Fu(s)o(endtype)o Fp(;)g Fu(i)p Fp(;)f(:)o(::)p Fx(\))o Fp(;)75 311 y Fx(and)15 b(receiv)o(ed)i(a)e (message)f(from)h(ev)o(ery)g(other)g(pro)q(cess)g(with)h(a)f(call)h(to) 189 388 y Fu(MPI)p 264 388 V 16 w(Recv)p Fx(\()p Fu(recvbuf)9 b Fx(+)h Fu(displs)p Fx([)p Fu(i)p Fx(])p Fp(;)e Fu(recvcount)o(s)p Fx([)p Fu(i)p Fx(])o Fp(;)g Fu(r)o(ecvtype)o Fp(;)g Fu(i)p Fp(;)f(:)o(::)p Fx(\))o Fp(:)166 464 y Fx(All)14 b(argumen)o(ts)d(on)h (all)i(pro)q(cesses)f(are)f(signi\014can)o(t.)20 b(The)12 b(argumen)o(t)g Fv(comm)6 b Fx(m)o(ust)12 b(ha)o(v)o(e)g(iden)o(tical) 75 521 y(v)m(alues)k(on)f(all)i(pro)q(cesses.)189 603 y Fl(R)n(ationale.)60 b Fx(The)19 b(de\014nitions)h(of)e Fv(MPI)p 911 603 14 2 v 16 w(ALL)l(TO)o(ALL)g Fx(and)h Fv(MPI)p 1338 603 V 16 w(ALL)l(TO)o(ALL)-5 b(V)18 b Fx(giv)o(e)h(as)f (m)o(uc)o(h)189 659 y(\015exibilit)o(y)e(as)d(one)h(w)o(ould)g(ac)o (hiev)o(e)g(b)o(y)f(sp)q(ecifying)j Fu(n)d Fx(indep)q(enden)o(t,)j(p)q (oin)o(t-to-p)q(oin)o(t)e(comm)o(uni-)189 716 y(cations,)i(with)h(t)o (w)o(o)f(exceptions:)23 b(all)18 b(messages)e(use)h(the)g(same)f(datat) o(yp)q(e,)g(and)h(messages)f(are)189 772 y(scattered)e(from)h(\(or)f (gathered)h(to\))f(sequen)o(tial)j(storage.)h(\()p Fl(End)e(of)g(r)n (ationale.)p Fx(\))189 854 y Fl(A)n(dvic)n(e)k(to)h(implementors.)70 b Fx(Although)21 b(the)f(discussion)i(of)e(collectiv)o(e)i(comm)o (unication)f(in)189 911 y(terms)f(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(op) q(eration)g(implies)i(that)d(eac)o(h)g(message)g(is)i(transferred)e (directly)189 967 y(from)i(sender)h(to)f(receiv)o(er,)j(implemen)o (tations)f(ma)o(y)e(use)h(a)g(tree)f(comm)o(unication)i(pattern.)189 1024 y(Messages)12 b(can)h(b)q(e)h(forw)o(arded)e(b)o(y)h(in)o (termediate)h(no)q(des)f(where)h(they)f(are)f(split)j(\(for)d (scatter\))f(or)189 1080 y(concatenated)k(\(for)f(gather\),)g(if)i (this)f(is)h(more)f(e\016cien)o(t.)20 b(\()p Fl(End)c(of)g(advic)n(e)g (to)h(implementors.)p Fx(\))75 1220 y Fn(4.9)59 b(Global)20 b(Reduction)e(Op)r(erations)75 1321 y Fx(The)e(functions)g(in)g(this)g (section)g(p)q(erform)f(a)g(global)h(reduce)g(op)q(eration)g(\(suc)o(h) f(as)g(sum,)g(max,)g(logical)75 1378 y(AND,)d(etc.\))19 b(across)12 b(all)i(the)e(mem)o(b)q(ers)h(of)f(a)h(group.)18 b(The)13 b(reduction)h(op)q(eration)f(can)g(b)q(e)g(either)h(one)e(of) 75 1434 y(a)j(prede\014ned)i(list)f(of)f(op)q(erations,)g(or)g(a)g (user-de\014ned)j(op)q(eration.)i(The)c(global)g(reduction)g(functions) 75 1491 y(come)h(in)h(sev)o(eral)g(\015a)o(v)o(ors:)23 b(a)17 b(reduce)h(that)f(returns)g(the)g(result)h(of)f(the)g(reduction) i(at)d(one)i(no)q(de,)g(an)75 1547 y(all-reduce)i(that)e(returns)g (this)g(result)h(at)e(all)i(no)q(des,)g(and)g(a)f(scan)g(\(parallel)h (pre\014x\))f(op)q(eration.)29 b(In)75 1604 y(addition,)15 b(a)f(reduce-scatter)g(op)q(eration)h(com)o(bines)g(the)f(functionalit) o(y)h(of)f(a)g(reduce)h(and)f(of)g(a)g(scatter)75 1660 y(op)q(eration.)75 1779 y Fj(4.9.1)49 b(Reduce)75 1959 y Fv(MPI)p 160 1959 V 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d(count,)h (datat)o(yp)q(e,)h(op,)e(ro)q(ot,)f(comm)m(\))117 2036 y Fk(IN)171 b Fv(sendbuf)428 b Fk(address)15 b(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 2108 y(OUT)124 b Fv(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 2164 y(ro)q(ot\))117 2236 y(IN)171 b Fv(count)466 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 2308 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 2379 y(IN)171 b Fv(op)525 b Fk(reduce)16 b(op)q(eration)e(\(handle\)) 117 2451 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 2523 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 2647 y Fu(int)23 b(MPI)p 245 2647 15 2 v 17 w(Reduce\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)h(count,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2704 V 17 w(Op)i(op,)f(int)h (root,)f(MPI)p 1505 2704 V 17 w(Comm)g(comm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 112 120 112 119 bop 75 -100 a Fx(112)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(REDUCE\(SENDBUF,)22 b(RECVBUF,)g(COUNT,)h(DATATYPE,)g (OP,)g(ROOT,)h(COMM,)f(IERROR\))170 102 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 158 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(ROOT,)h (COMM,)f(IERROR)166 244 y Fv(MPI)p 251 244 14 2 v 16 w(REDUCE)c Fx(com)o(bines)g(the)g(elemen)o(ts)g(pro)o(vided)g(in)g(the) g(input)g(bu\013er)g(of)f(eac)o(h)g(pro)q(cess)h(in)75 301 y(the)d(group,)g(using)h(the)f(op)q(eration)g Fv(op)p Fx(,)g(and)h(returns)f(the)g(com)o(bined)h(v)m(alue)g(in)g(the)f (output)g(bu\013er)g(of)75 357 y(the)j(pro)q(cess)f(with)h(rank)f Fv(ro)q(ot)p Fx(.)30 b(The)19 b(input)g(bu\013er)g(is)g(de\014ned)h(b)o (y)e(the)h(argumen)o(ts)e Fv(sendbuf)p Fx(,)22 b Fv(count)75 414 y Fx(and)d Fv(datat)o(yp)q(e)p Fx(;)i(the)d(output)h(bu\013er)f(is) h(de\014ned)h(b)o(y)e(the)g(argumen)o(ts)g Fv(recvbuf)p Fx(,)h Fv(count)h Fx(and)f Fv(datat)o(yp)q(e)p Fx(;)75 470 y(b)q(oth)e(ha)o(v)o(e)f(the)h(same)g(n)o(um)o(b)q(er)g(of)f (elemen)o(ts,)h(with)h(the)e(same)h(t)o(yp)q(e.)24 b(The)17 b(routine)h(is)f(called)h(b)o(y)f(all)75 527 y(group)c(mem)o(b)q(ers)f (using)i(the)f(same)f(argumen)o(ts)g(for)h Fv(count,)h(datat)o(yp)q(e,) g(op,)f(ro)q(ot)g Fx(and)g Fv(comm)m Fx(.)j(Th)o(us,)d(all)75 583 y(pro)q(cesses)i(pro)o(vide)f(input)i(bu\013ers)e(and)g(output)g (bu\013ers)h(of)e(the)i(same)f(length,)h(with)f(elemen)o(ts)h(of)f(the) 75 640 y(same)g(t)o(yp)q(e.)20 b(Eac)o(h)14 b(pro)q(cess)h(can)f(pro)o (vide)h(one)g(elemen)o(t,)g(or)f(a)g(sequence)h(of)g(elemen)o(ts,)f(in) i(whic)o(h)f(case)75 696 y(the)d(com)o(bine)h(op)q(eration)g(is)f (executed)h(elemen)o(t-wise)h(on)e(eac)o(h)g(en)o(try)g(of)g(the)g (sequence.)20 b(F)l(or)11 b(example,)75 753 y(if)h(the)f(op)q(eration)h (is)g Fi(MPI)p 506 753 13 2 v 14 w(MAX)f Fx(and)g(the)h(send)g (bu\013er)f(con)o(tains)h(t)o(w)o(o)e(elemen)o(ts)i(that)e(are)h (\015oating)g(p)q(oin)o(t)75 809 y(n)o(um)o(b)q(ers)h(\()p Fv(count)h Fx(=)f(2)g(and)g Fv(datat)o(yp)q(e)i Fx(=)e Fv(MPI)p 867 809 14 2 v 16 w(FLO)o(A)l(T)p Fx(\),)f(then)h Fv(recvbuf)s Fx(\()p Fv(1)p Fx(\))h(=)g(global)8 b(max\()p Fv(sendbuf)s Fx(\()p Fv(1)p Fx(\)\))75 865 y(and)15 b Fv(recvbuf)s Fx(\()p Fv(2)p Fx(\))e(=)g(global)8 b(max\()p Fv(sendbuf)s Fx(\()p Fv(2)p Fx(\)\).)166 922 y(Sec.)31 b(4.9.2,)17 b(lists)j(the)e(set)h(of)f(prede\014ned)i(op)q(erations)f (pro)o(vided)g(b)o(y)g Fv(MPI)p Fx(.)f(That)g(section)h(also)75 978 y(en)o(umerates)d(the)g(datat)o(yp)q(es)f(eac)o(h)i(op)q(eration)f (can)g(b)q(e)h(applied)h(to.)k(In)16 b(addition,)h(users)f(ma)o(y)g (de\014ne)75 1035 y(their)i(o)o(wn)e(op)q(erations)i(that)e(can)h(b)q (e)h(o)o(v)o(erloaded)f(to)g(op)q(erate)f(on)i(sev)o(eral)f(datat)o(yp) q(es,)f(either)i(basic)75 1091 y(or)d(deriv)o(ed.)21 b(This)15 b(is)h(further)f(explained)i(in)f(Sec.)21 b(4.9.4.)166 1148 y(The)11 b(op)q(eration)g Fv(op)h Fx(is)f(alw)o(a)o(ys)f(assumed)h (to)g(b)q(e)g(asso)q(ciativ)o(e.)19 b(All)13 b(prede\014ned)f(op)q (erations)f(are)g(also)75 1204 y(assumed)f(to)g(b)q(e)h(comm)o(utativ)o (e.)17 b(Users)10 b(ma)o(y)g(de\014ne)h(op)q(erations)g(that)e(are)h (assumed)h(to)e(b)q(e)i(asso)q(ciativ)o(e,)75 1261 y(but)i(not)f(comm)o (utativ)o(e.)19 b(The)13 b(\\canonical")g(ev)m(aluation)h(order)e(of)h (a)f(reduction)i(is)f(determined)h(b)o(y)f(the)75 1317 y(ranks)18 b(of)f(the)h(pro)q(cesses)g(in)h(the)f(group.)28 b(Ho)o(w)o(ev)o(er,)17 b(the)h(implemen)o(tation)h(can)f(tak)o(e)f(adv) m(an)o(tage)g(of)75 1374 y(asso)q(ciativit)o(y)l(,)d(or)f(asso)q (ciativit)o(y)g(and)h(comm)o(utativit)o(y)f(in)h(order)f(to)f(c)o (hange)i(the)f(order)g(of)g(ev)m(aluation.)75 1430 y(This)j(ma)o(y)e(c) o(hange)h(the)g(result)g(of)g(the)g(reduction)h(for)e(op)q(erations)i (that)e(are)h(not)f(strictly)i(asso)q(ciativ)o(e)75 1486 y(and)f(comm)o(utativ)o(e,)f(suc)o(h)i(as)f(\015oating)g(p)q(oin)o(t)h (addition.)189 1593 y Fl(A)n(dvic)n(e)h(to)i(implementors.)53 b Fx(It)17 b(is)h(strongly)f(recommended)i(that)d Fv(MPI)p 1480 1593 V 16 w(REDUCE)i Fx(b)q(e)g(imple-)189 1649 y(men)o(ted)e(so)g(that)g(the)g(same)h(result)f(b)q(e)i(obtained)f (whenev)o(er)f(the)h(function)g(is)g(applied)i(on)d(the)189 1706 y(same)f(argumen)o(ts,)f(app)q(earing)i(in)g(the)f(same)g(order.) 20 b(Note)14 b(that)h(this)h(ma)o(y)e(prev)o(en)o(t)h(optimiza-)189 1762 y(tions)k(that)f(tak)o(e)h(adv)m(an)o(tage)f(of)h(the)g(ph)o (ysical)h(lo)q(cation)g(of)f(pro)q(cessors.)31 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to)189 1819 y(implementors.)p Fx(\))166 1925 y(The)c Fv(datat)o(yp)q(e)i Fx(argumen)o(t)e(of)f Fv(MPI)p 783 1925 V 16 w(REDUCE)i Fx(m)o(ust)f(b)q(e)g(compatible)i (with)e Fv(op)p Fx(.)23 b(Prede\014ned)18 b(op-)75 1981 y(erators)h(w)o(ork)g(only)i(with)f(the)g Fv(MPI)g Fx(t)o(yp)q(es)g (listed)h(in)g(Sec.)35 b(4.9.2)18 b(and)j(Sec.)35 b(4.9.3.)d (User-de\014ned)75 2038 y(op)q(erators)14 b(ma)o(y)g(op)q(erate)h(on)g (general,)g(deriv)o(ed)h(datat)o(yp)q(es.)k(In)15 b(this)h(case,)e(eac) o(h)h(argumen)o(t)g(that)f(the)75 2094 y(reduce)21 b(op)q(eration)g(is) g(applied)h(to)e(is)h(one)f(elemen)o(t)h(describ)q(ed)i(b)o(y)d(suc)o (h)h(a)f(datat)o(yp)q(e,)g(whic)o(h)i(ma)o(y)75 2151 y(con)o(tain)15 b(sev)o(eral)h(basic)g(v)m(alues.)21 b(This)15 b(is)h(further)f(explained)i(in)f(Section)h(4.9.4.)75 2272 y Fj(4.9.2)49 b(Prede\014ned)15 b(reduce)g(op)q(erations)75 2358 y Fx(The)k(follo)o(wing)h(prede\014ned)h(op)q(erations)e(are)f (supplied)k(for)c Fv(MPI)p 1238 2358 V 16 w(REDUCE)i Fx(and)f(related)g(functions)75 2415 y Fv(MPI)p 160 2415 V 16 w(ALLREDUCE)p Fx(,)14 b Fv(MPI)p 541 2415 V 15 w(REDUCE)p 732 2415 V 17 w(SCA)l(TTER)p Fx(,)h(and)f Fv(MPI)p 1144 2415 V 16 w(SCAN)p Fx(.)g(These)g(op)q(erations)g(are)g(in)o(v)o(ok)o (ed)75 2471 y(b)o(y)h(placing)i(the)e(follo)o(wing)h(in)g Fv(op)p Fx(.)117 2631 y Fi(Name)727 b Fx(Meaning)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 113 121 113 120 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fx(113)117 45 y Fi(MPI)p 194 45 13 2 v 14 w(MAX)649 b Fx(maxim)o(um)117 99 y Fi(MPI)p 194 99 V 14 w(MIN)664 b Fx(minim)o(um)117 154 y Fi(MPI)p 194 154 V 14 w(SUM)653 b Fx(sum)117 208 y Fi(MPI)p 194 208 V 14 w(PROD)626 b Fx(pro)q(duct)117 263 y Fi(MPI)p 194 263 V 14 w(LAND)632 b Fx(logical)16 b(and)117 317 y Fi(MPI)p 194 317 V 14 w(BAND)626 b Fx(bit-wise)16 b(and)117 371 y Fi(MPI)p 194 371 V 14 w(LOR)661 b Fx(logical)16 b(or)117 426 y Fi(MPI)p 194 426 V 14 w(BOR)655 b Fx(bit-wise)16 b(or)117 480 y Fi(MPI)p 194 480 V 14 w(LX)o(OR)634 b Fx(logical)16 b(xor)117 535 y Fi(MPI)p 194 535 V 14 w(BX)o(OR)628 b Fx(bit-wise)16 b(xor)117 589 y Fi(MPI)p 194 589 V 14 w(MAXLOC)569 b Fx(max)14 b(v)m(alue)j(and)e(lo)q(cation)117 643 y Fi(MPI)p 194 643 V 14 w(MINLOC)584 b Fx(min)16 b(v)m(alue)g(and)f(lo)q(cation)166 726 y(The)22 b(t)o(w)o(o)e(op)q (erations)h Fi(MPI)p 662 726 V 14 w(MINLOC)h Fx(and)f Fi(MPI)p 1024 726 V 14 w(MAXLOC)g Fx(are)g(discussed)i(separately)e(in) i(Sec.)75 783 y(4.9.3.)g(F)l(or)17 b(the)g(other)f(prede\014ned)j(op)q (erations,)e(w)o(e)g(en)o(umerate)f(b)q(elo)o(w)i(the)f(allo)o(w)o(ed)g (com)o(binations)75 839 y(of)e Fv(op)g Fx(and)g Fv(datat)o(yp)q(e)i Fx(argumen)o(ts.)i(First,)14 b(de\014ne)j(groups)d(of)h Fv(MPI)g Fx(basic)h(datat)o(yp)q(es)e(in)i(the)f(follo)o(wing)75 896 y(w)o(a)o(y)l(.)117 1056 y Fi(C)e(integer:)661 b Fv(MPI)p 1032 1056 14 2 v 16 w(INT,)124 b(MPI)p 1343 1056 V 15 w(LONG,)h(MPI)p 1698 1056 V 16 w(SHORT,)905 1112 y(MPI)p 990 1112 V 16 w(UNSIGNED)p 1229 1112 V 17 w(SHORT,)135 b(MPI)p 1626 1112 V 16 w(UNSIGNED,)905 1169 y(MPI)p 990 1169 V 16 w(UNSIGNED)p 1229 1169 V 17 w(LONG)117 1223 y Fi(F)o(o)o(rtran)13 b(integer:)561 b Fv(MPI)p 1032 1223 V 16 w(INTEGER)117 1277 y Fi(Floating)13 b(p)q(oint:)574 b Fv(MPI)p 1032 1277 V 16 w(FLO)o(A)l(T,)82 b(MPI)p 1365 1277 V 16 w(DOUBLE,)h(MPI)p 1738 1277 V 16 w(REAL,)905 1334 y(MPI)p 990 1334 V 16 w(DOUBLE)p 1185 1334 V 17 w(PRECISION)117 1388 y Fi(Logical:)696 b Fv(MPI)p 1032 1388 V 16 w(LOGICAL)117 1443 y Fi(Com)o(plex:)667 b Fv(MPI)p 1032 1443 V 16 w(COMPLEX)117 1497 y Fi(Byte:)738 b Fv(MPI)p 1032 1497 V 16 w(BYTE)166 1580 y Fx(No)o(w,)14 b(the)h(v)m(alid)i (datat)o(yp)q(es)e(for)f(eac)o(h)i(option)f(is)h(sp)q(eci\014ed)h(b)q (elo)o(w.)117 1740 y Fi(Op)778 b Fx(Allo)o(w)o(ed)16 b(T)o(yp)q(es)117 1849 y Fi(MPI)p 194 1849 13 2 v 14 w(MAX,)d(MPI)p 400 1849 V 14 w(MIN)458 b(C)13 b(integer,)i(F)o(o)o (rtran)e(integer,)i(Floating)e(p)q(oint)117 1903 y(MPI)p 194 1903 V 14 w(SUM,)g(MPI)p 396 1903 V 14 w(PROD)424 b(C)13 b(integer,)i(F)o(o)o(rtran)e(integer,)i(Floating)e(p)q(oint,)h (Com)o(plex)117 1958 y(MPI)p 194 1958 V 14 w(LAND,)g(MPI)p 418 1958 V 14 w(LOR,)f(MPI)p 612 1958 V 15 w(LX)o(OR)215 b(C)13 b(integer,)i(Logical)117 2012 y(MPI)p 194 2012 V 14 w(BAND,)e(MPI)p 423 2012 V 14 w(BOR,)g(MPI)p 623 2012 V 14 w(BX)o(OR)199 b(C)13 b(integer,)i(Byte)75 2145 y Fq(Example)j(4.15)23 b Fx(A)10 b(routine)i(that)e(computes)h(the)g (dot)f(pro)q(duct)i(of)e(t)o(w)o(o)g(v)o(ectors)g(that)g(are)g (distributed)75 2201 y(across)k(a)h(group)g(of)g(pro)q(cesses)h(and)f (returns)g(the)g(answ)o(er)g(at)g(no)q(de)g(zero.)75 2307 y Fu(SUBROUTINE)22 b(PAR_BLAS1\(m,)h(a\(m\),)g(b\(m\),)g(c,)h (comm\))75 2364 y(REAL)f(a\(m\),)g(b\(m\))167 b(!)24 b(local)f(slice)g(of)h(array)75 2420 y(REAL)f(c)382 b(!)24 b(result)f(\(at)g(node)h(zero\))75 2477 y(REAL)f(sum)75 2533 y(INTEGER)g(m,)g(comm,)h(i,)f(ierr)75 2646 y(!)h(local)f(sum)75 2703 y(sum)g(=)h(0.0)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 114 122 114 121 bop 75 -100 a Fx(114)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fu(DO)24 b(i)f(=)h(1,)g(m) 147 102 y(sum)f(=)h(sum)f(+)h(a\(i\)*b\(i\))75 158 y(END)f(DO)75 271 y(!)h(global)f(sum)75 327 y(CALL)g(MPI_REDUCE\(sum,)f(c,)i(1,)f (MPI_REAL,)g(MPI_SUM,)g(0,)g(comm,)g(ierr\))75 440 y(!)h(return)f (result)g(at)g(node)h(zero)f(\(and)g(garbage)g(at)h(the)f(other)g (nodes\))75 497 y(RETURN\(c\))75 603 y Fq(Example)18 b(4.16)23 b Fx(A)c(routine)h(that)f(computes)h(the)f(pro)q(duct)h(of)g (a)f(v)o(ector)g(and)h(an)f(arra)o(y)f(that)h(are)75 659 y(distributed)e(across)d(a)h(group)g(of)g(pro)q(cesses)g(and)h (returns)f(the)g(answ)o(er)g(at)f(no)q(de)i(zero.)75 766 y Fu(SUBROUTINE)22 b(PAR_BLAS2\(m,)h(n,)g(a\(m\),)g(b\(m,n\),)g (c\(n\),)h(comm\))75 822 y(REAL)f(a\(m\),)g(b\(m,n\))95 b(!)24 b(local)f(slice)g(of)h(array)75 879 y(REAL)f(c\(m\))286 b(!)24 b(result)75 935 y(REAL)f(sum\(m\))75 992 y(INTEGER)g(n,)g(comm,) h(i,)f(j,)h(ierr)75 1104 y(!)g(local)f(sum)75 1161 y(DO)h(j=)f(1,)h(n) 123 1217 y(sum\(j\))f(=)g(0.0)123 1274 y(DO)g(i)h(=)g(1,)f(m)170 1330 y(sum\(j\))g(=)h(sum\(j\))f(+)h(a\(i\)*b\(i,j\))123 1387 y(END)f(DO)75 1443 y(END)g(DO)75 1556 y(!)h(global)f(sum)75 1613 y(CALL)g(MPI_REDUCE\(sum,)f(c,)i(m,)f(MPI_REAL,)g(MPI_SUM,)g(0,)g (comm,)g(ierr\))75 1725 y(!)h(return)f(result)g(at)g(node)h(zero)f (\(and)g(garbage)g(at)h(the)f(other)g(nodes\))75 1782 y(RETURN\(c\))75 1904 y Fj(4.9.3)49 b(MINLOC)18 b(and)e(MAXLOC)75 1989 y Fx(The)d(op)q(erator)e Fi(MPI)p 423 1989 13 2 v 15 w(MINLOC)h Fx(is)h(used)g(to)f(compute)g(a)h(global)g(minim)o(um)g (and)g(also)f(an)h(index)h(attac)o(hed)75 2046 y(to)f(the)h(minim)o(um) h(v)m(alue.)21 b Fi(MPI)p 620 2046 V 14 w(MINLOC)13 b Fx(similarly)j(computes)e(a)f(global)i(maxim)o(um)f(and)g(index.)20 b(One)75 2102 y(application)13 b(of)e(these)h(is)g(to)f(compute)g(a)g (global)h(minim)o(um)h(\(maxim)o(um\))e(and)g(the)h(rank)f(of)g(the)h (pro)q(cess)75 2159 y(con)o(taining)k(this)g(v)m(alue.)166 2215 y(The)f(op)q(eration)h(that)e(de\014nes)i Fi(MPI)p 787 2215 V 15 w(MAXLOC)e Fx(is:)189 2280 y Fe( )242 2323 y Fp(u)247 2380 y(i)289 2280 y Fe(!)332 2352 y Ft(\016)365 2280 y Fe( )418 2323 y Fp(v)420 2380 y(j)463 2280 y Fe(!)508 2352 y Fx(=)556 2280 y Fe( )610 2323 y Fp(w)614 2380 y(k)664 2280 y Fe(!)75 2488 y Fx(where)189 2582 y Fp(w)f Fx(=)g(max\()p Fp(u;)8 b(v)r Fx(\))75 2676 y(and)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 115 123 115 122 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fx(115)189 102 y Fp(k)13 b Fx(=)274 4 y Fe(8)274 41 y(>)274 53 y(<)274 128 y(>)274 141 y(:)332 46 y Fp(i)194 b Fx(if)15 b Fp(u)e(>)g(v)332 102 y Fx(min)q(\()p Fp(i;)8 b(j)s Fx(\))39 b(if)15 b Fp(u)e Fx(=)g Fp(v)332 159 y(j)191 b Fx(if)15 b Fp(u)e(<)g(v)166 251 y Fi(MPI)p 243 251 13 2 v 14 w(MINLOC)i Fx(is)h(de\014ned)g(similarly:)189 315 y Fe( )242 359 y Fp(u)247 416 y(i)289 315 y Fe(!)332 387 y Ft(\016)365 315 y Fe( )418 359 y Fp(v)420 416 y(j)463 315 y Fe(!)508 387 y Fx(=)556 315 y Fe( )610 359 y Fp(w)614 416 y(k)664 315 y Fe(!)75 524 y Fx(where)189 618 y Fp(w)d Fx(=)g(min)q(\()p Fp(u;)8 b(v)r Fx(\))75 712 y(and)189 852 y Fp(k)13 b Fx(=)274 753 y Fe(8)274 790 y(>)274 803 y(<)274 878 y(>)274 890 y(:)332 795 y Fp(i)194 b Fx(if)15 b Fp(u)e(<)g(v)332 851 y Fx(min)q(\()p Fp(i;)8 b(j)s Fx(\))39 b(if)15 b Fp(u)e Fx(=)g Fp(v)332 908 y(j)191 b Fx(if)15 b Fp(u)e(>)g(v)166 1000 y Fx(Both)g(op)q(erations)g(are)g (asso)q(ciativ)o(e)h(and)f(comm)o(utativ)o(e.)19 b(Note)12 b(that)h(if)h Fi(MPI)p 1491 1000 V 14 w(MAXLOC)e Fx(is)i(applied)75 1057 y(to)k(reduce)i(a)e(sequence)i(of)e(pairs)h(\()p Fp(u)729 1064 y Fg(0)748 1057 y Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fp(u)875 1064 y Fg(1)893 1057 y Fp(;)g Fx(1\))p Fp(;)g(:)g(:)f(:)t(;)h Fx(\()p Fp(u)1099 1064 y Ff(n)p Fh(\000)p Fg(1)1167 1057 y Fp(;)g(n)j Ft(\000)i Fx(1\),)19 b(then)g(the)f(v)m(alue)i(returned)f(is)75 1113 y(\()p Fp(u;)8 b(r)q Fx(\),)15 b(where)i Fp(u)f Fx(=)f(max)518 1120 y Ff(i)540 1113 y Fp(u)566 1120 y Ff(i)597 1113 y Fx(and)i Fp(r)g Fx(is)g(the)g(index)i(of)d(the)h(\014rst)f(global)i (maxim)o(um)f(in)g(the)g(sequence.)75 1170 y(Th)o(us,)12 b(if)h(eac)o(h)f(pro)q(cess)g(supplies)i(a)e(v)m(alue)h(and)f(its)h (rank)e(within)j(the)e(group,)g(then)g(a)g(reduce)h(op)q(eration)75 1226 y(with)18 b Fv(op)g Fx(=)g Fi(MPI)p 375 1226 V 14 w(MAXLOC)f Fx(will)i(return)f(the)g(maxim)o(um)f(v)m(alue)i(and)f(the)g (rank)f(of)h(the)f(\014rst)h(pro)q(cess)75 1283 y(with)g(that)f(v)m (alue.)29 b(Similarly)l(,)21 b Fi(MPI)p 708 1283 V 14 w(MINLOC)d Fx(can)g(b)q(e)g(used)h(to)e(return)h(a)f(minim)o(um)i(and)f (its)g(index.)75 1339 y(More)10 b(generally)l(,)i Fi(MPI)p 463 1339 V 14 w(MINLOC)e Fx(computes)h(a)f Fl(lexic)n(o)n(gr)n(aphic)h (minimum)p Fx(,)g(where)g(elemen)o(ts)g(are)f(ordered)75 1396 y(according)k(to)g(the)g(\014rst)g(comp)q(onen)o(t)g(of)f(eac)o(h) h(pair,)h(and)f(ties)g(are)g(resolv)o(ed)g(according)h(to)e(the)h (second)75 1452 y(comp)q(onen)o(t.)166 1509 y(The)i(reduce)h(op)q (eration)f(is)h(de\014ned)g(to)e(op)q(erate)h(on)g(argumen)o(ts)f(that) g(consist)i(of)e(a)h(pair:)22 b(v)m(alue)75 1565 y(and)12 b(index.)20 b(F)l(or)12 b(b)q(oth)g(F)l(ortran)f(and)h(C,)g(t)o(yp)q (es)g(are)g(pro)o(vided)h(to)e(describ)q(e)j(the)e(pair.)19 b(The)13 b(p)q(oten)o(tially)75 1621 y(mixed-t)o(yp)q(e)g(nature)e(of)g (suc)o(h)h(argumen)o(ts)f(is)h(a)f(problem)i(in)f(F)l(ortran.)17 b(The)12 b(problem)h(is)f(circum)o(v)o(en)o(ted,)75 1678 y(for)k(F)l(ortran,)g(b)o(y)h(ha)o(ving)h(the)f Fv(MPI)p Fx(-pro)o(vided)g(t)o(yp)q(e)g(consist)g(of)g(a)g(pair)g(of)g(the)g (same)f(t)o(yp)q(e)h(as)g(v)m(alue,)75 1734 y(and)j(co)q(ercing)g(the)g (index)g(to)f(this)h(t)o(yp)q(e)f(also.)33 b(In)20 b(C,)f(the)g Fv(MPI)p Fx(-pro)o(vided)h(pair)g(t)o(yp)q(e)f(has)h(distinct)75 1791 y(t)o(yp)q(es)15 b(and)h(the)f(index)h(is)g(an)f Fv(int)p Fx(.)166 1847 y(In)f(order)g(to)f(use)h Fi(MPI)p 547 1847 V 14 w(MINLOC)f Fx(and)h Fi(MPI)p 893 1847 V 15 w(MAXLOC)e Fx(in)j(a)e(reduce)i(op)q(eration,)f(one)f(m)o(ust)g(pro) o(vide)75 1904 y(a)19 b Fv(datat)o(yp)q(e)j Fx(argumen)o(t)d(that)g (represen)o(ts)h(a)f(pair)h(\(v)m(alue)h(and)f(index\).)34 b Fv(MPI)19 b Fx(pro)o(vides)i(sev)o(en)f(suc)o(h)75 1960 y(prede\014ned)14 b(datat)o(yp)q(es.)19 b(The)13 b(op)q(erations)g Fi(MPI)p 905 1960 V 14 w(MAXLOC)f Fx(and)h Fi(MPI)p 1264 1960 V 14 w(MINLOC)f Fx(can)h(b)q(e)h(used)f(with)g(eac)o (h)75 2017 y(of)i(the)g(follo)o(wing)h(datat)o(yp)q(es.)117 2120 y Fi(F)o(o)o(rtran:)117 2175 y(Name)727 b Fx(Description)117 2229 y Fi(MPI)p 194 2229 V 14 w(2REAL)618 b Fx(pair)15 b(of)g Fu(REAL)p Fx(s)117 2283 y Fi(MPI)p 194 2283 V 14 w(2DOUBLE)p 394 2283 V 14 w(PRECISION)328 b Fx(pair)15 b(of)g Fu(DOUBLE)23 b(PRECISION)14 b Fx(v)m(ariables)117 2338 y Fi(MPI)p 194 2338 V 14 w(2INTEGER)546 b Fx(pair)15 b(of)g Fu(INTEGER)p Fx(s)117 2524 y Fi(C:)117 2579 y(Name)727 b Fx(Description)117 2633 y Fi(MPI)p 194 2633 V 14 w(FLO)o(A)m(T)p 337 2633 V 14 w(INT)529 b Fv(\015oat)15 b Fx(and)h Fv(int)117 2688 y Fi(MPI)p 194 2688 V 14 w(DOUBLE)p 373 2688 V 14 w(INT)493 b Fv(double)16 b Fx(and)g Fv(int)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 116 124 116 123 bop 75 -100 a Fx(116)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)117 45 y Fi(MPI)p 194 45 13 2 v 14 w(LONG)p 318 45 V 15 w(INT)547 b Fv(long)15 b Fx(and)g Fv(int)117 99 y Fi(MPI)p 194 99 V 14 w(2INT)651 b Fx(pair)15 b(of)g Fv(int)166 182 y Fx(The)g(datat)o(yp)q(e)g Fi(MPI)p 527 182 V 14 w(2REAL)g Fx(is)h Fl(as)g(if)f Fx(de\014ned)h(b)o(y)g(the)f(follo)o(wing)h(\(see)f(Section)h(3.12\).) 75 286 y Fu(MPI_TYPE_CONTIGUOUS\(2,)21 b(MPI_REAL,)h(MPI_2REAL\))166 389 y Fx(Similar)17 b(statemen)o(ts)d(apply)i(for)e Fi(MPI)p 823 389 V 15 w(2INTEGER)p Fx(,)f Fi(MPI)p 1134 389 V 15 w(2DOUBLE)p 1335 389 V 13 w(PRECISION)p Fx(,)g(and)i Fi(MPI)p 1750 389 V 15 w(2INT)p Fx(.)166 446 y(The)f(datat)o(yp)q(e)g Fi(MPI)p 525 446 V 14 w(FLO)o(A)m(T)p 668 446 V 14 w(INT)g Fx(is)h Fl(as)g(if)e Fx(de\014ned)j(b)o(y)e(the)g(follo)o(wing)h (sequence)g(of)e(instructions.)75 538 y Fu(type[0])23 b(=)h(MPI_FLOAT)75 594 y(type[1])f(=)h(MPI_INT)75 651 y(disp[0])f(=)h(0)75 707 y(disp[1])f(=)h(sizeof\(float\))75 763 y(block[0])f(=)g(1)75 820 y(block[1])g(=)g(1)75 876 y(MPI_TYPE_STRUCT\(2,)e(block,)i(disp,)h(type,)f(MPI_FLOAT_INT\))75 968 y Fx(Similar)17 b(statemen)o(ts)d(apply)i(for)e Fi(MPI)p 732 968 V 15 w(LONG)p 857 968 V 15 w(INT)h Fx(and)g Fi(MPI)p 1119 968 V 15 w(DOUBLE)p 1299 968 V 14 w(INT)p Fx(.)75 1072 y Fq(Example)j(4.17)23 b Fx(Eac)o(h)12 b(pro)q(cess)g(has)g(an)g (arra)o(y)f(of)h(30)g Fu(double)p Fx(s,)f(in)i(C.)f(F)l(or)g(eac)o(h)g (of)g(the)g(30)g(lo)q(cations,)75 1128 y(compute)j(the)h(v)m(alue)g (and)f(rank)g(of)g(the)g(pro)q(cess)h(con)o(taining)g(the)f(largest)g (v)m(alue.)170 1232 y Fu(...)170 1288 y(/*)24 b(each)f(process)g(has)h (an)f(array)g(of)h(30)g(double:)f(a[30])194 1345 y(*/)170 1401 y(double)g(ain[30],)g(aout[30];)170 1458 y(int)48 b(ind[30];)170 1514 y(struct)23 b({)266 1571 y(double)g(val;)266 1627 y(int)71 b(rank;)170 1684 y(})24 b(in[30],)f(out[30];)170 1740 y(int)h(i,)f(myrank,)g(root;)170 1853 y (MPI_Comm_rank\(MPI_COMM_WORLD,)d(&myrank\);)170 1909 y(for)k(\(i=0;)f(i<30;)g(++i\))h({)266 1966 y(in[i].val)f(=)g(ain[i];) 266 2022 y(in[i].rank)f(=)i(myrank;)170 2079 y(})170 2135 y(MPI_Reduce\()f(in,)g(out,)h(30,)f(MPI_DOUBLE_INT,)f(MPI_MAXLOC,) g(root,)h(comm)h(\);)170 2192 y(/*)g(At)g(this)f(point,)g(the)g(answer) g(resides)g(on)h(process)f(root)194 2248 y(*/)170 2305 y(if)h(\(myrank)f(==)h(root\))f({)266 2361 y(/*)g(read)h(ranks)f(out) 290 2418 y(*/)266 2474 y(for)g(\(i=0;)h(i<30;)f(++i\))g({)361 2530 y(aout[i])g(=)h(out[i].val;)361 2587 y(ind[i])f(=)h(out[i].rank;) 266 2643 y(})170 2700 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 117 125 117 124 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fx(117)75 45 y Fq(Example)18 b(4.18)23 b Fx(Same)15 b(example,)g(in)h(F)l(ortran.)170 151 y Fu(...)170 208 y(!)24 b(each)f(process)g(has)h(an)f(array)h(of)f(30)h(double:)f (a[30])170 321 y(DOUBLE)g(PRECISION)g(ain\(30\),)g(aout\(30\))170 377 y(INTEGER)g(ind\(30\);)170 434 y(DOUBLE)g(PRECISION)g(in\(2,30\),)g (out\(2,30\))170 490 y(INTEGER)g(i,)h(myrank,)f(root;)170 603 y(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\);)170 659 y(DO)k(I=1,)f(30)266 716 y(in\(1,i\))g(=)h(ain\(i\))266 772 y(in\(2,i\))f(=)h(myrank)94 b(!)24 b(myrank)f(is)h(coerced)e(to)i (a)g(double)170 829 y(END)g(DO)170 942 y(MPI_REDUCE\()f(in,)g(out,)h (30,)f(MPI_DOUBLE,)f(MPI_MAXLOC,)h(root,)g(comm)g(\);)170 998 y(!)h(At)g(this)f(point,)g(the)h(answer)f(resides)g(on)g(process)g (root)170 1111 y(IF)h(\(myrank)f(.EQ.)g(root\))266 1168 y(!)h(read)f(ranks)g(out)266 1224 y(DO)g(I=)h(1,)g(30)361 1280 y(aout\(i\))f(=)h(out\(1,i\))361 1337 y(ind\(i\))f(=)h(out\(2,i\)) 47 b(!)23 b(rank)h(is)f(coerced)g(back)h(to)f(an)h(integer)75 1443 y Fq(Example)18 b(4.19)23 b Fx(Eac)o(h)16 b(pro)q(cess)h(has)f(a)h (non-empt)o(y)f(arra)o(y)g(of)g(v)m(alues.)25 b(Find)18 b(the)f(minim)o(um)g(global)75 1500 y(v)m(alue,)f(the)f(rank)g(of)g (the)g(pro)q(cess)h(that)e(holds)i(it)g(and)f(its)g(index)i(on)e(this)h (pro)q(cess.)75 1606 y Fu(#define)47 b(LEN)71 b(1000)75 1719 y(float)23 b(val[LEN];)190 b(/*)23 b(local)h(array)f(of)g(values)g (*/)75 1775 y(int)g(count;)310 b(/*)23 b(local)h(number)f(of)g(values)g (*/)75 1832 y(int)g(myrank,)g(minrank,)g(minindex;)75 1888 y(float)g(minval;)75 2001 y(struct)g({)170 2058 y(float)h(value;)170 2114 y(int)72 b(index;)75 2170 y(})24 b(in,)f(out;)170 2283 y(/*)h(local)f(minloc)g(*/)75 2340 y(in.value)g(=)g(val[0];)75 2396 y(in.index)g(=)g(0;)75 2453 y(for)g(\(i=1;)h(i)f(<)h(count;)f(i++\))170 2509 y(if)h(\(in.value)f(<)g(val[i]\))g({)266 2566 y(in.value)g(=)g(val[i];) 266 2622 y(in.index)g(=)g(i;)170 2679 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 118 126 118 125 bop 75 -100 a Fx(118)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 102 y Fu(/*)24 b(global)f(minloc)g(*/)75 158 y(MPI_Comm_rank\(MPI_COMM_WOR)o(LD,)e (&myrank\);)75 214 y(in.index)i(=)g(myrank*LEN)g(+)h(in.index;)75 271 y(MPI_Reduce\()e(in,)i(out,)f(1,)h(MPI_FLOAT_INT,)e(MPI_MINLOC,)g (root,)h(comm)h(\);)170 327 y(/*)g(At)g(this)f(point,)g(the)g(answer)g (resides)g(on)h(process)f(root)194 384 y(*/)75 440 y(if)h(\(myrank)e (==)i(root\))f({)170 497 y(/*)h(read)f(answer)g(out)194 553 y(*/)170 610 y(minval)g(=)h(out.value;)170 666 y(minrank)f(=)h (out.index)f(/)g(LEN;)170 723 y(minindex)g(=)h(out.index)f(\045)g(LEN;) 75 779 y(})189 883 y Fl(R)n(ationale.)84 b Fx(The)23 b(de\014nition)h(of)e Fi(MPI)p 921 883 13 2 v 14 w(MINLOC)h Fx(and)f Fi(MPI)p 1285 883 V 15 w(MAXLOC)f Fx(giv)o(en)i(here)g(has)g (the)189 939 y(adv)m(an)o(tage)14 b(that)h(it)h(do)q(es)f(not)g (require)h(an)o(y)f(sp)q(ecial-case)j(handling)f(of)e(these)g(t)o(w)o (o)f(op)q(erations:)189 996 y(they)i(are)g(handled)i(lik)o(e)g(an)o(y)e (other)g(reduce)i(op)q(eration.)24 b(A)16 b(programmer)g(can)g(pro)o (vide)h(his)g(or)189 1052 y(her)e(o)o(wn)g(de\014nition)i(of)f Fi(MPI)p 694 1052 V 14 w(MAXLOC)e Fx(and)i Fi(MPI)p 1058 1052 V 14 w(MINLOC)p Fx(,)f(if)h(so)f(desired.)22 b(The)16 b(disadv)m(an)o(tage)189 1108 y(is)d(that)f(v)m(alues)h(and)g(indices)i (ha)o(v)o(e)d(to)g(b)q(e)h(\014rst)f(in)o(terlea)o(v)o(ed,)i(and)e (that)g(indices)j(and)e(v)m(alues)g(ha)o(v)o(e)189 1165 y(to)h(b)q(e)i(co)q(erced)g(to)f(the)g(same)g(t)o(yp)q(e,)g(in)h(F)l (ortran.)i(\()p Fl(End)e(of)g(r)n(ationale.)p Fx(\))75 1286 y Fj(4.9.4)49 b(User-De\014ned)15 b(Op)q(erations)75 1467 y Fv(MPI)p 160 1467 14 2 v 16 w(OP)p 238 1467 V 17 w(CREA)l(TE\()h(function,)g(comm)m(ute,)d(op\))117 1544 y Fk(IN)155 b Fv(function)434 b Fk(user)15 b(de\014ned)g(function) f(\(function\))117 1619 y(IN)155 b Fv(comm)n(ute)409 b Fb(true)13 b Fk(if)g(comm)o(utativ)o(e;)e Fb(false)i Fk(otherwise.)117 1693 y(OUT)108 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))75 1818 y Fu(int)23 b(MPI)p 245 1818 15 2 v 17 w(Op)p 310 1818 V 17 w(create\(MPI)p 567 1818 V 16 w(Uop)g(*function,)g(int)g(commute,)g(MPI)p 1323 1818 V 17 w(Op)g(*op\))75 1904 y(MPI)p 150 1904 V 17 w(OP)p 215 1904 V 17 w(CREATE\()g(FUNCTION,)f(COMMUTE,)h(OP,)g (IERROR\))170 1961 y(EXTERNAL)g(FUNCTION)170 2017 y(LOGICAL)g(COMMUTE) 170 2074 y(INTEGER)g(OP,)h(IERROR)166 2160 y Fv(MPI)p 251 2160 14 2 v 16 w(OP)p 329 2160 V 17 w(CREA)l(TE)15 b Fx(binds)h(a)e(user-de\014ned)j(global)e(op)q(eration)g(to)f(a)g Fv(op)h Fx(handle)h(that)e(can)g(subse-)75 2216 y(quen)o(tly)36 b(b)q(e)h(used)f(in)g Fv(MPI)p 623 2216 V 16 w(REDUCE)p Fx(,)g Fv(MPI)p 946 2216 V 16 w(ALLREDUCE)p Fx(,)f Fv(MPI)p 1348 2216 V 16 w(REDUCE)p 1540 2216 V 17 w(SCA)l(TTER)p Fx(,)i(and)75 2273 y Fv(MPI)p 160 2273 V 16 w(SCAN)p Fx(.)17 b(The)g(user-de\014ned)i(op)q(eration)e(is)g(assumed)g(to)g(b)q (e)g(asso)q(ciativ)o(e.)25 b(If)18 b Fv(comm)m(ute)d Fx(=)i Fu(true)p Fx(,)75 2329 y(then)k(the)g(op)q(eration)g(should)h(b) q(e)g(b)q(oth)e(comm)o(utativ)o(e)h(and)g(asso)q(ciativ)o(e.)37 b(If)21 b Fv(comm)m(ute)e Fx(=)i Fu(false)p Fx(,)75 2386 y(then)16 b(the)g(order)g(of)f(op)q(erations)h(is)g(\014xed)h(and)f(is) g(de\014ned)h(to)e(b)q(e)i(in)f(ascending,)h(pro)q(cess)f(rank)g (order,)75 2442 y(b)q(eginning)h(with)f(pro)q(cess)f(zero.)166 2499 y Fv(function)20 b Fx(is)g(the)e(user-de\014ned)j(function,)f (whic)o(h)g(m)o(ust)e(ha)o(v)o(e)g(the)h(follo)o(wing)h(four)e(argumen) o(ts:)75 2555 y Fv(invec,)d(inoutvec,)i(len)e Fx(and)h Fv(datat)o(yp)q(e)p Fx(.)166 2612 y(The)f(ANSI-C)h(protot)o(yp)q(e)f (for)f(the)i(function)g(is)f(the)h(follo)o(wing.)75 2704 y Fu(typedef)23 b(void)g(MPI_Uop\()g(void)g(*invec,)g(void)h (*inoutvec,)e(int)h(*len,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 119 127 119 126 bop 75 -100 a Fm(4.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)814 b Fx(119)1220 45 y Fu(MPI_Datatype)23 b(*datatype\);)166 123 y Fx(The)15 b(F)l(ortran)f(declaration)i(of)f(the)g(user-de\014ned) i(function)f(app)q(ears)g(b)q(elo)o(w.)75 200 y Fu(FUNCTION)23 b(UOP\()g(INVEC\(*\),)g(INOUTVEC\(*\),)f(LEN,)h(TYPE\))75 256 y()g(INVEC\(LEN\),)f(INOUTVEC\(LEN\))99 313 y(INTEGER)h(LEN,)g(TYPE)166 390 y Fx(The)18 b Fv(datat)o(yp)q(e)h Fx(argumen)o(t)e(is)h(a)f(handle)i(to)e(the)h(data)e(t)o(yp)q(e)i(that) f(w)o(as)g(passed)h(in)o(to)f(the)h(call)h(to)75 447 y Fv(MPI)p 160 447 14 2 v 16 w(REDUCE)p Fx(.)f(The)g(user)g(reduce)h (function)g(should)g(b)q(e)f(written)g(suc)o(h)h(that)e(the)h(follo)o (wing)h(holds:)75 503 y(Let)h Fv(u[0],)g(...)32 b(,)20 b(u[len-1])g Fx(b)q(e)g(the)g Fv(len)g Fx(elemen)o(ts)g(in)g(the)g (comm)o(unication)g(bu\013er)f(describ)q(ed)j(b)o(y)d(the)75 560 y(argumen)o(ts)14 b Fv(invec,)i(len)f Fx(and)g Fv(datat)o(yp)q(e)i Fx(when)f(the)f(function)h(is)f(in)o(v)o(ok)o(ed;)g(let)h Fv(v[0],)e(...)19 b(,)c(v[len-1])h Fx(b)q(e)f Fv(len)75 616 y Fx(elemen)o(ts)e(in)g(the)f(comm)o(unication)h(bu\013er)f (describ)q(ed)i(b)o(y)e(the)g(argumen)o(ts)f Fv(inoutvec,)j(len)e Fx(and)g Fv(datat)o(yp)q(e)75 673 y Fx(when)17 b(the)g(function)g(is)h (in)o(v)o(ok)o(ed;)f(let)g Fv(w[0],)g(...)23 b(,)17 b(w[len-1])g Fx(b)q(e)g Fv(len)h Fx(elemen)o(ts)f(in)g(the)g(comm)o(unication)75 729 y(bu\013er)i(describ)q(ed)j(b)o(y)d(the)h(argumen)o(ts)e Fv(inoutvec,)k(len)e Fx(and)g Fv(datat)o(yp)q(e)h Fx(when)f(the)f (function)h(returns;)75 786 y(then)d Fv(w[i])f(=)h(u[i])p Ft(\016)p Fv(v[i])p Fx(,)g(for)f Fv(i=0)g(,)g(...)23 b(,)16 b(len-1)p Fx(,)h(where)g Ft(\016)f Fx(is)h(the)f(reduce)h(op)q (eration)g(that)f(the)g(function)75 842 y(computes.)166 898 y(Informally)l(,)j(w)o(e)e(can)g(think)h(of)f Fv(invec)h Fx(and)g Fv(inoutvec)h Fx(as)e(arra)o(ys)f(of)h Fv(len)h Fx(elemen)o(ts)g(that)e Fv(function)75 955 y Fx(is)i(com)o(bining.)28 b(The)18 b(result)g(of)f(the)h(reduction)h(o)o(v)o(er-writes)e(v)m (alues)h(in)h Fv(inoutvec)p Fx(,)g(hence)g(the)e(name.)75 1011 y(Eac)o(h)g(in)o(v)o(o)q(cation)g(of)f(the)h(function)h(results)f (in)h(the)f(p)q(oin)o(t)o(wise)h(ev)m(aluation)g(of)e(the)h(reduce)h (op)q(erator)75 1068 y(on)h Fv(len)h Fx(elemen)o(ts:)29 b(I.e,)21 b(the)e(function)h(returns)g(in)g Fv(inoutvec[i])h Fx(the)e(v)m(alue)i Fv(invec)p Fx([)p Fv(i)p Fx(])12 b Ft(\016)h Fv(inoutvec)p Fx([)p Fv(i)p Fx(],)21 b(for)75 1124 y Fv(i)12 b Fx(=)h Fv(0)p Fp(;)8 b(:)g(:)g(:)d(;)j Fv(count)j Ft(\000)f Fv(1)p Fx(,)15 b(where)g Ft(\016)g Fx(is)h(the)f(com)o(bining)h(op)q(eration)g(computed)f(b)o(y)g(the)h (function.)189 1208 y Fl(R)n(ationale.)47 b Fx(The)16 b(len)i(argumen)o(t)d(allo)o(ws)i Fv(MPI)p 1033 1208 V 16 w(REDUCE)g Fx(to)f(a)o(v)o(oid)g(calling)i(the)e(function)i(for) 189 1264 y(eac)o(h)13 b(elemen)o(t)h(in)g(the)f(input)h(bu\013er.)19 b(Rather,)13 b(the)h(system)e(can)h(c)o(ho)q(ose)h(to)e(apply)i(the)f (function)189 1321 y(to)k(c)o(h)o(unks)i(of)f(input.)30 b(In)19 b(C,)e(it)i(is)g(passed)f(in)h(as)f(a)g(reference)h(for)f (reasons)g(of)f(compatibilit)o(y)189 1377 y(with)e(F)l(ortran.)189 1449 y(By)e(in)o(ternally)h(comparing)f(the)g(v)m(alue)h(of)e(the)h Fv(datat)o(yp)q(e)h Fx(argumen)o(t)e(to)g(kno)o(wn,)h(global)g (handles,)189 1505 y(it)h(is)h(p)q(ossible)h(to)e(o)o(v)o(erload)g(the) g(use)h(of)f(a)g(single)h(user-de\014ned)i(function)e(for)e(sev)o (eral,)i(di\013eren)o(t)189 1562 y(data)f(t)o(yp)q(es.)20 b(\()p Fl(End)c(of)g(r)n(ationale.)p Fx(\))166 1645 y(General)e(datat)o (yp)q(es)f(ma)o(y)g(b)q(e)i(passed)f(to)f(the)g(user)h(function.)21 b(Ho)o(w)o(ev)o(er,)12 b(use)i(of)g(datat)o(yp)q(es)f(that)75 1702 y(are)i(not)g(con)o(tiguous)g(is)h(lik)o(ely)h(to)d(lead)i(to)f (ine\016ciencies.)166 1758 y(No)g Fv(MPI)g Fx(comm)o(unication)i (function)f(ma)o(y)f(b)q(e)h(called)h(inside)h(the)d(user)h(function.) 22 b Fv(MPI)p 1708 1758 V 16 w(ABORT)75 1814 y Fx(ma)o(y)14 b(b)q(e)i(called)h(inside)g(the)e(function)h(in)h(case)e(of)g(an)g (error.)189 1898 y Fl(A)n(dvic)n(e)g(to)i(users.)42 b Fx(Supp)q(ose)17 b(one)e(de\014nes)i(a)e(library)i(of)e(user-de\014ned) i(reduce)f(functions)h(that)189 1954 y(are)10 b(o)o(v)o(erloaded:)17 b(the)11 b Fv(datat)o(yp)q(e)h Fx(argumen)o(t)d(is)i(used)g(to)f (select)h(the)g(righ)o(t)f(execution)i(path)e(at)g(eac)o(h)189 2011 y(in)o(v)o(o)q(cation,)15 b(according)h(to)e(the)i(t)o(yp)q(es)f (of)g(the)g(op)q(erands.)21 b(The)15 b(user-de\014ned)i(reduce)f (function)189 2067 y(cannot)11 b(\\deco)q(de")h(the)f Fv(datat)o(yp)q(e)i Fx(argumen)o(t)d(that)h(it)h(is)f(passed,)h(and)g (cannot)f(iden)o(tify)l(,)i(b)o(y)e(itself,)189 2124 y(the)18 b(corresp)q(ondence)h(b)q(et)o(w)o(een)f(the)h(datat)o(yp)q(e) e(handles)i(and)g(the)f(datat)o(yp)q(e)f(they)h(represen)o(t.)189 2180 y(This)i(corresp)q(ondence)h(w)o(as)d(established)j(when)g(the)e (datat)o(yp)q(es)g(w)o(ere)g(created.)33 b(Before)20 b(the)189 2237 y(library)g(is)h(used,)h(a)d(library)i(initialization)i (pream)o(ble)e(m)o(ust)e(b)q(e)i(executed.)36 b(This)20 b(pream)o(ble)189 2293 y(co)q(de)14 b(will)i(de\014ne)f(the)e(datat)o (yp)q(es)h(that)f(are)g(used)i(b)o(y)f(the)g(library)l(,)g(and)g(store) f(handles)j(to)d(these)189 2349 y(datat)o(yp)q(es)i(in)h(global,)g (static)f(v)m(ariables)i(that)e(are)g(shared)h(b)o(y)f(the)h(user)g(co) q(de)g(and)g(the)f(library)189 2406 y(co)q(de.)189 2478 y(The)f(F)l(ortran)f(v)o(ersion)i(of)e Fv(MPI)p 734 2478 V 16 w(REDUCE)i Fx(will)h(in)o(v)o(ok)o(e)e(a)g(user-de\014ned)i (reduce)f(function)g(using)189 2534 y(the)g(F)l(ortran)g(calling)i(con) o(v)o(en)o(tions)f(and)g(will)h(pass)f(a)f(F)l(ortran-t)o(yp)q(e)g (datat)o(yp)q(e)g(argumen)o(t;)g(the)189 2591 y(C)d(v)o(ersion)h(will)h (use)f(C)g(calling)h(con)o(v)o(en)o(tion)f(and)g(the)f(C)h(represen)o (tation)f(of)h(a)f(datat)o(yp)q(e)g(handle.)189 2647 y(Users)i(who)h(plan)h(to)e(mix)h(languages)g(should)h(de\014ne)g (their)g(reduction)f(functions)h(accordingly)l(.)189 2704 y(\()p Fl(End)f(of)i(advic)n(e)f(to)g(users.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 120 128 120 127 bop 75 -100 a Fx(120)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fl(A)n(dvic)n(e)f(to)h (implementors.)39 b Fx(W)l(e)15 b(outline)h(b)q(elo)o(w)f(a)g(naiv)o(e) g(and)g(ine\016cien)o(t)h(implemen)o(tation)g(of)189 102 y Fv(MPI)p 274 102 14 2 v 15 w(REDUCE)p Fx(.)189 227 y Fu(if)23 b(\(rank)g(>)h(0\))g({)284 283 y(RECV\(tempbuf,)e (count,)h(datatype,)g(rank-1,...\))284 339 y(User_reduce\()f(tempbuf,)h (sendbuf,)g(count,)g(datatype\))189 396 y(})189 452 y(if)g(\(rank)g(<)h (groupsize-1\))e({)284 509 y(SEND\()h(sendbuf,)g(count,)g(datatype,)g (rank+1,)g(...\))189 565 y(})189 622 y(/*)g(answer)g(now)h(resides)f (in)g(process)g(groupsize-1)f(...)i(now)f(send)h(to)f(root)212 678 y(*/)189 735 y(if)g(\(rank)g(==)h(groupsize-1\))e({)284 791 y(SEND\()h(sendbuf,)g(count,)g(datatype,)g(root,)g(...\))189 848 y(})189 904 y(if)g(\(rank)g(==)h(root\))f({)284 960 y(RECV\(recvbuf,)f(count,)h(datatype,)g(groupsize-1,...\))189 1017 y(})189 1142 y Fx(The)10 b(reduction)h(computation)f(pro)q(ceeds,) i(sequen)o(tially)l(,)h(from)c(pro)q(cess)h Fu(0)g Fx(to)g(pro)q(cess)g Fu(groupsize-1)p Fx(.)189 1198 y(This)j(order)f(is)g(c)o(hosen)h(so)f (as)g(to)g(resp)q(ect)h(the)f(order)g(of)g(a)g(p)q(ossibly)i(non-comm)o (utativ)o(e)e(op)q(erator)189 1255 y(de\014ned)17 b(b)o(y)e(the)h (function)g Fu(User)p 765 1255 15 2 v 17 w(reduce\(\))p Fx(.)k(A)c(more)f(e\016cien)o(t)h(implemen)o(tation)h(is)f(ac)o(hiev)o (ed)189 1311 y(b)o(y)e(taking)g(adv)m(an)o(tage)g(of)f(asso)q(ciativit) o(y)i(and)f(using)h(a)f(logarithmic)h(tree)f(reduction.)21 b(Comm)o(u-)189 1368 y(tativit)o(y)d(can)h(b)q(e)h(used)f(to)g(adv)m (an)o(tage,)f(for)h(those)f(cases)h(in)h(whic)o(h)g(the)f Fv(comm)m(ute)e Fx(argumen)o(t)189 1424 y(to)g Fv(MPI)p 332 1424 14 2 v 16 w(OP)p 410 1424 V 17 w(CREA)l(TE)h Fx(is)h(true.)28 b(Also,)18 b(the)g(amoun)o(t)f(of)g(temp)q(orary)h (bu\013er)f(required)i(can)f(b)q(e)189 1481 y(reduced,)d(and)g(comm)o (unication)g(can)g(b)q(e)g(pip)q(elined)j(with)d(computation,)f(b)o(y)h (transferring)f(and)189 1537 y(reducing)i(the)f(elemen)o(ts)h(in)g(c)o (h)o(unks)g(of)f(size)h Fu(len)e Fp(<)p Fu(count)p Fx(.)189 1612 y(The)21 b(prede\014ned)h(reduce)g(op)q(erations)f(can)g(b)q(e)h (implemen)o(ted)g(as)f(a)f(library)i(of)e(user-de\014ned)189 1669 y(op)q(erations.)g(Ho)o(w)o(ev)o(er,)14 b(b)q(etter)h(p)q (erformance)g(migh)o(t)g(b)q(e)h(ac)o(hiev)o(ed)g(if)g Fv(MPI)p 1521 1669 V 16 w(REDUCE)g Fx(handles)189 1725 y(these)f(functions)h(as)f(a)g(sp)q(ecial)i(case.)j(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 1926 y Fv(MPI)p 160 1926 V 16 w(OP)p 238 1926 V 17 w(FREE\()f(op\))117 2003 y Fk(IN)155 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))75 2127 y Fu(int)23 b(MPI)p 245 2127 15 2 v 17 w(op)p 310 2127 V 17 w(free\()g(MPI)p 542 2127 V 17 w(Op)h(*op\))75 2214 y(MPI)p 150 2214 V 17 w(OP)p 215 2214 V 17 w(FREE\()f(OP,)g (IERROR\))170 2270 y(INTEGER)g(OP,)h(IERROR)166 2357 y Fx(Marks)12 b(a)h(user-de\014ned)i(reduction)f(op)q(eration)g(for)f (deallo)q(cation)i(and)e(sets)g Fv(op)g Fx(to)g Fi(MPI)p 1675 2357 13 2 v 14 w(OP)p 1747 2357 V 14 w(NULL)p Fx(.)75 2477 y Fv(Example)g(of)i(User-de\014ned)i(Reduce)75 2563 y Fx(It)e(is)h(time)f(for)g(an)g(example)h(of)f(user-de\014ned)i (reduction.)75 2669 y Fq(Example)h(4.20)23 b Fx(Compute)14 b(the)i(pro)q(duct)f(of)g(an)g(arra)o(y)f(of)h(complex)h(n)o(um)o(b)q (ers,)f(in)h(C.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 121 129 121 128 bop 75 -100 a Fm(4.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)814 b Fx(121)75 45 y Fu(typedef)23 b(struct)g({)170 102 y(double)g(real,imag;)75 158 y(})h(Complex;)75 271 y(/*)g(the)f(user-defined)f(function)99 327 y(*/)75 384 y(void)h(myProd\()g(Complex)g(*in,)g(Complex)g(*inout,)g(int)h(*len,)f (MPI_Datatype)f(*dptr)h(\))75 440 y({)170 497 y(int)h(i;)170 553 y(Complex)f(c;)170 666 y(for)h(\(i=0;)f(i<)h(*len;)f(++i\))g({)266 723 y(c.real)g(=)h(inout->real*in->real)d(-)528 779 y (inout->imag*in->imag;)266 835 y(c.imag)i(=)h(inout->real*in->imag)d(+) 528 892 y(inout->imag*in->real;)266 948 y(*inout)i(=)h(c;)266 1005 y(in++;)f(inout++;)170 1061 y(})75 1118 y(})75 1231 y(/*)h(and,)f(to)g(call)h(it...)99 1287 y(*/)75 1344 y(...)170 1456 y(/*)g(each)f(process)g(has)h(an)f(array)g(of)h(100)g (Complexes)194 1513 y(*/)170 1569 y(Complex)f(a[100],)g(answer[100];) 170 1626 y(MPI_Op)g(myOp;)170 1682 y(MPI_Datatype)g(ctype;)170 1795 y(/*)h(explain)f(to)h(MPI)f(how)g(type)h(Complex)f(is)g(defined) 194 1852 y(*/)170 1908 y(MPI_Type_contiguous\()f(2,)h(MPI_DOUBLE,)g (&ctype)g(\);)170 1965 y(MPI_Type_commit\()f(&ctype)h(\);)170 2021 y(/*)h(create)f(the)h(complex-product)d(user-op)194 2077 y(*/)170 2134 y(MPI_Op_create\()h(myProd,)h(True,)g(&myOp)h(\);) 170 2247 y(MPI_Reduce\()f(a,)g(answer,)g(100,)h(ctype,)f(myOp,)g(root,) g(comm)g(\);)170 2360 y(/*)h(At)g(this)f(point,)g(the)g(answer,)g (which)h(consists)e(of)i(100)f(Complexes,)194 2416 y(*)h(resides)f(on)h (process)e(root)194 2473 y(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 122 130 122 129 bop 75 -100 a Fx(122)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fj(4.9.5)49 b(All-Reduce)75 131 y Fv(MPI)17 b Fx(includes)j(v)m(arian)o(ts)e(of)f (eac)o(h)h(of)f(the)h(reduce)g(op)q(erations)g(where)g(the)g(result)g (is)g(returned)g(to)f(all)75 187 y(pro)q(cesses)i(in)h(the)f(group.)32 b Fv(MPI)18 b Fx(requires)i(that)e(all)i(pro)q(cesses)g(participating)g (in)g(these)f(op)q(erations)75 244 y(receiv)o(e)d(iden)o(tical)h (results.)75 395 y Fv(MPI)p 160 395 14 2 v 16 w(ALLREDUCE\()e(sendbuf,) i(recvbuf,)f(count,)g(datat)o(yp)q(e,)g(op,)f(comm)m(\))117 472 y Fk(IN)155 b Fv(sendbuf)444 b Fk(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 547 y(OUT)108 b Fv(recvbuf)450 b Fk(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er) g(\(c)o(hoice\))117 622 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 697 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 772 y(IN)155 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))117 847 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 972 y Fu(int)23 b(MPI)p 245 972 15 2 v 17 w(Allreduce\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 1028 y(MPI)p 468 1028 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1028 V 17 w(Op)i(op,)f(MPI)p 1266 1028 V 17 w(Comm)g(comm\))75 1115 y(MPI)p 150 1115 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g (DATATYPE,)g(OP,)g(COMM,)g(IERROR\))170 1171 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1227 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1314 y Fx(Same)18 b(as)g Fv(MPI)p 434 1314 14 2 v 15 w(REDUCE)h Fx(except)g(that)e(the)h(result)g(app)q(ears)h(in) f(the)g(receiv)o(e)h(bu\013er)f(of)g(all)h(the)75 1370 y(group)c(mem)o(b)q(ers.)189 1476 y Fl(A)n(dvic)n(e)20 b(to)h(implementors.)71 b Fx(The)21 b(all-reduce)h(op)q(erations)f(can) f(b)q(e)i(implemen)o(ted)g(as)e(a)g(re-)189 1533 y(duce,)d(follo)o(w)o (ed)h(b)o(y)e(a)h(broadcast.)24 b(Ho)o(w)o(ev)o(er,)16 b(a)g(direct)i(implemen)o(tation)g(can)f(lead)h(to)e(b)q(etter)189 1589 y(p)q(erformance.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (implementors.)p Fx(\))75 1695 y Fq(Example)i(4.21)23 b Fx(A)c(routine)h(that)f(computes)h(the)f(pro)q(duct)h(of)g(a)f(v)o (ector)g(and)h(an)f(arra)o(y)f(that)h(are)75 1751 y(distributed)13 b(across)d(a)h(group)g(of)g(pro)q(cesses)h(and)f(returns)g(the)h(answ)o (er)e(at)h(all)h(no)q(des)g(\(see)f(also)g(Example)75 1808 y(4.16\).)75 1913 y Fu(SUBROUTINE)22 b(PAR_BLAS2\(m,)h(n,)g (a\(m\),)g(b\(m,n\),)g(c\(n\),)h(comm\))75 1970 y(REAL)f(a\(m\),)g (b\(m,n\))95 b(!)24 b(local)f(slice)g(of)h(array)75 2026 y(REAL)f(c\(m\))286 b(!)24 b(result)75 2083 y(REAL)f(sum\(m\))75 2139 y(INTEGER)g(n,)g(comm,)h(i,)f(j,)h(ierr)75 2252 y(!)g(local)f(sum)75 2308 y(DO)h(j=)f(1,)h(n)123 2365 y(sum\(j\))f(=)g(0.0)123 2421 y(DO)g(i)h(=)g(1,)f(m)170 2478 y(sum\(j\))g(=)h(sum\(j\))f(+)h(a\(i\)*b\(i,j\))123 2534 y(END)f(DO)75 2591 y(END)g(DO)75 2704 y(!)h(global)f(sum)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 123 131 123 130 bop 75 -100 a Fm(4.10.)34 b(REDUCE-SCA)l(TTER)1173 b Fx(123)75 45 y Fu(CALL)23 b(MPI_ALLREDUCE\(sum,)f(c,)h(m,)h (MPI_REAL,)f(MPI_SUM,)f(0,)i(comm,)f(ierr\))75 158 y(!)h(return)f (result)g(at)g(all)h(nodes)75 214 y(RETURN\(c\))75 358 y Fn(4.10)59 b(Reduce-Scatte)o(r)75 459 y Fv(MPI)16 b Fx(includes)k(v)m(arian)o(ts)d(of)f(eac)o(h)h(of)g(the)g(reduce)h(op)q (erations)f(where)g(the)g(result)h(is)f(scattered)g(to)f(all)75 516 y(pro)q(cesses)g(in)g(the)f(group)g(on)g(return.)75 667 y Fv(MPI)p 160 667 14 2 v 16 w(REDUCE)p 352 667 V 17 w(SCA)l(TTER\()h(sendbuf,)h(recvbuf,)f(recvcounts,)g(datat)o(yp)q (e,)h(op,)e(comm)m(\))117 744 y Fk(IN)155 b Fv(sendbuf)444 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 819 y(OUT)108 b Fv(recvbuf)450 b Fk(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 894 y(IN)155 b Fv(recvcounts)388 b Fk(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g (the)h(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(in)f(re-)905 950 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 1007 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 1082 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 1157 y(IN)155 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))117 1232 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 1357 y Fu(int)23 b(MPI)p 245 1357 15 2 v 17 w(Reduce)p 406 1357 V 17 w(scatter\(void*)f(sendbuf,)g(void*)i (recvbuf,)e(int)i(*recvcounts,)393 1413 y(MPI)p 468 1413 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1413 V 17 w(Op)i(op,)f(MPI)p 1266 1413 V 17 w(Comm)g(comm\))75 1500 y(MPI)p 150 1500 V 17 w(REDUCE)p 311 1500 V 16 w(SCATTER\(SENDBUF,)f(RECVBUF,)h (RECVCOUNTS,)f(DATATYPE,)h(OP,)g(COMM,)393 1556 y(IERROR\))170 1613 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1669 y(INTEGER)g(RECVCOUNTS,)g(DATATYPE,)f(OP,)i(COMM,)f(IERROR)166 1756 y Fv(MPI)p 251 1756 14 2 v 16 w(REDUCE)p 443 1756 V 17 w(SCA)l(TTER)g Fx(\014rst)e(do)q(es)h(an)g(elemen)o(t-wise)h (reduction)g(on)e(v)o(ector)g(of)h Fv(count)j Fx(=)75 1780 y Fe(P)119 1825 y Fi(i)138 1812 y Fv(recvcounts)p Fx([)p Fv(i)p Fx(])18 b(elemen)o(ts)g(in)f(the)g(send)h(bu\013er)f (de\014ned)h(b)o(y)f Fv(sendbuf,)i(count)g Fx(and)e Fv(datat)o(yp)q(e)p Fx(.)26 b(Next,)75 1868 y(the)19 b(resulting)h(v)o(ector)e(of)g (results)h(is)h(split)g(in)o(to)e Fu(n)h Fx(disjoin)o(t)h(segmen)o(ts,) e(where)h Fu(n)g Fx(is)g(the)g(n)o(um)o(b)q(er)g(of)75 1925 y(mem)o(b)q(ers)c(in)h(the)f(group.)k(Segmen)o(t)c Fu(i)g Fx(con)o(tains)g Fv(recvcounts[i])h Fx(elemen)o(ts.)21 b(The)15 b Fu(i)p Fx(th)f(segmen)o(t)h(is)g(sen)o(t)75 1981 y(to)g(pro)q(cess)g Fu(i)g Fx(and)g(stored)g(in)h(the)f(receiv)o (e)i(bu\013er)e(de\014ned)h(b)o(y)f Fv(recvbuf,)h(recvcounts[i])h Fx(and)e Fv(datat)o(yp)q(e)p Fx(.)189 2088 y Fl(A)n(dvic)n(e)d(to)h (implementors.)37 b Fx(The)12 b Fv(MPI)p 875 2088 V 16 w(REDUCE)p 1067 2088 V 17 w(SCA)l(TTER)h Fx(routine)f(is)g (functionally)i(equiv)m(a-)189 2144 y(len)o(t)c(to:)17 b(A)10 b Fv(MPI)p 473 2144 V 16 w(REDUCE)h Fx(op)q(eration)g(function)g (with)g Fv(count)g Fx(equal)g(to)f(the)g(sum)g(of)g Fv(recvcounts[i]) 189 2201 y Fx(follo)o(w)o(ed)17 b(b)o(y)g Fv(MPI)p 516 2201 V 16 w(SCA)l(TTERV)h Fx(with)f Fv(sendcounts)k Fx(equal)c(to)g Fv(recvcounts)p Fx(.)27 b(Ho)o(w)o(ev)o(er,)16 b(a)g(direct)189 2257 y(implemen)o(tation)g(ma)o(y)f(run)g(faster.)k(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 124 132 124 131 bop 75 -100 a Fx(124)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fn(4.11)59 b(Scan)75 241 y Fv(MPI)p 160 241 14 2 v 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h(op,)e(comm)9 b(\))117 318 y Fk(IN)155 b Fv(sendbuf)444 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 393 y(OUT)108 b Fv(recvbuf)450 b Fk(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 469 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input) g(bu\013er)g(\(in)o(teger\))117 544 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 619 y(IN)155 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))117 694 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 818 y Fu(int)23 b(MPI)p 245 818 15 2 v 17 w(Scan\(void*)g(sendbuf,)f(void*)i(recvbuf,)e(int)i(count,)393 875 y(MPI)p 468 875 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 875 V 17 w(Op)i(op,)f(MPI)p 1266 875 V 17 w(Comm)g(comm)h(\))75 961 y(MPI)p 150 961 V 17 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 1018 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1074 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1161 y Fv(MPI)p 251 1161 14 2 v 16 w(SCAN)15 b Fx(is)h(used)f(to)g(p)q(erform)f(a)h(pre\014x)g(reduction)h(on)f (data)f(distributed)i(across)f(the)g(group.)75 1217 y(The)k(op)q (eration)g(returns,)h(in)f(the)g(receiv)o(e)h(bu\013er)f(of)f(the)h (pro)q(cess)g(with)h(rank)e Fu(i)p Fx(,)i(the)f(reduction)g(of)75 1274 y(the)h(v)m(alues)h(in)g(the)f(send)h(bu\013ers)e(of)h(pro)q (cesses)g(with)h(ranks)e Fu(0,...,i)g Fx(\(inclusiv)o(e\).)36 b(The)20 b(t)o(yp)q(e)g(of)75 1330 y(op)q(erations)c(supp)q(orted,)h (their)g(seman)o(tics,)f(and)g(the)h(constrain)o(ts)f(on)g(send)h(and)f (receiv)o(e)h(bu\013ers)f(are)75 1386 y(as)f(for)f Fv(MPI)p 285 1386 V 16 w(REDUCE)p Fx(.)189 1493 y Fl(R)n(ationale.)38 b Fx(W)l(e)13 b(ha)o(v)o(e)g(de\014ned)h(an)f(inclusiv)o(e)i(scan,)e (that)f(is,)i(the)f(pre\014x)g(reduction)h(on)f(pro)q(cess)189 1549 y Fu(i)k Fx(includes)j(the)e(data)f(from)f(pro)q(cess)i Fu(i)p Fx(.)27 b(An)18 b(alternativ)o(e)g(is)g(to)f(de\014ne)i(scan)e (in)i(an)e(exclusiv)o(e)189 1606 y(manner,)11 b(where)f(the)h(result)f (on)h Fu(i)f Fx(only)h(includes)h(data)e(up)h(to)e Fu(i-1)p Fx(.)18 b(Both)10 b(de\014nitions)i(are)e(useful.)189 1662 y(The)k(latter)g(has)g(some)g(adv)m(an)o(tages:)19 b(the)c(inclusiv)o(e)h(scan)f(can)f(alw)o(a)o(ys)g(b)q(e)h(computed)f (from)g(the)189 1719 y(exclusiv)o(e)k(scan)f(with)g(no)g(additional)h (comm)o(unication;)g(for)f(non-in)o(v)o(ertible)i(op)q(erations)e(suc)o (h)189 1775 y(as)f(max)g(and)g(min,)i(comm)o(unication)f(is)g(required) g(to)f(compute)h(the)f(exclusiv)o(e)j(scan)d(from)g(the)189 1831 y(inclusiv)o(e)i(scan.)23 b(There)16 b(is,)g(ho)o(w)o(ev)o(er,)f (a)h(complication)i(with)e(exclusiv)o(e)i(scan)e(since)h(one)g(m)o(ust) 189 1888 y(de\014ne)g(the)f(\\unit")g(elemen)o(t)h(for)e(the)h (reduction)h(in)g(this)f(case.)22 b(That)15 b(is,)i(one)f(m)o(ust)f (explicitly)189 1944 y(sa)o(y)21 b(what)h(o)q(ccurs)g(for)g(pro)q(cess) g Fu(0)p Fx(.)41 b(This)23 b(w)o(as)e(though)o(t)g(to)h(b)q(e)h (complex)g(for)e(user-de\014ned)189 2001 y(op)q(erations)15 b(and)g(hence,)h(the)g(exclusiv)o(e)g(scan)g(w)o(as)e(dropp)q(ed.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))75 2123 y Fj(4.11.1)49 b(Example)15 b(using)i(MPI)p 646 2123 15 2 v 18 w(SCAN)75 2208 y Fq(Example)h(4.22)23 b Fx(This)17 b(example)h(uses)f(a)f(user-de\014ned)j(op)q(eration)e(to)f(pro)q(duce) i(a)e Fl(se)n(gmente)n(d)g(sc)n(an)p Fx(.)75 2265 y(A)k(segmen)o(ted)g (scan)g(tak)o(es,)f(as)h(input,)h(a)f(set)f(of)h(v)m(alues)h(and)f(a)f (set)h(of)f(logicals,)j(and)e(the)g(logicals)75 2321 y(delineate)d(the)e(v)m(arious)h(segmen)o(ts)f(of)f(the)i(scan.)k(F)l (or)14 b(example:)321 2419 y Fp(v)r(al)q(ues)66 b(v)540 2426 y Fg(1)650 2419 y Fp(v)672 2426 y Fg(2)781 2419 y Fp(v)803 2426 y Fg(3)913 2419 y Fp(v)935 2426 y Fg(4)1143 2419 y Fp(v)1165 2426 y Fg(5)1323 2419 y Fp(v)1345 2426 y Fg(6)1455 2419 y Fp(v)1477 2426 y Fg(7)1587 2419 y Fp(v)1609 2426 y Fg(8)321 2476 y Fp(l)q(og)r(ical)q(s)49 b Fx(0)109 b(0)g(1)g(1)206 b(1)158 b(0)109 b(0)g(1)321 2532 y Fp(r)q(esul)q(t)76 b(v)540 2539 y Fg(1)601 2532 y Fp(v)623 2539 y Fg(1)653 2532 y Fx(+)10 b Fp(v)720 2539 y Fg(2)781 2532 y Fp(v)803 2539 y Fg(3)865 2532 y Fp(v)887 2539 y Fg(3)917 2532 y Fx(+)g Fp(v)984 2539 y Fg(4)1045 2532 y Fp(v)1067 2539 y Fg(3)1097 2532 y Fx(+)h Fp(v)1165 2539 y Fg(4)1194 2532 y Fx(+)g Fp(v)1262 2539 y Fg(5)1323 2532 y Fp(v)1345 2539 y Fg(6)1406 2532 y Fp(v)1428 2539 y Fg(6)1458 2532 y Fx(+)g Fp(v)1526 2539 y Fg(7)1587 2532 y Fp(v)1609 2539 y Fg(8)166 2631 y Fx(The)k(op)q(erator)g(that)f(pro)q(duces)i(this)g(e\013ect)f(is,) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 125 133 125 132 bop 75 -100 a Fm(4.11.)29 b(SCAN)1482 b Fx(125)711 51 y Fe( )764 95 y Fp(u)769 151 y(i)811 51 y Fe(!)854 123 y Ft(\016)887 51 y Fe( )940 95 y Fp(v)942 151 y(j)985 51 y Fe(!)1030 123 y Fx(=)1078 51 y Fe( )1132 95 y Fp(w)1138 151 y(j)1186 51 y Fe(!)1227 123 y Fp(;)166 234 y Fx(where,)735 369 y Fp(w)13 b Fx(=)829 297 y Fe(\()883 340 y Fp(u)d Fx(+)h Fp(v)43 b Fx(if)16 b Fp(i)c Fx(=)h Fp(j)883 397 y(v)125 b Fx(if)16 b Fp(i)c Ft(6)p Fx(=)h Fp(j)1203 369 y(:)166 480 y Fx(Note)20 b(that)g(this)h(is)g(a)f(non-comm)o(utativ)o (e)g(op)q(erator.)36 b(C)20 b(co)q(de)h(that)f(implemen)o(ts)i(it)e(is) h(giv)o(en)75 537 y(b)q(elo)o(w.)75 638 y Fu(typedef)i(struct)g({)170 694 y(double)g(val;)170 751 y(int)h(log;)75 807 y(})g(SegScanPair;)75 920 y(/*)g(the)f(user-defined)f(function)99 977 y(*/)75 1033 y(void)h(segScan\()g(SegScanPair)f(*in,)i(SegScanPair)e(*inout,)h (int)g(*len,)75 1090 y(MPI_Datatype)f(*dptr)h(\))75 1146 y({)170 1202 y(int)h(i;)170 1259 y(SegScanPair)f(c;)170 1372 y(for)h(\(i=0;)f(i<)h(*len;)f(++i\))g({)75 1428 y(if)h(\()f(in->log)g(==)h(inout->log)e(\))170 1485 y(c.val)i(=)f (in->val)g(+)h(inout->val;)75 1541 y(else)170 1598 y(c.val)g(=)f (inout->val;)75 1654 y(c.log)g(=)h(inout->log;)266 1711 y(*inout)f(=)h(c;)266 1767 y(in++;)f(inout++;)170 1823 y(})75 1880 y(})166 1981 y Fx(Note)c(that)f(the)i Fv(inout)g Fx(argumen)o(t)e(to)h(the)g(user-de\014ned)i(function)f(corresp)q(onds) g(to)f(the)g(righ)o(t-)75 2038 y(hand)13 b(op)q(erand)g(of)g(the)f(op)q (erator.)19 b(When)13 b(using)g(this)g(op)q(erator,)f(w)o(e)h(m)o(ust)f (b)q(e)h(careful)h(to)e(sp)q(ecify)i(that)75 2094 y(it)h(is)h(non-comm) o(utativ)o(e,)f(as)f(in)i(the)g(follo)o(wing.)170 2195 y Fu(int)24 b(i;)170 2252 y(SeqScanPair)47 b(a,)23 b(answer;)170 2308 y(MPI_Op)167 b(myOp;)170 2365 y(MPI_Datatype)23 b(type[2])g(=)g({MPI_DOUBLE,)g(MPI_INT};)170 2421 y(MPI_Aint)119 b(disp[2];)170 2478 y(int)239 b(blocklen[2])22 b(=)i({)f(1,)h(1};)170 2534 y(MPI_Datatype)f(sspair;)170 2647 y(/*)h(explain)f(to)h(MPI)f(how) g(type)h(SegScanPair)e(is)i(defined)194 2704 y(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 126 134 126 133 bop 75 -100 a Fx(126)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 45 y Fu(MPI_Address\()23 b(a,)g(disp\);)170 102 y(MPI_Address\()g(a.log,)g(disp+1\);)170 158 y(for)h(\(i=0;)f(i<2;)g(++i\))h(disp[i])f(-=)g(disp[0];)170 214 y(MPI_Type_struct\()f(2,)i(blocklen,)e(disp,)i(type,)f(&sspair)g (\);)170 271 y(MPI_Type_commit\()f(&sspair)h(\);)170 327 y(/*)h(create)f(the)h(segmented-scan)d(user-op)194 384 y(*/)170 440 y(MPI_Op_create\()h(segScan,)h(False,)g(&myOp)g(\);) 170 497 y(...)170 553 y(MPI_Scan\()g(a,)h(answer,)f(1,)g(sspair,)g (myOp,)g(root,)g(comm)h(\);)75 696 y Fn(4.12)59 b(Co)n(rrectness)75 798 y Fx(A)11 b(correct,)f(p)q(ortable)h(program)f(m)o(ust)g(in)o(v)o (ok)o(e)g(collectiv)o(e)i(comm)o(unications)g(so)e(that)g(deadlo)q(c)o (k)h(will)h(not)75 854 y(o)q(ccur,)h(whether)g(collectiv)o(e)h(comm)o (unications)g(are)e(sync)o(hronizing)i(or)e(not.)19 b(The)13 b(follo)o(wing)g(examples)75 911 y(illustrate)j(dangerous)f(use)h(of)f (collectiv)o(e)i(routines.)75 1017 y Fq(Example)h(4.23)23 b Fx(The)15 b(follo)o(wing)h(is)g(erroneous.)75 1123 y Fu(switch\(rank\))22 b({)170 1180 y(case)i(0:)266 1236 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h(comm\);)266 1293 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(1,)h(comm\);)266 1349 y(break;)170 1406 y(case)g(1:)266 1462 y(MPI_Bcast\(buf2,)e (count,)h(type,)g(1,)h(comm\);)266 1519 y(MPI_Bcast\(buf1,)e(count,)h (type,)g(0,)h(comm\);)266 1575 y(break;)75 1631 y(})166 1738 y Fx(W)l(e)19 b(assume)h(that)f(the)g(group)g(of)g Fv(comm)14 b Fx(is)20 b Ft(f)p Fx(0,1)p Ft(g)p Fx(.)31 b(Tw)o(o)19 b(pro)q(cesses)g(execute)i(t)o(w)o(o)d(broadcast)75 1794 y(op)q(erations)d(in)h(rev)o(erse)f(order.)20 b(If)15 b(the)h(op)q(eration)f(is)h(sync)o(hronizing)h(then)e(a)g(deadlo)q(c)o (k)h(will)h(o)q(ccur.)166 1851 y(Collectiv)o(e)h(op)q(erations)e(m)o (ust)g(b)q(e)g(executed)i(in)f(the)f(same)g(order)g(at)f(all)j(mem)o(b) q(ers)e(of)g(the)g(com-)75 1907 y(m)o(unication)g(group.)75 2013 y Fq(Example)i(4.24)23 b Fx(The)15 b(follo)o(wing)h(is)g (erroneous.)75 2120 y Fu(switch\(rank\))22 b({)170 2176 y(case)i(0:)266 2233 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h (comm0\);)266 2289 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(2,)h(comm2\);) 266 2345 y(break;)170 2402 y(case)g(1:)266 2458 y(MPI_Bcast\(buf1,)e (count,)h(type,)g(1,)h(comm1\);)266 2515 y(MPI_Bcast\(buf2,)e(count,)h (type,)g(0,)h(comm0\);)266 2571 y(break;)170 2628 y(case)g(2:)266 2684 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(2,)h(comm2\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 127 135 127 134 bop 75 -100 a Fm(4.12.)34 b(CORRECTNESS)1257 b Fx(127)266 45 y Fu(MPI_Bcast\(buf2,)22 b(count,)h(type,)g(1,)h (comm1\);)266 102 y(break;)75 158 y(})166 244 y Fx(Assume)13 b(that)f(the)i(group)e(of)h Fv(comm)m(0)d Fx(is)j Ft(f)p Fx(0,1)p Ft(g)p Fx(,)f(of)h Fv(comm)m(1)c Fx(is)14 b Ft(f)p Fx(1,)e(2)p Ft(g)h Fx(and)g(of)g Fv(comm)m(2)d Fx(is)j Ft(f)p Fx(2,0)p Ft(g)p Fx(.)18 b(If)75 300 y(the)12 b(broadcast)g(is)h(a)f(sync)o(hronizing)i(op)q(eration,)e(then)h(there) g(is)f(a)g(cyclic)j(dep)q(endency:)20 b(the)13 b(broadcast)75 356 y(in)i Fv(comm)m(2)c Fx(completes)k(only)f(after)g(the)g(broadcast) f(in)i Fv(comm)m(0)p Fx(;)c(the)j(broadcast)f(in)i Fv(comm)n(0)10 b Fx(completes)75 413 y(only)18 b(after)e(the)h(broadcast)g(in)h Fv(comm)m(1)p Fx(;)c(and)j(the)h(broadcast)e(in)i Fv(comm)m(1)c Fx(completes)k(only)f(after)g(the)75 469 y(broadcast)d(in)i Fv(comm)n(2)o Fx(.)h(Th)o(us,)e(the)g(co)q(de)h(will)h(deadlo)q(c)o(k.) 166 526 y(Collectiv)o(e)c(op)q(erations)f(m)o(ust)f(b)q(e)i(executed)g (in)f(an)g(order)g(so)f(that)g(no)h(cyclic)i(dep)q(endences)g(o)q (ccur.)75 611 y Fq(Example)k(4.25)23 b Fx(The)15 b(follo)o(wing)h(is)g (erroneous.)75 697 y Fu(switch\(rank\))22 b({)170 753 y(case)i(0:)266 810 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h(comm\);) 266 866 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)266 923 y(break;)170 979 y(case)h(1:)266 1036 y(MPI_Recv\(buf2,)e(count,)h (type,)g(0,)h(tag,)f(comm\);)266 1092 y(MPI_Bcast\(buf1,)f(count,)h (type,)g(0,)h(comm\);)266 1149 y(break;)75 1205 y(})166 1291 y Fx(Pro)q(cess)15 b(zero)f(executes)i(a)e(broadcast,)g(follo)o(w) o(ed)h(b)o(y)g(a)f(blo)q(c)o(king)j(send)e(op)q(eration.)20 b(Pro)q(cess)15 b(one)75 1347 y(\014rst)21 b(executes)g(a)g(blo)q(c)o (king)h(receiv)o(e)g(that)e(matc)o(hes)h(the)g(send,)h(follo)o(w)o(ed)g (b)o(y)e(broadcast)h(call)h(that)75 1403 y(matc)o(hes)14 b(the)h(broadcast)e(of)i(pro)q(cess)f(zero.)20 b(This)15 b(program)e(ma)o(y)h(deadlo)q(c)o(k.)21 b(The)15 b(broadcast)e(call)j (on)75 1460 y(pro)q(cess)g(zero)f Fl(may)g Fx(blo)q(c)o(k)h(un)o(til)h (pro)q(cess)e(one)h(executes)f(the)h(matc)o(hing)f(broadcast)f(call,)i (so)f(that)g(the)75 1516 y(send)i(is)h(not)e(executed.)26 b(Pro)q(cess)17 b(one)g(will)h(de\014nitely)h(blo)q(c)o(k)f(on)e(the)h (receiv)o(e)h(and)f(so,)f(in)i(this)f(case,)75 1573 y(nev)o(er)e (executes)h(the)f(broadcast.)166 1629 y(The)j(relativ)o(e)g(order)f(of) g(execution)h(of)f(collectiv)o(e)i(op)q(erations)f(and)g(p)q(oin)o (t-to-p)q(oin)o(t)g(op)q(erations)75 1686 y(should)f(b)q(e)h(suc)o(h,)e (so)g(that)g(ev)o(en)h(if)g(the)g(collectiv)o(e)h(op)q(erations)e(and)h (the)g(p)q(oin)o(t-to-p)q(oin)o(t)g(op)q(erations)75 1742 y(are)e(sync)o(hronizing,)h(no)f(deadlo)q(c)o(k)i(will)f(o)q (ccur.)75 1828 y Fq(Example)i(4.26)23 b Fx(A)15 b(correct,)f(but)h (non-deterministic)j(program.)75 1913 y Fu(switch\(rank\))k({)170 1970 y(case)i(0:)266 2026 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h (comm\);)266 2083 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f (comm\);)266 2139 y(break;)170 2195 y(case)h(1:)266 2252 y(MPI_Recv\(buf2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h(comm\);) 266 2308 y(MPI_Bcast\(buf1,)f(count,)h(type,)g(0,)h(comm\);)266 2365 y(MPI_Recv\(buf2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h (comm\);)266 2421 y(break;)170 2478 y(case)h(2:)266 2534 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)266 2591 y(MPI_Bcast\(buf1,)f(count,)h(type,)g(0,)h(comm\);)266 2647 y(break;)75 2704 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 128 136 128 135 bop 75 -100 a Fx(128)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)383 4 y 18945146 16482269 0 0 29733355 25918013 startTexFig 383 4 a %%BeginDocument: coll-matchings.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -50 0 translate -40.0 436.0 translate 0.900 -0.900 scale 0.500 setlinewidth n 469 134 m 334 134 l gs col-1 s gr n 342.000 136.000 m 334.000 134.000 l 342.000 132.000 l gs 2 setlinejoin col-1 s gr n 154 399 m 284 399 l gs col-1 s gr n 276.000 397.000 m 284.000 399.000 l 276.000 401.000 l gs 2 setlinejoin col-1 s gr n 469 454 m 329 454 l gs col-1 s gr n 337.000 456.000 m 329.000 454.000 l 337.000 452.000 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 279 59 m gs 1 -1 scale (First Execution) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 274 299 m gs 1 -1 scale (Second Execution) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 139 94 m gs 1 -1 scale (0) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 319 94 m gs 1 -1 scale (1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 509 94 m gs 1 -1 scale (2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 194 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 194 m gs 1 -1 scale (send) col-1 show gr n 159 189 m 279 189 l gs col-1 s gr n 271.000 187.000 m 279.000 189.000 l 271.000 191.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 294 139 m gs 1 -1 scale (recv) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 44 94 m gs 1 -1 scale (process:) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 139 m gs 1 -1 scale (send) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 199 184 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 384 129 m gs 1 -1 scale (match) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 429 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 404 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 459 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 459 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 484 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 404 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 379 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 194 394 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 379 449 m gs 1 -1 scale (match) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1146 a Fx(Figure)d(4.10:)k(A)c(race)f(condition)i(causes)f (non-deterministic)i(matc)o(hing)d(of)g(sends)i(and)e(receiv)o(es.)20 b(One)75 1202 y(cannot)15 b(rely)h(on)f(sync)o(hronization)h(from)e(a)h (broadcast)g(to)f(mak)o(e)h(the)g(program)f(deterministic.)166 1331 y(All)19 b(three)f(pro)q(cesses)g(participate)h(in)g(a)e (broadcast.)28 b(Pro)q(cess)17 b(0)h(sends)g(a)g(message)f(to)h(pro)q (cess)75 1388 y(1)h(after)g(the)h(broadcast,)f(and)h(pro)q(cess)f(2)h (sends)g(a)f(message)g(to)g(pro)q(cess)h(1)f(b)q(efore)g(the)h (broadcast.)75 1444 y(Pro)q(cess)15 b(1)g(receiv)o(es)h(b)q(efore)g (and)f(after)f(the)i(broadcast,)e(with)h(a)g(wildcard)i(source)e (argumen)o(t.)166 1500 y(Tw)o(o)10 b(p)q(ossible)j(executions)g(of)e (this)g(program,)g(with)h(di\013eren)o(t)f(matc)o(hings)g(of)g(sends)h (and)f(receiv)o(es,)75 1557 y(are)k(illustrated)i(in)f(\014gure)g (4.10.)j(Note)c(that)g(the)g(second)h(execution)g(has)g(the)f(p)q (eculiar)i(e\013ect)f(that)e(a)75 1613 y(send)k(executed)g(after)f(the) h(broadcast)e(is)i(receiv)o(ed)h(at)e(another)g(no)q(de)h(b)q(efore)g (the)f(broadcast.)26 b(This)75 1670 y(example)15 b(illustrates)g(the)g (fact)f(that)f(one)i(should)g(not)f(rely)h(on)f(collectiv)o(e)i(comm)o (unication)f(functions)75 1726 y(to)h(ha)o(v)o(e)g(particular)h(sync)o (hronization)h(e\013ects.)23 b(A)17 b(program)f(that)f(w)o(orks)h (correctly)h(only)g(when)g(the)75 1783 y(\014rst)e(execution)h(o)q (ccurs)g(\(only)f(when)h(broadcast)e(is)i(sync)o(hronizing\))g(is)g (erroneous.)166 1881 y(Finally)l(,)h(in)g(m)o(ultithreaded)g(implemen)o (tations,)g(one)f(can)g(ha)o(v)o(e)g(more)g(than)f(one,)h(concurren)o (tly)75 1938 y(executing,)e(collectiv)o(e)h(comm)o(unication)f(call)h (at)d(a)h(pro)q(cess.)20 b(In)13 b(these)h(situations,)f(it)h(is)g(the) f(user's)g(re-)75 1994 y(sp)q(onsibilit)o(y)i(to)d(ensure)h(that)f(the) h(same)f(comm)o(unicator)h(is)g(not)f(used)h(concurren)o(tly)h(b)o(y)e (t)o(w)o(o)g(di\013eren)o(t)75 2051 y(collectiv)o(e)17 b(comm)o(unication)f(calls)g(at)f(the)g(same)g(pro)q(cess.)189 2150 y Fl(A)n(dvic)n(e)e(to)h(implementors.)39 b Fx(Assume)13 b(that)f(broadcast)g(is)h(implemen)o(ted)i(using)e(p)q(oin)o(t-to-p)q (oin)o(t)189 2206 y Fv(MPI)h Fx(comm)o(unication.)21 b(Supp)q(ose)16 b(the)f(follo)o(wing)h(t)o(w)o(o)e(rules)i(are)f(follo) o(w)o(ed.)231 2298 y(1.)22 b(All)16 b(receiv)o(es)g(sp)q(ecify)h(their) e(source)h(explicitly)i(\(no)c(wildcards\).)231 2369 y(2.)22 b(Eac)o(h)13 b(pro)q(cess)g(sends)h(all)g(messages)f(that)f(p)q (ertain)i(to)f(one)g(collectiv)o(e)i(call)f(b)q(efore)g(sending)289 2425 y(an)o(y)g(message)h(that)g(p)q(ertain)h(to)e(a)h(subsequen)o(t)h (collectiv)o(e)h(call.)189 2517 y(Then,)g(messages)f(b)q(elonging)j(to) d(successiv)o(e)i(broadcasts)e(cannot)g(b)q(e)i(confused,)f(as)f(the)h (order)189 2573 y(of)d(p)q(oin)o(t-to-p)q(oin)o(t)i(messages)f(is)h (preserv)o(ed.)189 2647 y(It)g(is)h(the)g(implemen)o(tor's)g(resp)q (onsibilit)o(y)i(to)d(ensure)i(that)e(p)q(oin)o(t-to-p)q(oin)o(t)h (messages)f(are)g(not)189 2704 y(confused)d(with)g(collectiv)o(e)h (messages.)19 b(One)13 b(w)o(a)o(y)f(to)g(accomplish)i(this)f(is,)g (whenev)o(er)g(a)f(comm)o(u-)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 129 137 129 136 bop 75 -100 a Fm(4.12.)34 b(CORRECTNESS)1257 b Fx(129)189 45 y(nicator)15 b(is)g(created,)g(to)g(also)g(create)g(a)g (\\hidden)h(comm)o(unicator")f(for)f(collectiv)o(e)j(comm)o(unica-)189 102 y(tion.)j(One)15 b(could)h(ac)o(hiev)o(e)f(a)f(similar)i(e\013ect)e (more)g(c)o(heaply)l(,)i(for)e(example,)h(b)o(y)g(using)g(a)f(hidden) 189 158 y(tag)f(or)h(con)o(text)g(bit)h(to)f(indicate)i(whether)f(the)f (comm)o(unicator)g(is)h(used)g(for)f(p)q(oin)o(t-to-p)q(oin)o(t)h(or) 189 214 y(collectiv)o(e)i(comm)o(unication.)j(\()p Fl(End)c(of)g(advic) n(e)g(to)h(implementors.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 130 138 130 137 bop 75 356 a Fo(Chapter)34 b(5)75 564 y Fr(Groups,)40 b(Con)m(texts,)75 689 y(Comm)m(unicators,)g(and)g(Cac)m(hing)75 929 y Fn(5.1)59 b(Intro)r(duction)75 1031 y Fx(This)20 b(c)o(hapter)g(in)o(tro)q(duces)h Fv(MPI)e Fx(features)g(that)h(supp)q (ort)g(the)f(dev)o(elopmen)o(t)i(of)e(parallel)j(libraries.)75 1087 y(P)o(arallel)d(libraries)g(are)e(needed)i(to)e(encapsulate)i(the) f(distracting)g(complications)h(inheren)o(t)g(in)f(par-)75 1144 y(allel)i(implemen)o(tations)g(of)e(k)o(ey)h(algorithms.)30 b(They)18 b(help)i(ensure)f(consisten)o(t)g(correctness)f(of)h(suc)o(h) 75 1200 y(pro)q(cedures,)j(and)e(pro)o(vide)h(a)f(\\higher)h(lev)o(el") g(of)f(p)q(ortabilit)o(y)h(than)f Fv(MPI)g Fx(itself)h(can)f(pro)o (vide.)36 b(As)75 1257 y(suc)o(h,)23 b(libraries)g(prev)o(en)o(t)e(eac) o(h)g(programmer)f(from)h(rep)q(eating)h(the)g(w)o(ork)e(of)h (de\014ning)i(consisten)o(t)75 1313 y(data)14 b(structures,)f(data)h (la)o(y)o(outs,)g(and)g(metho)q(ds)g(that)g(implemen)o(t)i(k)o(ey)e (algorithms)g(\(suc)o(h)g(as)g(matrix)75 1369 y(op)q(erations\).)24 b(Since)18 b(the)e(b)q(est)h(libraries)h(come)e(with)h(sev)o(eral)g(v)m (ariations)g(on)f(parallel)i(systems)e(\(dif-)75 1426 y(feren)o(t)f(data)g(la)o(y)o(outs,)g(di\013eren)o(t)g(strategies)g (dep)q(ending)j(on)e(the)f(size)i(of)e(the)g(system)h(or)f(problem,)g (or)75 1482 y(t)o(yp)q(e)g(of)g(\015oating)g(p)q(oin)o(t\),)g(this)h (to)q(o)e(needs)i(to)f(b)q(e)h(hidden)h(from)d(the)h(user.)166 1539 y(W)l(e)h(refer)f(the)g(reader)h(to)f([26)o(])g(and)h([3)o(])f (for)g(further)g(information)h(on)g(writing)g(libraries)h(in)f Fv(MPI)p Fx(,)75 1595 y(using)g(the)f(features)g(describ)q(ed)i(in)f (this)g(c)o(hapter.)75 1717 y Fj(5.1.1)49 b(F)o(eatures)14 b(Needed)i(to)g(Supp)q(o)o(rt)g(Lib)o(ra)o(ries)75 1803 y Fx(The)f(k)o(ey)h(features)e(needed)j(to)d(supp)q(ort)i(the)f (creation)g(of)g(robust)g(parallel)i(libraries)f(are)f(as)g(follo)o (ws:)143 1897 y Ft(\017)23 b Fx(Safe)18 b(comm)o(unication)i(space,)g (that)e(guaran)o(tees)g(that)g(libraries)i(can)f(comm)o(unicate)g(as)g (they)189 1953 y(need)d(to,)e(without)h(con\015icting)i(with)e(comm)o (unication)h(extraneous)f(to)g(the)g(library)l(,)143 2047 y Ft(\017)23 b Fx(Group)14 b(scop)q(e)h(for)g(collectiv)o(e)h(op)q (erations,)f(that)f(allo)o(w)h(libraries)h(to)e(a)o(v)o(oid)h (unnecessarily)i(syn-)189 2103 y(c)o(hronizing)f(unin)o(v)o(olv)o(ed)h (pro)q(cesses)e(\(p)q(oten)o(tially)i(running)f(unrelated)g(co)q(de\),) 143 2197 y Ft(\017)23 b Fx(Abstract)15 b(pro)q(cess)h(naming)h(to)e (allo)o(w)h(libraries)i(to)d(describ)q(e)j(their)e(comm)o(unication)h (in)g(terms)189 2254 y(suitable)f(to)f(their)g(o)o(wn)g(data)g (structures)f(and)i(algorithms,)143 2347 y Ft(\017)23 b Fx(The)14 b(abilit)o(y)i(to)e(\\adorn")f(a)h(set)h(of)f(comm)o (unicating)h(pro)q(cesses)g(with)f(additional)i(user-de\014ned)189 2404 y(attributes,)23 b(suc)o(h)f(as)g(extra)g(collectiv)o(e)i(op)q (erations.)41 b(This)23 b(mec)o(hanism)f(should)i(pro)o(vide)e(a)189 2460 y(means)15 b(for)f(the)h(user)g(or)g(library)h(writer)f (e\013ectiv)o(ely)h(to)e(extend)i(a)f(message-passing)g(notation.)75 2554 y(In)f(addition,)h(a)f(uni\014ed)h(mec)o(hanism)g(or)e(ob)s(ject)g (is)i(needed)g(for)e(con)o(v)o(enien)o(tly)i(denoting)f(comm)o(unica-) 75 2611 y(tion)g(con)o(text,)f(the)h(group)f(of)g(comm)o(unicating)h (pro)q(cesses,)g(to)f(house)h(abstract)f(pro)q(cess)h(naming,)g(and)75 2667 y(to)h(store)f(adornmen)o(ts.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 131 139 131 138 bop 75 -100 a Fm(5.1.)34 b(INTR)o(ODUCTION)1256 b Fx(131)75 45 y Fj(5.1.2)49 b(MPI's)16 b(Supp)q(o)o(rt)g(fo)o(r)g(Lib) o(ra)o(ries)75 131 y Fx(The)f(corresp)q(onding)h(concepts)g(that)e Fv(MPI)h Fx(pro)o(vides,)g(sp)q(eci\014cally)j(to)c(supp)q(ort)h (robust)g(libraries,)h(are)75 187 y(as)f(follo)o(ws:)143 281 y Ft(\017)23 b Fq(Con)o(texts)14 b Fx(of)h(comm)o(unication,)143 375 y Ft(\017)23 b Fq(Groups)14 b Fx(of)h(pro)q(cesses,)143 469 y Ft(\017)23 b Fq(Virtual)18 b(top)q(ologies)p Fx(,)143 563 y Ft(\017)23 b Fq(A)o(ttribute)17 b(cac)o(hing)p Fx(,)143 656 y Ft(\017)23 b Fq(Comm)o(unicators)p Fx(.)75 750 y Fq(Comm)o(unicators)18 b Fx(\(see)g([16)o(,)f(24,)h(27)o(]\))g (encapsulate)h(all)g(of)f(these)g(ideas)h(in)g(order)f(to)f(pro)o(vide) i(the)75 807 y(appropriate)h(scop)q(e)h(for)f(all)h(comm)o(unication)g (op)q(erations)f(in)h Fv(MPI)p Fx(.)e(Comm)o(unicators)h(are)g(divided) 75 863 y(in)o(to)15 b(t)o(w)o(o)f(kinds:)21 b(in)o(tra-comm)o (unicators)15 b(for)f(op)q(erations)h(within)i(a)e(single)h(group)f(of) g(pro)q(cesses,)g(and)75 920 y(in)o(ter-comm)o(unicators,)g(for)f(p)q (oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)g(b)q(et)o(w)o(een)g(t)o(w)o (o)e(groups)g(of)h(pro)q(cesses.)75 1040 y Fv(Caching.)46 b Fx(Comm)o(unicators)17 b(\(see)g(b)q(elo)o(w\))i(pro)o(vide)f(a)f (\\cac)o(hing")h(mec)o(hanism)h(that)e(allo)o(ws)h(one)g(to)75 1096 y(asso)q(ciate)e(new)g(attributes)f(with)h(comm)o(unicators,)g(on) f(par)h(with)g Fv(MPI)f Fx(built-in)j(features.)j(This)c(can)75 1153 y(b)q(e)22 b(used)g(b)o(y)f(adv)m(anced)i(users)e(to)g(adorn)g (comm)o(unicators)g(further,)h(and)g(b)o(y)f Fv(MPI)g Fx(to)g(implemen)o(t)75 1209 y(some)f(comm)o(unicator)g(functions.)35 b(F)l(or)19 b(example,)j(the)f(virtual-top)q(ology)f(functions)h (describ)q(ed)h(in)75 1266 y(Chapter)15 b(6)g(are)g(lik)o(ely)i(to)d(b) q(e)i(supp)q(orted)g(this)f(w)o(a)o(y)l(.)75 1386 y Fv(Groups.)46 b Fx(Groups)16 b(de\014ne)h(an)f(ordered)g(collection)i(of)e(pro)q (cesses,)g(eac)o(h)g(with)h(a)f(rank,)f(and)i(it)f(is)h(this)75 1442 y(group)d(that)g(de\014nes)i(the)f(lo)o(w-lev)o(el)h(names)e(for)g (in)o(ter-pro)q(cess)i(comm)o(unication)f(\(ranks)f(are)g(used)i(for)75 1498 y(sending)k(and)g(receiving\).)33 b(Th)o(us,)20 b(groups)e(de\014ne)j(a)d(scop)q(e)i(for)f(pro)q(cess)g(names)g(in)h(p) q(oin)o(t-to-p)q(oin)o(t)75 1555 y(comm)o(unication.)i(In)16 b(addition,)h(groups)e(de\014ne)i(the)e(scop)q(e)i(of)e(collectiv)o(e)i (op)q(erations.)k(Groups)16 b(ma)o(y)75 1611 y(b)q(e)k(manipulated)h (separately)f(from)e(comm)o(unicators)h(in)i Fv(MPI)p Fx(,)d(but)i(only)g(comm)o(unicators)f(can)g(b)q(e)75 1668 y(used)d(to)e(in)o(v)o(ok)o(e)h(comm)o(unication)h(op)q(erations.) 75 1788 y Fv(Intra-comm)m(unicato)o(rs.)43 b Fx(The)18 b(most)f(commonly)g(used)i(means)e(for)g(message)g(passing)h(in)h Fv(MPI)e Fx(is)h(via)75 1844 y(in)o(tra-comm)o(unicators.)41 b(In)o(tra-comm)o(unicators)22 b(con)o(tain)g(an)h(instance)g(of)f(a)g (group,)i(con)o(texts)d(of)75 1901 y(comm)o(unication)g(for)e(b)q(oth)h (p)q(oin)o(t-to-p)q(oin)o(t)h(and)f(collectiv)o(e)i(comm)o(unication,)g (and)e(the)g(abilit)o(y)i(to)75 1957 y(include)c(virtual)d(top)q(ology) g(and)h(other)e(attributes.)20 b(These)c(features)f(w)o(ork)f(as)h (follo)o(ws:)143 2051 y Ft(\017)23 b Fq(Con)o(texts)13 b Fx(pro)o(vide)i(the)g(abilit)o(y)h(to)d(ha)o(v)o(e)h(separate)g(safe) g(\\univ)o(erses")h(of)f(message)g(passing)h(in)189 2108 y Fv(MPI)p Fx(.)f(A)h(con)o(text)g(is)h(akin)f(to)g(an)g(additional)i (tag)d(that)h(di\013eren)o(tiates)g(messages.)20 b(The)15 b(system)189 2164 y(manages)20 b(this)h(di\013eren)o(tiation)g(pro)q (cess.)36 b(The)21 b(use)g(of)f(separate)g(comm)o(unication)i(con)o (texts)189 2220 y(b)o(y)13 b(distinct)i(libraries)g(\(or)e(distinct)i (library)f(in)o(v)o(o)q(cations\))g(insulates)h(comm)o(unication)f(in)o (ternal)189 2277 y(to)h(the)h(library)h(execution)g(from)e(external)h (comm)o(unication.)23 b(This)17 b(allo)o(ws)f(the)g(in)o(v)o(o)q (cation)g(of)189 2333 y(the)e(library)g(ev)o(en)h(if)f(there)g(are)g(p) q(ending)i(comm)o(unications)e(on)g(\\other")f(comm)o(unicators,)h(and) 189 2390 y(a)o(v)o(oids)f(the)h(need)g(to)f(sync)o(hronize)i(en)o(try)e (or)g(exit)i(in)o(to)e(library)i(co)q(de.)20 b(P)o(ending)14 b(p)q(oin)o(t-to-p)q(oin)o(t)189 2446 y(comm)o(unications)j(are)g(also) f(guaran)o(teed)h(not)f(to)g(in)o(terfere)i(with)f(collectiv)o(e)h (comm)o(unications)189 2503 y(within)e(a)f(single)h(comm)o(unicator.) 143 2597 y Ft(\017)23 b Fx(Group)14 b(de\014ne)j(the)e(participan)o(ts) h(in)g(the)f(comm)o(unication)h(\(see)f(ab)q(o)o(v)o(e\))g(of)f(a)h (comm)o(unicator.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 132 140 132 139 bop 75 -100 a Fx(132)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)143 45 y Ft(\017)23 b Fx(Virtual)13 b(top)q(ology)g(de\014nes)g(a)g(sp)q (ecial)h(mapping)g(of)e(the)h(ranks)g(in)g(a)g(group)f(to)g(and)h(from) f(a)h(user-)189 102 y(de\014ned)k(top)q(ology)l(.)j(Sp)q(ecial)e (constructors)d(for)f(comm)o(unicators)i(are)f(de\014ned)i(in)f(c)o (hapter)f(6)h(to)189 158 y(pro)o(vide)j(this)f(feature.)30 b(In)o(tra-comm)o(unicators)17 b(as)h(describ)q(ed)j(in)e(this)g(c)o (hapter)f(do)g(not)g(ha)o(v)o(e)189 214 y(top)q(ologies.)143 308 y Ft(\017)23 b Fx(A)o(ttributes)14 b(cac)o(he)h(de\014nes)h(the)f (lo)q(cal)g(information)g(that)f(the)h(user)g(or)f(library)h(has)g (added)g(to)f(a)189 365 y(comm)o(unicator)g(for)h(later)g(reference.) 189 471 y Fl(A)n(dvic)n(e)f(to)i(users.)40 b Fx(The)15 b(curren)o(t)f(practice)i(in)f(man)o(y)f(comm)o(unication)i(libraries)g (is)f(that)f(there)189 527 y(is)k(a)f(unique,)i(prede\014ned)g(comm)o (unication)g(univ)o(erse)f(that)f(includes)j(all)e(pro)q(cesses)g(a)o (v)m(ailable)189 584 y(when)f(the)f(parallel)i(program)d(is)i (initiated;)i(the)d(pro)q(cesses)h(are)f(assigned)h(consecutiv)o(e)h (ranks.)189 640 y(P)o(articipan)o(ts)d(in)h(a)g(p)q(oin)o(t-to-p)q(oin) o(t)g(comm)o(unication)g(are)g(iden)o(ti\014ed)h(b)o(y)f(their)g(rank;) f(a)g(collec-)189 697 y(tiv)o(e)i(comm)o(unication)h(\(suc)o(h)g(as)f (broadcast\))f(alw)o(a)o(ys)g(in)o(v)o(olv)o(es)i(all)h(pro)q(cesses.) 26 b(This)18 b(practice)189 753 y(can)e(b)q(e)g(follo)o(w)o(ed)h(in)g Fv(MPI)e Fx(b)o(y)h(using)h(the)f(prede\014ned)i(comm)o(unicator)d Fv(MPI)p 1529 753 14 2 v 16 w(COMM)p 1687 753 V 16 w(W)o(ORLD)p Fx(.)189 810 y Fl(Users)i(who)h(ar)n(e)g(satis\014e)n(d)f(with)h(this)g (pr)n(actic)n(e)f(c)n(an)h(plug)g(in)f Fv(MPI)p 1351 810 V 16 w(COMM)p 1509 810 V 16 w(W)o(ORLD)h Fl(wher)n(ever)189 866 y(a)f(c)n(ommunic)n(ator)g(ar)n(gument)g(is)f(r)n(e)n(quir)n(e)n (d,)g(and)h(c)n(an)f(c)n(onse)n(quently)f(disr)n(e)n(gar)n(d)h(the)h(r) n(est)f(of)h(this)189 923 y(chapter.)k Fx(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))75 1043 y Fv(Inter-comm)n (unicato)o(rs.)42 b Fx(The)18 b(discussion)h(has)f(dealt)g(so)f(far)g (with)g Fq(in)o(tra-comm)o(unication)p Fx(:)27 b(com-)75 1099 y(m)o(unication)21 b(within)h(a)e(group.)36 b Fv(MPI)19 b Fx(also)i(supp)q(orts)f Fq(in)o(ter-comm)o(unication)p Fx(:)32 b(comm)o(unication)75 1156 y(b)q(et)o(w)o(een)18 b(t)o(w)o(o)f(non-o)o(v)o(erlapping)i(groups.)28 b(When)18 b(an)g(application)i(is)e(built)h(b)o(y)f(comp)q(osing)h(sev)o(eral)75 1212 y(parallel)e(mo)q(dules,)g(it)f(is)h(con)o(v)o(enien)o(t)f(to)f (allo)o(w)h(one)g(mo)q(dule)h(to)e(comm)o(unicate)i(with)f(another)f (using)75 1269 y(lo)q(cal)22 b(ranks)e(for)f(addressing)i(within)h(the) f(second)f(mo)q(dule.)37 b(This)21 b(is)g(esp)q(ecially)i(con)o(v)o (enien)o(t)e(in)g(a)75 1325 y(clien)o(t-serv)o(er)e(computing)f (paradigm,)g(where)g(either)h(clien)o(t)g(or)e(serv)o(er)g(are)h (parallel.)29 b(The)18 b(supp)q(ort)75 1381 y(of)f(in)o(ter-comm)o (unication)h(also)f(pro)o(vides)g(a)g(mec)o(hanism)g(for)f(the)i (extension)f(of)g Fv(MPI)f Fx(to)g(a)h(dynamic)75 1438 y(mo)q(del)h(where)g(not)f(all)i(pro)q(cesses)f(are)f(preallo)q(cated)h (at)f(initialization)k(time.)27 b(In)18 b(suc)o(h)g(a)f(situation,)75 1494 y(it)h(b)q(ecomes)g(necessary)g(to)g(supp)q(ort)f(comm)o (unication)i(across)e(\\univ)o(erses.")28 b(In)o(ter-comm)o(unication) 75 1551 y(is)17 b(supp)q(orted)f(b)o(y)g(ob)s(jects)g(called)h Fq(in)o(ter-comm)o(unicators)p Fx(.)23 b(These)16 b(ob)s(jects)g(bind)h (t)o(w)o(o)e(groups)g(to-)75 1607 y(gether)c(with)h(comm)o(unication)g (con)o(texts)e(shared)i(b)o(y)f(b)q(oth)g(groups.)18 b(F)l(or)11 b(in)o(ter-comm)o(unicators,)h(these)75 1664 y(features)j(w)o(ork)f(as)h(follo)o(ws:)143 1758 y Ft(\017)23 b Fq(Con)o(texts)16 b Fx(pro)o(vide)i(the)f(abilit)o(y)i(to)d(ha)o(v)o (e)h(a)g(separate)g(safe)f(\\univ)o(erse")i(of)f(message)g(passing)189 1814 y(b)q(et)o(w)o(een)k(the)f(t)o(w)o(o)g(groups.)36 b(A)21 b(send)g(in)g(the)g(lo)q(cal)h(group)e(is)i(alw)o(a)o(ys)e(a)g (receiv)o(e)i(in)f(the)g(re-)189 1870 y(mote)d(group,)g(and)h(vice)h(v) o(ersa.)29 b(The)19 b(system)f(manages)g(this)h(di\013eren)o(tiation)g (pro)q(cess.)31 b(The)189 1927 y(use)19 b(of)f(separate)g(comm)o (unication)i(con)o(texts)e(b)o(y)g(distinct)i(libraries)h(\(or)d (distinct)h(library)h(in-)189 1983 y(v)o(o)q(cations\))f(insulates)i (comm)o(unication)g(in)o(ternal)f(to)g(the)g(library)g(execution)h (from)f(external)189 2040 y(comm)o(unication.)29 b(This)19 b(allo)o(ws)f(the)g(in)o(v)o(o)q(cation)h(of)f(the)g(library)h(ev)o(en) f(if)h(there)f(are)g(p)q(ending)189 2096 y(comm)o(unications)c(on)f (\\other")f(comm)o(unicators,)h(and)h(a)o(v)o(oids)f(the)g(need)h(to)f (sync)o(hronize)h(en)o(try)189 2153 y(or)k(exit)h(in)o(to)g(library)h (co)q(de.)32 b(There)19 b(is)g(no)g(general-purp)q(ose)i(collectiv)o(e) f(comm)o(unication)g(on)189 2209 y(in)o(ter-comm)o(unicators,)d(so)g (con)o(texts)f(are)h(used)h(just)e(to)h(isolate)h(p)q(oin)o(t-to-p)q (oin)o(t)f(comm)o(unica-)189 2266 y(tion.)143 2359 y Ft(\017)23 b Fx(A)10 b(lo)q(cal)h(and)f(remote)g(group)g(sp)q(ecify)h (the)g(recipien)o(ts)g(and)g(destinations)f(for)g(an)g(in)o(ter-comm)o (unicator.)143 2453 y Ft(\017)23 b Fx(Virtual)16 b(top)q(ology)e(is)i (unde\014ned)h(for)e(an)g(in)o(ter-comm)o(unicator.)143 2547 y Ft(\017)23 b Fx(As)16 b(b)q(efore,)g(attributes)g(cac)o(he)h (de\014nes)g(the)f(lo)q(cal)i(information)e(that)g(the)g(user)g(or)g (library)h(has)189 2604 y(added)e(to)g(a)g(comm)o(unicator)g(for)f (later)i(reference.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 133 141 133 140 bop 75 -100 a Fm(5.2.)34 b(BASIC)16 b(CONCEPTS)1215 b Fx(133)166 45 y Fv(MPI)17 b Fx(pro)o(vides)i(mec)o(hanisms)g(for)e (creating)h(and)h(manipulating)g(in)o(ter-comm)o(unicators.)29 b(They)75 102 y(are)18 b(used)h(for)f(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication)g(in)g(an)f(related)h(manner)f(to)g(in)o(tra-comm)o (unicators.)75 158 y(Users)j(who)g(do)g(not)g(need)h(in)o(ter-comm)o (unication)h(in)f(their)f(applications)i(can)e(safely)h(ignore)g(this) 75 214 y(extension.)k(Users)18 b(who)e(need)i(collectiv)o(e)h(op)q (erations)f(via)f(in)o(ter-comm)o(unicators)g(m)o(ust)g(la)o(y)o(er)g (it)g(on)75 271 y(top)e(of)h Fv(MPI)p Fx(.)f(Users)g(who)h(require)h (in)o(ter-comm)o(unication)g(b)q(et)o(w)o(een)f(o)o(v)o(erlapping)g (groups)g(m)o(ust)f(also)75 327 y(la)o(y)o(er)g(this)h(capabilit)o(y)g (on)f(top)g(of)g Fv(MPI)p Fx(.)75 471 y Fn(5.2)59 b(Basic)19 b(Concepts)75 572 y Fx(In)d(this)f(section,)h(w)o(e)f(turn)g(to)f(a)h (more)g(formal)g(de\014nition)i(of)e(the)g(concepts)h(in)o(tro)q(duced) g(ab)q(o)o(v)o(e.)75 694 y Fj(5.2.1)49 b(Groups)75 780 y Fx(A)24 b Fq(group)h Fx(is)g(an)f(ordered)h(set)f(of)g(pro)q(cess)g (iden)o(ti\014ers)i(\(henceforth)f(pro)q(cesses\);)j(pro)q(cesses)d (are)75 836 y(implemen)o(tation-dep)q(endent)e(ob)s(jects.)36 b(Eac)o(h)20 b(pro)q(cess)i(in)f(a)g(group)f(is)h(asso)q(ciated)g(with) h(an)e(in)o(te-)75 893 y(ger)e Fq(rank)p Fx(.)28 b(Ranks)18 b(are)g(con)o(tiguous)g(and)g(start)f(from)g(zero.)28 b(Groups)18 b(are)f(represen)o(ted)i(b)o(y)f(opaque)75 949 y Fq(group)g(ob)s(jects)p Fx(,)f(and)f(hence)h(cannot)f(b)q(e)h (directly)g(transferred)f(from)f(one)i(pro)q(cess)f(to)g(another.)22 b(A)75 1005 y(group)13 b(is)h(used)g(within)g(a)f(comm)o(unicator)g(to) g(describ)q(e)i(the)e(participan)o(ts)h(in)g(a)f(comm)o(unication)h (\\uni-)75 1062 y(v)o(erse")d(and)i(to)e(rank)g(suc)o(h)i(participan)o (ts)f(\(th)o(us)f(giving)i(them)f(unique)h(names)f(within)h(that)f (\\univ)o(erse")75 1118 y(of)j(comm)o(unication\).)166 1175 y(There)20 b(is)f(a)h(sp)q(ecial)h(pre-de\014ned)g(group:)28 b Fi(MPI)p 1017 1175 13 2 v 14 w(GROUP)p 1173 1175 V 13 w(EMPTY)p Fx(,)19 b(whic)o(h)h(is)g(a)f(group)g(with)h(no)75 1231 y(mem)o(b)q(ers.)33 b(The)20 b(prede\014ned)h(constan)o(t)d Fi(MPI)p 884 1231 V 15 w(GROUP)p 1041 1231 V 13 w(NULL)i Fx(is)g(the)g(v)m(alue)g(used)h(for)d(in)o(v)m(alid)k(group)75 1288 y(handles.)189 1394 y Fl(A)n(dvic)n(e)c(to)i(users.)62 b Fi(MPI)p 650 1394 V 14 w(GROUP)p 806 1394 V 13 w(EMPTY)p Fx(,)18 b(whic)o(h)i(is)f(a)g(v)m(alid)h(handle)g(to)e(an)h(empt)o(y)g (group,)189 1450 y(should)h(not)f(b)q(e)h(confused)h(with)e Fi(MPI)p 863 1450 V 15 w(GROUP)p 1020 1450 V 13 w(NULL)p Fx(,)h(whic)o(h)g(in)g(turn)g(is)g(an)f(in)o(v)m(alid)j(handle.)189 1507 y(The)e(former)f(ma)o(y)g(b)q(e)h(used)g(as)g(an)f(argumen)o(t)g (to)h(group)f(op)q(erations;)j(the)e(latter,)g(whic)o(h)g(is)189 1563 y(returned)15 b(when)h(a)f(group)g(is)g(freed,)h(in)g(not)e(a)h(v) m(alid)i(argumen)o(t.)i(\()p Fl(End)d(of)g(advic)n(e)g(to)h(users.)p Fx(\))189 1670 y Fl(A)n(dvic)n(e)g(to)h(implementors.)53 b Fx(A)18 b(group)f(ma)o(y)f(b)q(e)i(represen)o(ted)g(b)o(y)f(a)g (virtual-to-real)i(pro)q(cess-)189 1726 y(address-translation)11 b(table.)19 b(Eac)o(h)11 b(comm)o(unicator)f(ob)s(ject)h(\(see)g(b)q (elo)o(w\))g(w)o(ould)g(ha)o(v)o(e)g(a)f(p)q(oin)o(ter)189 1782 y(to)k(suc)o(h)i(a)f(table.)189 1858 y(Simple)j(implemen)o (tations)g(of)e Fv(MPI)g Fx(will)i(en)o(umerate)e(groups,)g(suc)o(h)h (as)f(in)i(a)e(table.)24 b(Ho)o(w)o(ev)o(er,)189 1914 y(more)11 b(adv)m(anced)i(data)e(structures)g(mak)o(e)g(sense)i(in)f (order)g(to)f(impro)o(v)o(e)g(scalabilit)o(y)j(and)e(memory)189 1971 y(usage)j(with)g(large)h(n)o(um)o(b)q(ers)f(of)g(pro)q(cesses.)21 b(Suc)o(h)16 b(implemen)o(tations)g(are)f(p)q(ossible)i(with)f Fv(MPI)p Fx(.)189 2027 y(\()p Fl(End)f(of)i(advic)n(e)f(to)g (implementors.)p Fx(\))75 2149 y Fj(5.2.2)49 b(Contexts)75 2235 y Fx(A)18 b Fq(con)o(text)g Fx(is)h(a)f(prop)q(ert)o(y)f(of)h (comm)o(unicators)g(\(de\014ned)h(next\))f(that)f(allo)o(ws)h (partitioning)h(of)f(the)75 2291 y(comm)o(unication)13 b(space.)19 b(A)12 b(message)g(sen)o(t)g(in)h(one)f(con)o(text)g (cannot)g(b)q(e)h(receiv)o(ed)g(in)g(another)f(con)o(text.)75 2347 y(F)l(urthermore,)17 b(where)g(p)q(ermitted,)h(collectiv)o(e)h(op) q(erations)e(are)g(indep)q(enden)o(t)i(of)e(p)q(ending)i(p)q(oin)o (t-to-)75 2404 y(p)q(oin)o(t)g(op)q(erations.)28 b(Con)o(texts)17 b(are)h(not)f(explicit)k Fv(MPI)c Fx(ob)s(jects;)h(they)g(app)q(ear)h (only)f(as)g(part)f(of)h(the)75 2460 y(realization)e(of)f(comm)o (unicators)g(\(b)q(elo)o(w\).)189 2567 y Fl(A)n(dvic)n(e)h(to)i (implementors.)48 b Fx(Distinct)17 b(comm)o(unicators)f(in)h(the)g (same)f(pro)q(cess)h(ha)o(v)o(e)f(distinct)189 2623 y(con)o(texts.)31 b(A)20 b(con)o(text)f(is)g(essen)o(tially)i(a)e(system-managed)g(tag)g (\(or)f(tags\))g(needed)j(to)d(mak)o(e)189 2680 y(a)i(comm)o(unicator)h (safe)g(for)f(p)q(oin)o(t-to-p)q(oin)o(t)i(and)f Fv(MPI)p Fx(-de\014ned)h(collectiv)o(e)h(comm)o(unication.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 134 142 134 141 bop 75 -100 a Fx(134)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)189 45 y Fx(Safet)o(y)j(means)h(that)f(collectiv)o(e)j(and)e(p)q(oin)o (t-to-p)q(oin)o(t)g(comm)o(unication)h(within)g(one)f(comm)o(u-)189 102 y(nicator)d(do)g(not)g(in)o(terfere,)h(and)g(that)e(comm)o (unication)i(o)o(v)o(er)f(distinct)h(comm)o(unicators)f(don't)189 158 y(in)o(terfere.)189 230 y(A)c(p)q(ossible)i(implemen)o(tation)f (for)e(a)h(con)o(text)g(is)g(as)g(a)g(supplemen)o(tal)h(tag)f(attac)o (hed)f(to)g(messages)189 287 y(on)j(send)g(and)h(matc)o(hed)f(on)g (receiv)o(e.)20 b(Eac)o(h)14 b(in)o(tra-comm)o(unicator)g(stores)g(the) g(v)m(alue)h(of)f(its)g(t)o(w)o(o)189 343 y(tags)i(\(one)i(for)f(p)q (oin)o(t-to-p)q(oin)o(t)h(and)g(one)g(for)f(collectiv)o(e)i(comm)o (unication\).)28 b(Comm)o(unicator-)189 400 y(generating)19 b(functions)i(use)f(a)f(collectiv)o(e)j(comm)o(unication)e(to)f(agree)g (on)h(a)f(new)h(group-wide)189 456 y(unique)c(con)o(text.)189 528 y(Analogously)l(,)c(in)f(in)o(ter-comm)o(unication)h(\(whic)o(h)f (is)g(strictly)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unication\),)189 585 y(t)o(w)o(o)f(con)o(text)h(tags)g(are)h(stored)f(p)q(er)i(comm)o (unicator,)f(one)g(used)g(b)o(y)g(group)g(A)f(to)h(send)g(and)g(group) 189 641 y(B)j(to)g(receiv)o(e,)g(and)h(a)f(second)g(used)h(b)o(y)f (group)g(B)g(to)g(send)h(and)f(for)g(group)g(A)g(to)f(receiv)o(e.)189 713 y(Since)h(con)o(texts)f(are)f(not)h(explicit)i(ob)s(jects,)e(other) f(implemen)o(tations)j(are)d(also)h(p)q(ossible.)22 b(\()p Fl(End)189 770 y(of)16 b(advic)n(e)g(to)h(implementors.)p Fx(\))75 889 y Fj(5.2.3)49 b(Intra-Comm)o(unicato)o(rs)75 975 y Fx(In)o(tra-comm)o(unicators)26 b(bring)i(together)e(the)h (concepts)h(of)e(group)h(and)g(con)o(text.)55 b(T)l(o)27 b(supp)q(ort)75 1031 y(implemen)o(tation-sp)q(eci\014c)15 b(optimizations,)e(and)f(application)h(top)q(ologies)g(\(de\014ned)f (in)h(the)f(next)g(c)o(hap-)75 1087 y(ter,)17 b(c)o(hapter)g(6\),)f (comm)o(unicators)h(ma)o(y)f(also)h(\\cac)o(he")f(additional)j (information)e(\(see)g(section)h(5.7\).)75 1144 y Fv(MPI)h Fx(comm)o(unication)g(op)q(erations)h(reference)f(comm)o(unicators)g (to)g(determine)h(the)f(scop)q(e)h(and)f(the)75 1200 y(\\comm)o(unication)d(univ)o(erse")g(in)g(whic)o(h)g(a)e(p)q(oin)o (t-to-p)q(oin)o(t)i(or)f(collectiv)o(e)i(op)q(eration)e(is)h(to)f(op)q (erate.)166 1257 y(Eac)o(h)g(comm)o(unicator)h(con)o(tains)g(a)f(group) g(of)h(v)m(alid)h(participan)o(ts;)f(this)g(group)g(alw)o(a)o(ys)f (includes)75 1313 y(the)k(lo)q(cal)h(pro)q(cess.)32 b(The)20 b(source)f(and)g(destination)h(of)f(a)f(message)h(is)h(iden)o(ti\014ed) h(b)o(y)e(pro)q(cess)g(rank)75 1370 y(within)d(that)f(group.)166 1426 y(F)l(or)10 b(collectiv)o(e)j(comm)o(unication,)f(the)f(in)o (tra-comm)o(unicator)g(sp)q(eci\014es)i(the)e(set)g(of)f(pro)q(cesses)i (that)75 1483 y(participate)g(in)g(the)g(collectiv)o(e)h(op)q(eration)e (\(and)h(their)f(order,)h(when)g(signi\014can)o(t\).)19 b(Th)o(us,)12 b(the)f(comm)o(u-)75 1539 y(nicator)16 b(restricts)g(the)g(\\spatial")g(scop)q(e)h(of)e(comm)o(unication,)i (and)f(pro)o(vides)h(mac)o(hine-indep)q(endent)75 1596 y(pro)q(cess)e(addressing)h(through)f(ranks.)166 1652 y(In)o(tra-comm)o(unicators)h(are)h(represen)o(ted)g(b)o(y)g(opaque)g Fq(in)o(tra-comm)o(unicator)k(ob)s(jects)p Fx(,)c(and)75 1708 y(hence)f(cannot)f(b)q(e)h(directly)h(transferred)d(from)h(one)g (pro)q(cess)h(to)e(another.)75 1827 y Fj(5.2.4)49 b(Prede\014ned)15 b(Intra-Comm)o(unicato)o(rs)75 1913 y Fx(An)22 b(initial)i(in)o (tra-comm)o(unicator)e Fi(MPI)p 785 1913 13 2 v 14 w(COMM)p 929 1913 V 15 w(W)o(ORLD)f Fx(of)h(all)h(pro)q(cesses)f(the)g(lo)q(cal) h(pro)q(cess)f(can)75 1970 y(comm)o(unicate)d(with)g(after)f (initialization)k(\(itself)d(included\))j(is)d(de\014ned)h(once)f Fv(MPI)p 1578 1970 14 2 v 16 w(INIT)f Fx(has)h(b)q(een)75 2026 y(called.)i(In)13 b(addition,)h(the)e(comm)o(unicator)g Fi(MPI)p 911 2026 13 2 v 15 w(COMM)p 1056 2026 V 14 w(SELF)h Fx(is)g(pro)o(vided,)g(whic)o(h)h(includes)h(only)e(the)75 2083 y(pro)q(cess)i(itself.)166 2139 y(The)i(prede\014ned)i(constan)o (t)d Fi(MPI)p 745 2139 V 14 w(COMM)p 889 2139 V 15 w(NULL)i Fx(is)f(the)g(v)m(alue)h(used)g(for)e(in)o(v)m(alid)j(comm)o(unicator) 75 2195 y(handles.)166 2252 y(In)d(a)g(static-pro)q(cess-mo)q(del)h (implemen)o(tation)g(of)f Fv(MPI)p Fx(,)e(all)j(pro)q(cesses)g(that)e (participate)h(in)h(the)75 2308 y(computation)i(are)f(a)o(v)m(ailable)j (after)d Fv(MPI)g Fx(is)h(initialized.)34 b(F)l(or)18 b(this)h(case,)g Fi(MPI)p 1479 2308 V 15 w(COMM)p 1624 2308 V 14 w(W)o(ORLD)g Fx(is)g(a)75 2365 y(comm)o(unicator)g(of)g(all)i (pro)q(cesses)f(a)o(v)m(ailable)h(for)e(the)g(computation;)i(this)f (comm)o(unicator)g(has)f(the)75 2421 y(same)h(v)m(alue)h(in)g(all)f (pro)q(cesses.)35 b(In)21 b(an)e(implemen)o(tation)j(of)d Fv(MPI)h Fx(where)g(pro)q(cesses)g(can)g(dynami-)75 2478 y(cally)d(join)g(an)f Fv(MPI)g Fx(execution,)h(it)f(ma)o(y)g(b)q(e)g (the)h(case)f(that)f(a)h(pro)q(cess)g(starts)f(an)h Fv(MPI)g Fx(computation)75 2534 y(without)j(ha)o(ving)g(access)g(to)f(all)i (other)f(pro)q(cesses.)31 b(In)20 b(suc)o(h)f(situations,)h Fi(MPI)p 1479 2534 V 14 w(COMM)p 1623 2534 V 15 w(W)o(ORLD)f Fx(is)g(a)75 2591 y(comm)o(unicator)d(incorp)q(orating)h(all)h(pro)q (cesses)f(with)g(whic)o(h)g(the)g(joining)h(pro)q(cess)f(can)f (immediately)75 2647 y(comm)o(unicate.)28 b(Therefore,)17 b Fi(MPI)p 675 2647 V 15 w(COMM)p 820 2647 V 14 w(W)o(ORLD)h Fx(ma)o(y)f(sim)o(ultaneously)i(ha)o(v)o(e)e(di\013eren)o(t)h(v)m (alues)h(in)75 2704 y(di\013eren)o(t)c(pro)q(cesses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 135 143 135 142 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fx(135)166 45 y(All)18 b Fv(MPI)d Fx(implemen)o(tations)j(are)e (required)h(to)f(pro)o(vide)h(the)f Fi(MPI)p 1315 45 13 2 v 15 w(COMM)p 1460 45 V 14 w(W)o(ORLD)g Fx(comm)o(unica-)75 102 y(tor.)34 b(It)21 b(cannot)f(b)q(e)h(deallo)q(cated)h(during)f(the) f(life)i(of)e(a)g(pro)q(cess.)36 b(The)20 b(group)g(corresp)q(onding)i (to)75 158 y(this)14 b(comm)o(unicator)g(do)q(es)g(not)g(app)q(ear)g (as)g(a)f(pre-de\014ned)j(constan)o(t,)d(but)h(it)h(ma)o(y)e(b)q(e)i (accessed)f(using)75 214 y Fv(MPI)p 160 214 14 2 v 16 w(COMM)p 318 214 V 16 w(GROUP)21 b Fx(\(see)f(b)q(elo)o(w\).)34 b Fv(MPI)19 b Fx(do)q(es)h(not)g(sp)q(ecify)h(the)f(corresp)q(ondence)h (b)q(et)o(w)o(een)f(the)75 271 y(pro)q(cess)15 b(rank)f(in)h Fi(MPI)p 466 271 13 2 v 15 w(COMM)p 611 271 V 14 w(W)o(ORLD)g Fx(and)f(its)h(\(mac)o(hine-dep)q(enden)o(t\))i(absolute)d(address.)20 b(Neither)75 327 y(do)q(es)14 b Fv(MPI)e Fx(sp)q(ecify)j(the)f (function)g(of)f(the)g(host)g(pro)q(cess,)h(if)g(an)o(y)l(.)19 b(Other)13 b(implemen)o(tation-dep)q(end)q(en)o(t,)75 384 y(prede\014ned)k(comm)o(unicators)e(ma)o(y)f(also)h(b)q(e)h(pro)o (vided.)75 527 y Fn(5.3)59 b(Group)20 b(Management)75 629 y Fx(This)h(section)g(describ)q(es)h(the)e(manipulation)i(of)e(pro) q(cess)h(groups)f(in)h Fv(MPI)p Fx(.)e(These)i(op)q(erations)f(are)75 685 y(lo)q(cal)c(and)g(their)f(execution)i(do)e(not)g(require)h(in)o (terpro)q(cess)f(comm)o(unication.)75 807 y Fj(5.3.1)49 b(Group)16 b(Accesso)o(rs)75 987 y Fv(MPI)p 160 987 14 2 v 16 w(GROUP)p 328 987 V 18 w(SIZE\(group,)e(size\))117 1064 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 1139 y(OUT)108 b Fv(size)519 b Fk(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)g(\(in)o(teger\))75 1264 y Fu(int)23 b(MPI)p 245 1264 15 2 v 17 w(Group)p 382 1264 V 17 w(size\(MPI)p 591 1264 V 16 w(Group)g(group,)g(int)g (*size\))75 1350 y(MPI)p 150 1350 V 17 w(GROUP)p 287 1350 V 16 w(SIZE\(GROUP,)g(SIZE,)g(IERROR\))170 1407 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 1588 y Fv(MPI)p 160 1588 14 2 v 16 w(GROUP)p 328 1588 V 18 w(RANK\(group,)14 b(rank\))117 1665 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 1740 y(OUT)108 b Fv(rank)504 b Fk(rank)50 b(of)g(the)h(calling)e(pro)q(cess)j(in)e(group,)59 b(or)905 1796 y Fi(MPI)p 982 1796 13 2 v 15 w(UNDEFINED)13 b Fk(if)h(the)h(pro)q (cess)i(is)d(not)h(a)f(mem)o(b)q(er)f(\(in-)905 1853 y(teger\))75 1977 y Fu(int)23 b(MPI)p 245 1977 15 2 v 17 w(Group)p 382 1977 V 17 w(rank\(MPI)p 591 1977 V 16 w(Group)g(group,)g(int)g(*rank\))75 2064 y(MPI)p 150 2064 V 17 w(GROUP)p 287 2064 V 16 w(RANK\(GROUP,)g(RANK,)g(IERROR\))170 2120 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 136 144 136 143 bop 75 -100 a Fx(136)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(TRANSLA)l(TE)p 602 45 V 17 w(RANKS)h(\(group1,)e(n,)h(ranks1,)g (group2,)g(ranks2\))117 122 y Fk(IN)155 b Fv(group1)456 b Fk(group1)14 b(\(handle\))117 197 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)11 b(of)h(ranks)g(in)23 b Fi(ranks1)13 b Fk(and)f Fi(ranks2)g Fk(arra)o(ys)g(\(in)o(teger\))117 273 y(IN)155 b Fv(ranks1)464 b Fk(arra)o(y)14 b(of)f(zero)i(or)f(more)f (v)n(alid)f(ranks)i(in)g(group1)117 348 y(IN)155 b Fv(group2)456 b Fk(group2)14 b(\(handle\))117 423 y(OUT)108 b Fv(ranks2)464 b Fk(arra)o(y)64 b(of)f(corresp)q(onding)i(ranks)f(in)g(group2,)905 479 y Fi(MPI)p 982 479 13 2 v 15 w(UNDEFINED)12 b Fk(when)j(no)e (corresp)q(ondence)k(exists.)75 604 y Fu(int)23 b(MPI)p 245 604 15 2 v 17 w(Group)p 382 604 V 17 w(translate)p 615 604 V 16 w(ranks)g(\(MPI)p 870 604 V 16 w(Group)h(group1,)f(int)g (n,)h(int)f(*ranks1,)393 660 y(MPI)p 468 660 V 17 w(Group)g(group2,)g (int)g(*ranks2\))75 747 y(MPI)p 150 747 V 17 w(GROUP)p 287 747 V 16 w(TRANSLATE)p 519 747 V 16 w(RANKS\(GROUP1,)f(N,)i (RANKS1,)f(GROUP2,)g(RANKS2,)g(IERROR\))170 803 y(INTEGER)g(GROUP1,)g (N,)h(RANKS1\(*\),)e(GROUP2,)h(RANKS2\(*\),)g(IERROR)166 890 y Fx(This)12 b(function)g(is)g(imp)q(ortan)o(t)f(for)f(determining) j(the)e(relativ)o(e)h(n)o(um)o(b)q(ering)g(of)f(the)h(same)f(pro)q (cesses)75 946 y(in)h(t)o(w)o(o)e(di\013eren)o(t)h(groups.)18 b(F)l(or)11 b(instance,)h(if)g(one)f(kno)o(ws)f(the)i(ranks)e(of)h (certain)h(pro)q(cesses)f(in)h(the)f(group)75 1002 y(of)k Fi(MPI)p 204 1002 13 2 v 14 w(COMM)p 348 1002 V 15 w(W)o(ORLD)p Fx(,)f(one)i(migh)o(t)f(w)o(an)o(t)f(to)g(kno)o(w)h(their)h(ranks)f(in) h(a)f(subset)g(of)g(that)f(group.)75 1153 y Fv(MPI)p 160 1153 14 2 v 16 w(GROUP)p 328 1153 V 18 w(COMP)l(ARE\(group1,)h (group2,)g(result\))117 1231 y Fk(IN)155 b Fv(group1)456 b Fk(\014rst)15 b(group)f(\(handle\))117 1306 y(IN)155 b Fv(group2)456 b Fk(second)15 b(group)f(\(handle\))117 1381 y(OUT)108 b Fv(result)484 b Fk(result)15 b(\(in)o(teger\))75 1505 y Fu(int)23 b(MPI)p 245 1505 15 2 v 17 w(Group)p 382 1505 V 17 w(compare\(MPI)p 663 1505 V 15 w(Group)h(group1,MPI)p 1062 1505 V 15 w(Group)g(group2,)e(int)i(*result\))75 1592 y(MPI)p 150 1592 V 17 w(GROUP)p 287 1592 V 16 w(COMPARE\(GROUP1,)e (GROUP2,)h(RESULT,)g(IERROR\))170 1648 y(INTEGER)g(GROUP1,)g(GROUP2,)g (RESULT,)g(IERROR)75 1735 y Fi(MPI)p 152 1735 13 2 v 14 w(IDENT)12 b Fx(results)g(if)h(the)f(group)f(mem)o(b)q(ers)h(and)g (group)g(order)g(is)g(exactly)g(the)g(same)g(in)h(b)q(oth)f(groups.)75 1791 y(This)i(happ)q(ens)h(for)f(instance)g(if)h Fv(group1)e Fx(and)h Fv(group2)g Fx(are)f(the)h(same)g(handle.)21 b Fi(MPI)p 1520 1791 V 14 w(SIMILAR)13 b Fx(results)h(if)75 1848 y(the)f(group)f(mem)o(b)q(ers)h(are)f(the)h(same)f(but)h(the)g (order)f(is)h(di\013eren)o(t.)19 b Fi(MPI)p 1316 1848 V 15 w(UNEQUAL)11 b Fx(results)i(otherwise.)75 1969 y Fj(5.3.2)49 b(Group)16 b(Constructo)o(rs)75 2055 y Fx(Group)j (constructors)f(are)h(used)g(to)g(subset)g(and)g(sup)q(erset)h (existing)g(groups.)31 b(These)20 b(constructors)75 2112 y(construct)14 b(new)g(groups)g(from)f(existing)i(groups.)k(These)14 b(are)g(lo)q(cal)h(op)q(erations,)f(and)g(distinct)h(groups)75 2168 y(ma)o(y)20 b(b)q(e)h(de\014ned)h(on)e(di\013eren)o(t)g(pro)q (cesses;)j(a)e(pro)q(cess)f(ma)o(y)g(also)g(de\014ne)i(a)e(group)g (that)g(do)q(es)g(not)75 2225 y(include)i(itself.)33 b(Consisten)o(t)19 b(de\014nitions)i(are)e(required)h(when)g(groups)f (are)g(used)h(as)f(argumen)o(ts)f(in)75 2281 y(comm)o (unicator-building)23 b(functions.)39 b Fv(MPI)20 b Fx(do)q(es)h(not)g (pro)o(vide)h(a)e(mec)o(hanism)i(to)f(build)i(a)d(group)75 2337 y(from)j(scratc)o(h,)i(but)f(only)h(from)e(other,)i(previously)g (de\014ned)g(groups.)46 b(The)24 b(base)g(group,)h(up)q(on)75 2394 y(whic)o(h)d(all)g(other)e(groups)g(are)h(de\014ned,)i(is)f(the)f (group)f(asso)q(ciated)h(with)g(the)g(initial)i(comm)o(unica-)75 2450 y(tor)14 b Fv(MPI)p 233 2450 14 2 v 16 w(COMM)p 391 2450 V 17 w(W)o(ORLD)h Fx(\(accessible)h(through)f(the)h(function)g Fv(MPI)p 1318 2450 V 15 w(COMM)p 1475 2450 V 17 w(GROUP)p Fx(\).)189 2557 y Fl(R)n(ationale.)74 b Fx(In)21 b(what)g(follo)o(ws,)h (there)e(is)i(no)f(group)f(duplication)j(function)f(analogous)f(to)189 2613 y Fv(MPI)p 274 2613 V 15 w(COMM)p 431 2613 V 17 w(DUP)p Fx(,)16 b(de\014ned)j(later)e(in)g(this)h(c)o(hapter.)24 b(There)17 b(is)h(no)f(need)g(for)g(a)f(group)h(dupli-)189 2670 y(cator.)27 b(A)18 b(group,)g(once)g(created,)g(can)g(ha)o(v)o(e)g (sev)o(eral)g(references)g(to)g(it)g(b)o(y)g(making)g(copies)g(of)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 137 145 137 144 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fx(137)189 45 y(the)15 b(handle.)23 b(The)15 b(follo)o(wing)i (constructors)d(address)i(the)g(need)g(for)f(subsets)h(and)f(sup)q (ersets)h(of)189 102 y(existing)g(groups.)j(\()p Fl(End)d(of)g(r)n (ationale.)p Fx(\))189 208 y Fl(A)n(dvic)n(e)j(to)h(implementors.)65 b Fx(Eac)o(h)19 b(group)g(constructor)g(b)q(eha)o(v)o(es)h(as)e(if)i (it)g(returned)g(a)f(new)189 264 y(group)h(ob)s(ject.)35 b(When)20 b(this)h(new)g(group)f(is)h(a)f(cop)o(y)g(of)g(an)g(existing) i(group,)f(then)f(one)h(can)189 321 y(a)o(v)o(oid)16 b(creating)g(suc)o(h)h(new)g(ob)s(jects,)e(using)j(a)e(reference-coun)o (t)h(mec)o(hanism.)24 b(\()p Fl(End)17 b(of)g(advic)n(e)189 377 y(to)f(implementors.)p Fx(\))75 578 y Fv(MPI)p 160 578 14 2 v 16 w(COMM)p 318 578 V 16 w(GROUP\(comm)n(,)c(group\))117 655 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 730 y(OUT)108 b Fv(group)479 b Fk(group)14 b(corresp)q(onding)h(to)f Fi(comm)8 b Fk(\(handle\))75 855 y Fu(int)23 b(MPI)p 245 855 15 2 v 17 w(Comm)p 358 855 V 17 w(group\(MPI)p 591 855 V 16 w(Comm)g(comm,)g(MPI)p 941 855 V 17 w(Group)g(*group\))75 941 y(MPI)p 150 941 V 17 w(COMM)p 263 941 V 16 w(GROUP\(COMM,)g(GROUP,)g(IERROR\))170 998 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)166 1084 y Fv(MPI)p 251 1084 14 2 v 16 w(COMM)p 409 1084 V 16 w(GROUP)17 b Fx(returns)e(in)h Fv(group)f Fx(a)g(handle)h(to)f(the)g(group)g(of)g Fv(comm)m Fx(.)75 1235 y Fv(MPI)p 160 1235 V 16 w(GROUP)p 328 1235 V 18 w(UNION\(group1,)f(group2,)h(newgroup\))117 1312 y Fk(IN)155 b Fv(group1)456 b Fk(\014rst)15 b(group)f(\(handle\)) 117 1387 y(IN)155 b Fv(group2)456 b Fk(second)15 b(group)f(\(handle\)) 117 1463 y(OUT)108 b Fv(newgroup)405 b Fk(union)14 b(group)f (\(handle\))75 1587 y Fu(int)23 b(MPI)p 245 1587 15 2 v 17 w(Group)p 382 1587 V 17 w(union\(MPI)p 615 1587 V 16 w(Group)g(group1,)g(MPI)p 1037 1587 V 17 w(Group)g(group2,)g(MPI)p 1460 1587 V 16 w(Group)h(*newgroup\))75 1673 y(MPI)p 150 1673 V 17 w(GROUP)p 287 1673 V 16 w(UNION\(GROUP1,)e(GROUP2,)h (NEWGROUP,)g(IERROR\))170 1730 y(INTEGER)g(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 1911 y Fv(MPI)p 160 1911 14 2 v 16 w(GROUP)p 328 1911 V 18 w(INTERSECTION\(group1,)15 b(group2,)f(newgroup\))117 1988 y Fk(IN)155 b Fv(group1)456 b Fk(\014rst)15 b(group)f(\(handle\))117 2063 y(IN)155 b Fv(group2)456 b Fk(second)15 b(group)f(\(handle\))117 2138 y(OUT)108 b Fv(newgroup)405 b Fk(in)o(tersection)15 b(group)f(\(handle\))75 2263 y Fu(int)23 b(MPI)p 245 2263 15 2 v 17 w(Group)p 382 2263 V 17 w(intersection\(MPI)p 783 2263 V 15 w(Group)g(group1,)g(MPI)p 1204 2263 V 17 w(Group)g(group2,)393 2319 y(MPI)p 468 2319 V 17 w(Group)g(*newgroup\)) 75 2406 y(MPI)p 150 2406 V 17 w(GROUP)p 287 2406 V 16 w(INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 2462 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 138 146 138 145 bop 75 -100 a Fx(138)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(DIFFERENCE\(group1,)f(group2,)g(newgroup\))117 122 y Fk(IN)155 b Fv(group1)456 b Fk(\014rst)15 b(group)f(\(handle\))117 197 y(IN)155 b Fv(group2)456 b Fk(second)15 b(group)f(\(handle\))117 273 y(OUT)108 b Fv(newgroup)405 b Fk(di\013erence)16 b(group)e(\(handle\))75 397 y Fu(int)23 b(MPI)p 245 397 15 2 v 17 w(Group)p 382 397 V 17 w(difference\(MPI)p 735 397 V 15 w(Group)g(group1,)g(MPI)p 1156 397 V 17 w(Group)g(group2,)393 453 y(MPI)p 468 453 V 17 w(Group)g(*newgroup\))75 540 y(MPI)p 150 540 V 17 w(GROUP)p 287 540 V 16 w(DIFFERENCE\(GROUP1,)f (GROUP2,)h(NEWGROUP,)f(IERROR\))170 596 y(INTEGER)h(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 683 y Fx(The)15 b(set-lik)o(e)i(op)q(erations)e (are)g(de\014ned)i(as)d(follo)o(ws:)75 777 y Fq(union)24 b Fx(All)19 b(elemen)o(ts)g(of)f(the)h(\014rst)f(group)g(\()p Fv(group1)p Fx(\),)g(follo)o(w)o(ed)h(b)o(y)f(all)h(elemen)o(ts)g(of)f (second)h(group)189 833 y(\()p Fv(group2)p Fx(\))14 b(not)h(in)h (\014rst.)75 927 y Fq(in)o(tersect)23 b Fx(all)18 b(elemen)o(ts)h(of)e (the)g(\014rst)h(group)f(that)g(are)g(also)g(in)i(the)f(second)g (group,)f(ordered)h(as)f(in)189 983 y(\014rst)d(group.)75 1077 y Fq(di\013erence)22 b Fx(all)c(elemen)o(ts)f(of)g(the)g(\014rst)f (group)g(that)g(are)h(not)f(in)h(the)g(second)g(group,)g(ordered)g(as)f (in)189 1134 y(the)f(\014rst)g(group.)75 1227 y(Note)k(that)g(for)g (these)h(op)q(erations)g(the)f(order)h(of)f(pro)q(cesses)h(in)g(the)g (output)f(group)h(is)g(determined)75 1284 y(primarily)g(b)o(y)f(order)g (in)h(the)f(\014rst)g(group)f(\(if)h(p)q(ossible\))i(and)e(then,)h(if)f (necessary)l(,)h(b)o(y)f(order)g(in)h(the)75 1340 y(second)c(group.)j (Neither)d(union)h(nor)e(in)o(tersection)h(are)e(comm)o(utativ)o(e,)h (but)g(b)q(oth)g(are)g(asso)q(ciativ)o(e.)166 1397 y(The)g(new)h(group) f(can)g(b)q(e)h(empt)o(y)l(,)f(that)f(is,)h(equal)h(to)f Fi(MPI)p 1184 1397 13 2 v 14 w(GROUP)p 1340 1397 V 13 w(EMPTY)p Fx(.)75 1548 y Fv(MPI)p 160 1548 14 2 v 16 w(GROUP)p 328 1548 V 18 w(INCL\(group,)e(n,)j(ranks,)f(newgroup\))117 1625 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 1700 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g (in)f(arra)o(y)h(ranks)g(\(and)g(size)h(of)e Fi(new-)905 1757 y(group)p Fk(\))f(\(in)o(teger\))117 1832 y(IN)155 b Fv(ranks)487 b Fk(ranks)13 b(of)f(pro)q(cesses)i(in)e Fi(group)h Fk(to)f(app)q(ear)h(in)f Fi(newgroup)i Fk(\(ar-)905 1888 y(ra)o(y)g(of)f(in)o(tegers\))117 1963 y(OUT)108 b Fv(newgroup)405 b Fk(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v) o(e,)h(in)g(the)h(order)h(de\014ned)f(b)o(y)905 2020 y Fi(ranks)j Fk(\(handle\))75 2144 y Fu(int)23 b(MPI)p 245 2144 15 2 v 17 w(Group)p 382 2144 V 17 w(incl\(MPI)p 591 2144 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 2144 V 17 w(Group)h(*newgroup\))75 2231 y(MPI)p 150 2231 V 17 w(GROUP)p 287 2231 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 2287 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 2374 y Fx(The)17 b(function)h Fv(MPI)p 526 2374 14 2 v 16 w(GROUP)p 694 2374 V 18 w(INCL)f Fx(creates)g(a)g(group)f Fv(newgroup)i Fx(that)f(consists)g(of)g(the)g Fv(n)h Fx(pro-)75 2430 y(cesses)e(in)h Fv(group)f Fx(with)h(ranks)e Fv(rank[0],)p Fp(:)8 b(:)g(:)n Fv(,)16 b(rank[n-1])p Fx(;)g(the)h(pro)q(cess)f(with)g(rank)g Fv(i)g Fx(in)h Fv(newgroup)f Fx(is)h(the)75 2487 y(pro)q(cess)h(with)f(rank)g Fv(ranks[i])h Fx(in)g Fv(group)p Fx(.)26 b(Eac)o(h)17 b(of)g(the)g Fv(n)h Fx(elemen)o(ts)g(of)f Fv(ranks)h Fx(m)o(ust)e(b)q(e)i(a)f(v)m(alid)i(rank)75 2543 y(in)h Fv(group)f Fx(and)g(all)h(elemen)o(ts)g(m)o(ust)f(b)q(e)g(distinct,)i (or)e(else)h(the)f(program)f(is)i(erroneous.)31 b(If)20 b Fv(n)f Fx(=)39 b(0,)75 2599 y(then)19 b Fv(newgroup)f Fx(is)h Fi(MPI)p 508 2599 13 2 v 14 w(GROUP)p 664 2599 V 13 w(EMPTY)p Fx(.)f(This)g(function)h(can,)g(for)f(instance,)h(b)q(e) g(used)g(to)e(reorder)75 2656 y(the)e(elemen)o(ts)h(of)f(a)g(group.)k (See)d(also)f Fv(MPI)p 832 2656 14 2 v 16 w(GROUP)p 1000 2656 V 18 w(COMP)l(ARE)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 139 147 139 146 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fx(139)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(EX)o(CL\(group,)14 b(n,)h(ranks,)g(newgroup\))117 122 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 196 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 269 y(IN)155 b Fv(ranks)487 b Fk(arra)o(y)15 b(of)f(in)o(teger)i(ranks)f(in)g Fi(group)g Fk(not)g(to)g(app)q(ear)g(in)g Fi(new-)905 326 y(group)117 399 y Fk(OUT)108 b Fv(newgroup)405 b Fk(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving)h (the)f(order)905 455 y(de\014ned)d(b)o(y)28 b Fi(group)14 b Fk(\(handle\))75 580 y Fu(int)23 b(MPI)p 245 580 15 2 v 17 w(Group)p 382 580 V 17 w(excl\(MPI)p 591 580 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 580 V 17 w(Group)h(*newgroup\))75 666 y(MPI)p 150 666 V 17 w(GROUP)p 287 666 V 16 w(EXCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 723 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 809 y Fx(The)12 b(function)h Fv(MPI)p 516 809 14 2 v 16 w(GROUP)p 684 809 V 17 w(EX)o(CL)f Fx(creates)g(a)f(group)h(of)g(pro)q(cesses)g Fv(newgroup)h Fx(that)e(is)h(obtained)75 866 y(b)o(y)k(deleting)i(from)e Fv(group)g Fx(those)g(pro)q(cesses)h(with)f(ranks)g Fv(ranks[0])h(,)p Fp(:)8 b(:)g(:)14 b Fv(ranks[n-1])p Fx(.)24 b(The)17 b(ordering)f(of)75 922 y(pro)q(cesses)f(in)h Fv(newgroup)g Fx(is)f(iden)o(tical)i(to)e(the)g(ordering)g(in)h Fv(group)p Fx(.)j(Eac)o(h)c(of)g(the)g Fv(n)g Fx(elemen)o(ts)h(of)e Fv(ranks)75 979 y Fx(m)o(ust)h(b)q(e)h(a)f(v)m(alid)i(rank)e(in)h Fv(group)f Fx(and)h(all)g(elemen)o(ts)g(m)o(ust)f(b)q(e)h(distinct;)g (otherwise,)f(the)g(program)f(is)75 1035 y(erroneous.)20 b(If)15 b Fv(n)h Fx(=)28 b(0,)14 b(then)i Fv(newgroup)g Fx(is)g(iden)o(tical)h(to)d Fv(group)p Fx(.)75 1186 y Fv(MPI)p 160 1186 V 16 w(GROUP)p 328 1186 V 18 w(RANGE)p 494 1186 V 17 w(INCL\(group,)g(n,)h(ranges,)g(newgroup\))117 1263 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 1336 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(triplets)g(in)f (arra)o(y)28 b Fi(ranges)14 b Fk(\(in)o(teger\))117 1410 y(IN)155 b Fv(ranges)466 b Fk(a)11 b(one-dimensional)e(arra)o(y)i(of)g (in)o(teger)g(triplets,)h(of)f(the)h(form)905 1466 y(\(\014rst)i(rank,) e(last)g(rank,)h(stride\))g(indicating)e(ranks)i(in)f Fi(group)905 1523 y Fk(of)h(pro)q(cesses)k(to)d(b)q(e)g(included)g(in)g Fi(newgroup)117 1596 y Fk(OUT)108 b Fv(newgroup)405 b Fk(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v)o(e,)h(in)g(the)h (order)h(de\014ned)f(b)o(y)905 1653 y Fi(ranges)j Fk(\(handle\))75 1777 y Fu(int)23 b(MPI)p 245 1777 15 2 v 17 w(Group)p 382 1777 V 17 w(range)p 519 1777 V 16 w(incl\(MPI)p 727 1777 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 1834 y(MPI)p 468 1834 V 17 w(Group)g(*newgroup\))75 1920 y(MPI)p 150 1920 V 17 w(GROUP)p 287 1920 V 16 w(RANGE)p 423 1920 V 17 w(INCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1976 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 2063 y Fx(If)31 b Fv(ranges)15 b Fx(consist)g(of)g(the)h(triplets)189 2148 y(\()p Fp(f)5 b(ir)q(st)309 2155 y Fg(1)328 2148 y Fp(;)j(l)q(ast)425 2155 y Fg(1)444 2148 y Fp(;)g(str)q(ide)585 2155 y Fg(1)604 2148 y Fx(\))p Fp(;)g(:::;)g Fx(\()p Fp(f)d(irst)822 2155 y Ff(n)844 2148 y Fp(;)j(l)q(ast)941 2155 y Ff(n)964 2148 y Fp(;)g(str)q(ide)1105 2155 y Ff(n)1127 2148 y Fx(\))75 2233 y(then)16 b Fv(newgroup)f Fx(consists)h(of)f(the)g (sequence)h(of)f(pro)q(cesses)h(in)g Fv(group)f Fx(with)g(ranks)189 2337 y Fp(f)5 b(ir)q(st)291 2344 y Fg(1)311 2337 y Fp(;)j(f)d(ir)q(st) 434 2344 y Fg(1)463 2337 y Fx(+)10 b Fp(str)q(ide)628 2344 y Fg(1)648 2337 y Fp(;)e(:::;)g(f)d(ir)q(st)831 2344 y Fg(1)859 2337 y Fx(+)904 2277 y Fe(\026)933 2306 y Fp(l)q(ast)1009 2313 y Fg(1)1039 2306 y Ft(\000)11 b Fp(f)5 b(ir)q(st)1187 2313 y Fg(1)p 933 2326 274 2 v 1000 2368 a Fp(str)q(ide)1120 2375 y Fg(1)1212 2277 y Fe(\027)1243 2337 y Fp(str)q(ide)1363 2344 y Fg(1)1383 2337 y Fp(;)j(:::)189 2479 y(f)d(ir)q(st)291 2486 y Ff(n)314 2479 y Fp(;)j(f)d(ir)q(st)437 2486 y Ff(n)471 2479 y Fx(+)10 b Fp(str)q(ide)636 2486 y Ff(n)659 2479 y Fp(;)e(:::;)g(f)d(ir) q(st)842 2486 y Ff(n)874 2479 y Fx(+)919 2420 y Fe(\026)948 2448 y Fp(l)q(ast)1024 2455 y Ff(n)1058 2448 y Ft(\000)10 b Fp(f)5 b(ir)q(st)1205 2455 y Ff(n)p 948 2469 282 2 v 1017 2510 a Fp(str)q(ide)1137 2517 y Ff(n)1234 2420 y Fe(\027)1266 2479 y Fp(str)q(ide)1386 2486 y Ff(n)1409 2479 y Fp(:)166 2591 y Fx(Eac)o(h)17 b(computed)h(rank)f(m)o(ust)g(b)q (e)i(a)e(v)m(alid)i(rank)e(in)i Fv(group)e Fx(and)h(all)g(computed)g (ranks)f(m)o(ust)g(b)q(e)75 2647 y(distinct,)c(or)f(else)g(the)g (program)f(is)h(erroneous.)19 b(Note)11 b(that)g(w)o(e)h(ma)o(y)f(ha)o (v)o(e)h Fp(f)5 b(ir)q(st)1466 2654 y Ff(i)1493 2647 y Fp(>)13 b(l)q(ast)1617 2654 y Ff(i)1631 2647 y Fx(,)f(and)g Fp(str)q(ide)1861 2654 y Ff(i)75 2704 y Fx(ma)o(y)i(b)q(e)i(negativ)o (e,)f(but)h(cannot)f(b)q(e)g(zero.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 140 148 140 147 bop 75 -100 a Fx(140)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)166 45 y Fx(The)j(functionalit)o(y)h(of)e(this)i(routine)f(is)g(sp)q (eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i(the)f (arra)o(y)75 102 y(of)g(ranges)g(to)g(an)h(arra)o(y)e(of)h(the)h (included)i(ranks)d(and)h(passing)g(the)g(resulting)g(arra)o(y)e(of)i (ranks)f(and)75 158 y(other)e(argumen)o(ts)g(to)g Fv(MPI)p 560 158 14 2 v 16 w(GROUP)p 728 158 V 18 w(INCL)p Fx(.)g(A)g(call)i(to) e Fv(MPI)p 1148 158 V 16 w(GROUP)p 1316 158 V 18 w(INCL)g Fx(is)h(equiv)m(alen)o(t)h(to)e(a)h(call)75 214 y(to)e Fv(MPI)p 216 214 V 16 w(GROUP)p 384 214 V 18 w(RANGE)p 550 214 V 18 w(INCL)g Fx(with)i(eac)o(h)f(rank)g Fv(i)g Fx(in)h Fv(ranks)f Fx(replaced)i(b)o(y)e(the)g(triplet)h Fu(\(i,i,1\))e Fx(in)75 271 y(the)g(argumen)o(t)g Fv(ranges)p Fx(.)75 422 y Fv(MPI)p 160 422 V 16 w(GROUP)p 328 422 V 18 w(RANGE)p 494 422 V 17 w(EX)o(CL\(group,)g(n,)g(ranges,)g (newgroup\))117 499 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 574 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 649 y(IN)155 b Fv(ranges)466 b Fk(a)13 b(one-dimensional)d(arra)o(y)j (of)f(in)o(teger)h(triplets)g(of)f(the)i(form)905 706 y(\(\014rst)j(rank,)f(last)g(rank,)g(stride\),)h(indicating)e(the)i (ranks)f(in)905 762 y Fi(group)22 b Fk(of)f(pro)q(cesses)j(to)d(b)q(e)h (excluded)g(from)e(the)i(output)905 819 y(group)14 b Fi(newgroup)p Fk(.)117 894 y(OUT)108 b Fv(newgroup)405 b Fk(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 950 y(in)c Fi(group)g Fk(\(handle\))75 1075 y Fu(int)23 b(MPI)p 245 1075 15 2 v 17 w(Group)p 382 1075 V 17 w(range)p 519 1075 V 16 w(excl\(MPI)p 727 1075 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 1131 y(MPI)p 468 1131 V 17 w(Group)g(*newgroup\))75 1218 y(MPI)p 150 1218 V 17 w(GROUP)p 287 1218 V 16 w(RANGE)p 423 1218 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1274 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1361 y Fx(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q(e)i(a)f(v)m (alid)h(rank)f(in)h Fv(group)f Fx(and)g(all)h(computed)f(ranks)g(m)o (ust)f(b)q(e)i(distinct,)75 1417 y(or)i(else)h(the)f(program)f(is)i (erroneous.)166 1473 y(The)i(functionalit)o(y)h(of)e(this)i(routine)f (is)g(sp)q(eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i (the)f(arra)o(y)75 1530 y(of)g(ranges)f(to)h(an)g(arra)o(y)f(of)h(the)g (excluded)i(ranks)e(and)g(passing)h(the)f(resulting)h(arra)o(y)e(of)h (ranks)g(and)75 1586 y(other)c(argumen)o(ts)g(to)g Fv(MPI)p 554 1586 14 2 v 16 w(GROUP)p 722 1586 V 18 w(EX)o(CL)p Fx(.)g(A)h(call)h(to)e Fv(MPI)p 1146 1586 V 16 w(GROUP)p 1314 1586 V 18 w(EX)o(CL)g Fx(is)i(equiv)m(alen)o(t)g(to)e(a)h(call)75 1643 y(to)g Fv(MPI)p 216 1643 V 15 w(GROUP)p 383 1643 V 18 w(RANGE)p 549 1643 V 18 w(EX)o(CL)g Fx(with)h(eac)o(h)f(rank)g Fv(i)g Fx(in)h Fv(ranks)g Fx(replaced)g(b)o(y)f(the)h(triplet)g Fu(\(i,i,1\))e Fx(in)75 1699 y(the)h(argumen)o(t)g Fv(ranges)p Fx(.)189 1806 y Fl(A)n(dvic)n(e)22 b(to)h(users.)86 b Fx(The)23 b(range)f(op)q(erations)h(do)f(not)h(explicitly)i(en)o (umerate)e(ranks,)h(and)189 1862 y(therefore)c(are)h(more)f(scalable)i (if)f(implemen)o(ted)i(e\016cien)o(tly)l(.)38 b(Hence,)23 b(w)o(e)d(recommend)i Fv(MPI)189 1918 y Fx(programmers)14 b(to)i(use)g(them)h(whenenev)o(er)g(p)q(ossible,)g(as)f(high-qualit)o (y)i(implemen)o(tations)f(will)189 1975 y(tak)o(e)d(adv)m(an)o(tage)h (of)g(this)g(fact.)k(\()p Fl(End)d(of)g(advic)n(e)h(to)f(users.)p Fx(\))189 2081 y Fl(A)n(dvic)n(e)h(to)h(implementors.)52 b Fx(The)17 b(range)g(op)q(erations)g(should)h(b)q(e)g(implemen)o(ted,) h(if)f(p)q(ossible,)189 2138 y(without)e(en)o(umerating)i(the)f(group)f (mem)o(b)q(ers,)h(in)h(order)e(to)g(obtain)i(b)q(etter)e(scalabilit)o (y)j(\(time)189 2194 y(and)c(space\).)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 2316 y Fj(5.3.3)49 b(Group)16 b(Destructo)o(rs)75 2496 y Fv(MPI)p 160 2496 V 16 w(GROUP)p 328 2496 V 18 w(FREE\(group\))117 2573 y Fk(INOUT)62 b Fv(group)479 b Fk(group)14 b(\(handle\))75 2698 y Fu(int)23 b(MPI)p 245 2698 15 2 v 17 w(Group)p 382 2698 V 17 w(free\(MPI)p 591 2698 V 16 w(Group)g(*group\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 141 149 141 148 bop 75 -100 a Fm(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b Fx(141)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(FREE\(GROUP,)23 b(IERROR\))170 102 y(INTEGER)g(GROUP,)g(IERROR)166 188 y Fx(This)f(op)q(eration)h (marks)e(a)g(group)h(ob)s(ject)f(for)h(deallo)q(cation.)41 b(The)22 b(handle)i Fv(group)d Fx(is)i(set)f(to)75 244 y Fi(MPI)p 152 244 13 2 v 14 w(GROUP)p 308 244 V 13 w(NULL)17 b Fx(b)o(y)g(the)f(call.)24 b(An)o(y)17 b(on-going)f(op)q(eration)g (using)h(this)g(group)f(will)i(complete)f(nor-)75 301 y(mally)l(.)189 395 y Fl(A)n(dvic)n(e)c(to)h(implementors.)39 b Fx(One)13 b(can)h(k)o(eep)f(a)f(reference)i(coun)o(t)f(that)f(is)h (incremen)o(ted)i(for)d(eac)o(h)189 452 y(call)j(to)f Fv(MPI)p 411 452 14 2 v 16 w(COMM)p 569 452 V 16 w(CREA)l(TE)i Fx(and)e Fv(MPI)p 939 452 V 16 w(COMM)p 1097 452 V 17 w(DUP)p Fx(,)g(and)g(decremen)o(ted)i(for)d(eac)o(h)i(call)h(to)189 508 y Fv(MPI)p 274 508 V 15 w(GROUP)p 441 508 V 18 w(FREE)h Fx(or)g Fv(MPI)p 725 508 V 15 w(COMM)p 882 508 V 17 w(FREE)p Fx(;)g(the)g(group)f(ob)s(ject)g(is)i(ultimately)g(deallo)q(cated)189 565 y(when)d(the)h(reference)g(coun)o(t)f(drops)g(to)f(zero.)20 b(\()p Fl(End)c(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 706 y Fn(5.4)59 b(Communicato)n(r)19 b(Management)75 808 y Fx(This)e(section)g(describ)q(es)h(the)f(manipulation)h(of)e (comm)o(unicators)g(in)h Fv(MPI)p Fx(.)f(Op)q(erations)h(that)f(access) 75 864 y(comm)o(unicators)f(are)g(lo)q(cal)h(and)f(their)h(execution)g (do)q(es)g(not)f(require)h(in)o(terpro)q(cess)g(comm)o(unication.)75 921 y(Op)q(erations)g(that)e(create)h(comm)o(unicators)f(are)h (collectiv)o(e)i(and)e(ma)o(y)f(require)i(in)o(terpro)q(cess)f(comm)o (u-)75 977 y(nication.)189 1072 y Fl(A)n(dvic)n(e)k(to)h(implementors.) 65 b Fx(High-qualit)o(y)21 b(implemen)o(tations)g(should)f(amortize)f (the)h(o)o(v)o(er-)189 1128 y(heads)15 b(asso)q(ciated)h(with)g(the)f (creation)h(of)e(comm)o(unicators)h(\(for)g(the)g(same)g(group,)g(or)g (subsets)189 1185 y(thereof)t(\))d(o)o(v)o(er)h(sev)o(eral)g(calls,)i (b)o(y)e(allo)q(cating)i(m)o(ultiple)g(con)o(texts)e(with)g(one)h (collectiv)o(e)h(comm)o(u-)189 1241 y(nication.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 1361 y Fj(5.4.1)49 b(Comm)o(unicato)o(r)13 b(Accesso)o(rs)75 1447 y Fx(The)i(follo)o(wing)h(are)f(all)h(lo)q(cal)h(op)q(erations.)75 1598 y Fv(MPI)p 160 1598 V 16 w(COMM)p 318 1598 V 16 w(SIZE\(comm)m(,)12 b(size\))117 1675 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1749 y(OUT)108 b Fv(size)519 b Fk(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g (group)g(of)27 b Fi(comm)8 b Fk(\(in)o(teger\))75 1873 y Fu(int)23 b(MPI)p 245 1873 15 2 v 17 w(Comm)p 358 1873 V 17 w(size\(MPI)p 567 1873 V 16 w(Comm)g(comm,)g(int)h(*size\))75 1960 y(MPI)p 150 1960 V 17 w(COMM)p 263 1960 V 16 w(SIZE\(COMM,)f (SIZE,)g(IERROR\))170 2016 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)189 2140 y Fl(R)n(ationale.)51 b Fx(This)17 b(function)h(is)g(equiv)m(alen) o(t)g(to)f(accessing)h(the)f(comm)o(unicator's)f(group)h(with)189 2197 y Fv(MPI)p 274 2197 14 2 v 15 w(COMM)p 431 2197 V 17 w(GROUP)k Fx(\(see)g(b)q(elo)o(w\),)g(computing)g(the)f(size)i (using)f Fv(MPI)p 1494 2197 V 15 w(GROUP)p 1661 2197 V 18 w(SIZE)p Fx(,)f(and)189 2253 y(then)13 b(freeing)h(the)f(group)g (temp)q(orary)f(via)h Fv(MPI)p 1016 2253 V 16 w(GROUP)p 1184 2253 V 18 w(FREE)p Fx(.)g(Ho)o(w)o(ev)o(er,)f(this)i(function)g (is)f(so)189 2310 y(commonly)i(used,)g(that)g(this)h(shortcut)e(w)o(as) h(in)o(tro)q(duced.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 2404 y Fl(A)n(dvic)n(e)k(to)i(users.)80 b Fx(This)23 b(function)f(indicates)i(the)e(n)o(um)o(b)q(er)g(of)f(pro)q(cesses)i (in)o(v)o(olv)o(ed)g(in)f(a)189 2461 y(comm)o(unicator.)35 b(F)l(or)21 b Fi(MPI)p 684 2461 13 2 v 14 w(COMM)p 828 2461 V 14 w(W)o(ORLD)p Fx(,)g(it)g(indicates)h(the)e(total)g(n)o(um)o (b)q(er)h(of)g(pro)q(cesses)189 2517 y(a)o(v)m(ailable)c(\(for)f(this)g (v)o(ersion)h(of)f Fv(MPI)p Fx(,)f(there)h(is)h(no)f(standard)g(w)o(a)o (y)f(to)g(c)o(hange)i(the)f(n)o(um)o(b)q(er)g(of)189 2574 y(pro)q(cesses)f(once)h(initialization)i(has)d(tak)o(en)g (place\).)189 2647 y(This)20 b(call)h(is)f(often)g(used)g(with)h(the)f (next)g(call)h(to)e(determine)i(the)f(amoun)o(t)f(of)g(concurrency)189 2704 y(a)o(v)m(ailable)j(for)e(a)h(sp)q(eci\014c)i(library)e(or)g (program.)36 b(The)21 b(follo)o(wing)g(call,)i Fv(MPI)p 1580 2704 14 2 v 16 w(COMM)p 1738 2704 V 17 w(RANK)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 142 150 142 149 bop 75 -100 a Fx(142)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)189 45 y Fx(indicates)g(the)f(rank)f(of)h(the)g(pro)q(cess)g(that)f(calls)i (it)f(in)h(the)f(range)f(from)g(0)8 b Fp(:)g(:)g(:)n Fv(size)p Ft(\000)p Fx(1,)14 b(where)g Fv(size)189 102 y Fx(is)h(the)h(return)f(v)m(alue)h(of)f Fv(MPI)p 706 102 14 2 v 16 w(COMM)p 864 102 V 16 w(SIZE)p Fx(.\()p Fl(End)g(of)i(advic)n(e)f(to)g(users.)p Fx(\))75 290 y Fv(MPI)p 160 290 V 16 w(COMM)p 318 290 V 16 w(RANK\(comm)m(,)c (rank\))117 367 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 440 y(OUT)108 b Fv(rank)504 b Fk(rank)12 b(of)f(the)h(calling)e(pro)q(cess)j(in)f(group)f(of)23 b Fi(comm)5 b Fk(\(in)o(teger\))75 565 y Fu(int)23 b(MPI)p 245 565 15 2 v 17 w(Comm)p 358 565 V 17 w(rank\(MPI)p 567 565 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 651 y(MPI)p 150 651 V 17 w(COMM)p 263 651 V 16 w(RANK\(COMM,)f(RANK,)g(IERROR\))170 708 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)189 831 y Fl(R)n(ationale.)51 b Fx(This)17 b(function)h(is)g(equiv)m(alen)o(t)g(to)f(accessing)h(the) f(comm)o(unicator's)f(group)h(with)189 888 y Fv(MPI)p 274 888 14 2 v 15 w(COMM)p 431 888 V 17 w(GROUP)h Fx(\(see)e(b)q(elo)o (w\),)h(computing)g(the)g(size)g(using)h Fv(MPI)p 1467 888 V 15 w(GROUP)p 1634 888 V 18 w(RANK)p Fx(,)f(and)189 944 y(then)c(freeing)h(the)f(group)g(temp)q(orary)f(via)h Fv(MPI)p 1016 944 V 16 w(GROUP)p 1184 944 V 18 w(FREE)p Fx(.)g(Ho)o(w)o(ev)o(er,)f(this)i(function)g(is)f(so)189 1001 y(commonly)i(used,)g(that)g(this)h(shortcut)e(w)o(as)h(in)o(tro)q (duced.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 1094 y Fl(A)n(dvic)n(e)c(to)h(users.)38 b Fx(This)14 b(function)f(giv)o(es)g(the)g(rank)f(of)h(the)f(pro)q(cess)h(in)h(the)f (particular)g(comm)o(u-)189 1151 y(nicator's)h(group.)20 b(It)15 b(is)h(useful,)g(as)f(noted)g(ab)q(o)o(v)o(e,)f(in)i (conjunction)h(with)e Fv(MPI)p 1569 1151 V 16 w(COMM)p 1727 1151 V 16 w(SIZE)p Fx(.)189 1224 y(Man)o(y)e(programs)g(will)i(b)q (e)g(written)f(with)g(the)g(master-sla)o(v)o(e)f(mo)q(del,)i(where)f (one)g(pro)q(cess)g(\(suc)o(h)189 1280 y(as)19 b(the)g(rank-zero)h(pro) q(cess\))f(will)i(pla)o(y)f(a)f(sup)q(ervisory)h(role,)h(and)f(the)f (other)g(pro)q(cesses)h(will)189 1337 y(serv)o(e)g(as)f(compute)h(no)q (des.)35 b(In)21 b(this)g(framew)o(ork,)e(the)h(t)o(w)o(o)f(preceding)j (calls)f(are)e(useful)j(for)189 1393 y(determining)c(the)e(roles)h(of)f (the)h(v)m(arious)g(pro)q(cesses)g(of)f(a)g(comm)o(unicator.)24 b(\()p Fl(End)17 b(of)g(advic)n(e)h(to)189 1450 y(users.)p Fx(\))75 1638 y Fv(MPI)p 160 1638 V 16 w(COMM)p 318 1638 V 16 w(COMP)l(ARE\(comm)n(1,)12 b(comm)m(2,)g(result\))117 1715 y Fk(IN)155 b Fv(comm)n(1)445 b Fk(\014rst)15 b(comm)o(unicator)c (\(handle\))117 1788 y(IN)155 b Fv(comm)n(2)445 b Fk(second)15 b(comm)o(unicator)c(\(handle\))117 1862 y(OUT)108 b Fv(result)484 b Fk(result)15 b(\(in)o(teger\))75 1986 y Fu(int)23 b(MPI)p 245 1986 15 2 v 17 w(Comm)p 358 1986 V 17 w(compare\(MPI)p 639 1986 V 16 w(Comm)g(comm1,MPI)p 990 1986 V 16 w(Comm)g(comm2,)g(int) h(*result\))75 2073 y(MPI)p 150 2073 V 17 w(COMM)p 263 2073 V 16 w(COMPARE\(COMM1,)e(COMM2,)h(RESULT,)g(IERROR\))170 2129 y(INTEGER)g(COMM1,)g(COMM2,)g(RESULT,)g(IERROR)75 2215 y Fi(MPI)p 152 2215 13 2 v 14 w(IDENT)11 b Fx(results)g(if)h(and)f (only)g(if)h Fv(comm)m(1)c Fx(and)j Fv(comm)m(2)c Fx(are)k(handles)h (for)e(the)h(same)g(ob)s(ject)f(\(iden)o(tical)75 2272 y(groups)15 b(and)g(same)g(con)o(texts\).)j Fi(MPI)p 715 2272 V 15 w(CONGRUENT)c Fx(results)h(if)h(the)f(underlying)i (groups)d(are)h(iden)o(tical)75 2328 y(in)j(constituen)o(ts)f(and)h (rank)f(order;)h(these)f(comm)o(unicators)g(di\013er)h(only)f(b)o(y)h (con)o(text.)25 b Fi(MPI)p 1703 2328 V 15 w(SIMILAR)75 2385 y Fx(results)12 b(if)h(the)f(group)g(mem)o(b)q(ers)g(of)g(b)q(oth) g(comm)o(unicators)g(are)g(the)g(same)g(but)g(the)g(rank)g(order)g (di\013ers.)75 2441 y Fi(MPI)p 152 2441 V 14 w(UNEQUAL)i Fx(results)i(otherwise.)75 2561 y Fj(5.4.2)49 b(Comm)o(unicato)o(r)13 b(Constructo)o(rs)75 2647 y Fx(The)i(follo)o(wing)g(are)g(collectiv)o (e)h(functions)f(that)f(are)h(in)o(v)o(ok)o(ed)g(b)o(y)f(all)i(pro)q (cesses)f(in)g(the)g(group)f(asso)q(ci-)75 2704 y(ated)h(with)h Fv(comm)m Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 143 151 143 150 bop 75 -100 a Fm(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b Fx(143)189 45 y Fl(R)n(ationale.)37 b Fx(Note)10 b(that)g(there)h(is)g(a)g(c)o(hic)o(k)o(en-and-egg)h(asp)q (ect)f(to)f Fv(MPI)g Fx(in)i(that)e(a)g(comm)o(unicator)189 102 y(is)18 b(needed)h(to)e(create)g(a)h(new)f(comm)o(unicator.)27 b(The)18 b(base)g(comm)o(unicator)f(for)g(all)i Fv(MPI)e Fx(com-)189 158 y(m)o(unicators)f(is)h(prede\014ned)h(outside)f(of)f Fv(MPI)p Fx(,)g(and)g(is)h Fi(MPI)p 1227 158 13 2 v 15 w(COMM)p 1372 158 V 14 w(W)o(ORLD)p Fx(.)f(This)h(mo)q(del)h(w)o(as)189 214 y(arriv)o(ed)c(at)g(after)g(considerable)i(debate,)e(and)h(w)o(as)e (c)o(hosen)i(to)f(increase)h(\\safet)o(y")e(of)h(programs)189 271 y(written)h(in)h Fv(MPI)p Fx(.)e(\()p Fl(End)i(of)g(r)n(ationale.)p Fx(\))75 472 y Fv(MPI)p 160 472 14 2 v 16 w(COMM)p 318 472 V 16 w(DUP\(comm)m(,)c(new)o(comm)n(\))117 549 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 624 y(OUT)108 b Fv(new)o(comm)393 b Fk(cop)o(y)14 b(of)f Fi(comm)8 b Fk(\(handle\))75 748 y Fu(int)23 b(MPI)p 245 748 15 2 v 17 w(Comm)p 358 748 V 17 w(dup\(MPI)p 543 748 V 16 w(Comm)g(comm,)h(MPI)p 894 748 V 16 w(Comm)g(*newcomm\))75 835 y(MPI)p 150 835 V 17 w(COMM)p 263 835 V 16 w(DUP\(COMM,)f(NEWCOMM,) g(IERROR\))170 891 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)166 978 y Fv(MPI)p 251 978 14 2 v 16 w(COMM)p 409 978 V 16 w(DUP)12 b Fx(Duplicates)i(the)e(existing)h(comm)o(unicator)f Fv(comm)6 b Fx(with)13 b(asso)q(ciated)f(k)o(ey)g(v)m(al-)75 1034 y(ues.)19 b(F)l(or)11 b(eac)o(h)h(k)o(ey)f(v)m(alue,)j(the)d(resp) q(ectiv)o(e)i(cop)o(y)f(callbac)o(k)g(function)h(determines)f(the)g (attribute)g(v)m(alue)75 1091 y(asso)q(ciated)f(with)h(this)f(k)o(ey)g (in)h(the)f(new)h(comm)o(unicator;)f(one)h(particular)f(action)h(that)e (a)h(cop)o(y)g(callbac)o(k)75 1147 y(ma)o(y)18 b(tak)o(e)g(is)h(to)f (delete)h(the)g(attribute)g(from)e(the)i(new)g(comm)o(unicator.)30 b(Returns)19 b(in)g Fv(new)o(comm)13 b Fx(a)75 1204 y(new)h(comm)o (unicator)f(with)g(the)h(same)f(group,)g(an)o(y)g(copied)i(cac)o(hed)f (information,)f(but)h(a)f(new)h(con)o(text)75 1260 y(\(see)h(section)h (5.7.1\).)189 1366 y Fl(A)n(dvic)n(e)d(to)h(users.)39 b Fx(This)14 b(op)q(eration)f(is)g(used)h(to)e(pro)o(vide)i(a)e (parallel)j(library)f(call)g(with)f(a)g(dupli-)189 1423 y(cate)f(comm)o(unication)i(space)f(that)f(has)h(the)f(same)h(prop)q (erties)g(as)g(the)g(original)h(comm)o(unicator.)189 1479 y(This)i(includes)j(an)o(y)d(attributes)g(\(see)g(b)q(elo)o(w\),)g (and)g(top)q(ologies)h(\(see)f(c)o(hapter)g(6\).)22 b(This)17 b(call)g(is)189 1536 y(v)m(alid)g(ev)o(en)g(if)f(there)g(are)g(p)q (ending)i(p)q(oin)o(t-to-p)q(oin)o(t)e(comm)o(unications)h(in)o(v)o (olving)g(the)f(comm)o(u-)189 1592 y(nicator)i Fv(comm)m Fx(.)25 b(A)18 b(t)o(ypical)h(call)g(migh)o(t)e(in)o(v)o(olv)o(e)i(a)f Fv(MPI)p 1206 1592 V 15 w(COMM)p 1363 1592 V 17 w(DUP)g Fx(at)f(the)h(b)q(eginning)i(of)189 1649 y(the)e(parallel)h(call,)g (and)f(an)g Fv(MPI)p 775 1649 V 16 w(COMM)p 933 1649 V 16 w(FREE)h Fx(of)e(that)g(duplicated)j(comm)o(unicator)e(at)f(the) 189 1705 y(end)e(of)g(the)h(call.)21 b(Other)15 b(mo)q(dels)h(of)f (comm)o(unicator)g(managemen)o(t)f(are)h(also)g(p)q(ossible.)189 1780 y(This)g(call)i(applies)g(to)d(b)q(oth)h(in)o(tra-)h(and)f(in)o (ter-comm)o(unicators.)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (users.)p Fx(\))189 1887 y Fl(A)n(dvic)n(e)c(to)j(implementors.)38 b Fx(One)13 b(need)g(not)f(actually)i(cop)o(y)e(the)h(group)f (information,)h(but)f(only)189 1943 y(add)17 b(a)g(new)g(reference)h (and)g(incremen)o(t)g(the)f(reference)h(coun)o(t.)26 b(Cop)o(y)17 b(on)g(write)g(can)g(b)q(e)h(used)189 1999 y(for)c(the)h(cac)o(hed)h(information.\()p Fl(End)g(of)g(advic)n(e)g (to)h(implementors.)p Fx(\))75 2200 y Fv(MPI)p 160 2200 V 16 w(COMM)p 318 2200 V 16 w(CREA)l(TE\(comm)n(,)12 b(group,)j(new)o(comm)m(\))117 2277 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 2353 y(IN)155 b Fv(group)479 b Fk(Group,)14 b(whic)o(h)h(is)f(a)g(subset)i(of)e(the)h (group)g(of)e Fi(comm)c Fk(\(han-)905 2409 y(dle\))117 2484 y(OUT)108 b Fv(new)o(comm)393 b Fk(new)15 b(comm)o(uni)o(cator)d (\(handle\))75 2609 y Fu(int)23 b(MPI)p 245 2609 15 2 v 17 w(Comm)p 358 2609 V 17 w(create\(MPI)p 615 2609 V 16 w(Comm)g(comm,)g(MPI)p 965 2609 V 17 w(Group)g(group,)g(MPI)p 1364 2609 V 17 w(Comm)g(*newcomm\))75 2695 y(MPI)p 150 2695 V 17 w(COMM)p 263 2695 V 16 w(CREATE\(COMM,)g(GROUP,)g(NEWCOMM,)f (IERROR\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 144 152 144 151 bop 75 -100 a Fx(144)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)170 45 y Fu(INTEGER)23 b(COMM,)h(GROUP,)f(NEWCOMM,)f(IERROR)75 132 y Fx(This)14 b(function)g(creates)f(a)g(new)h(comm)o(unicator)f Fv(new)o(comm)8 b Fx(with)14 b(comm)o(unication)g(group)f(de\014ned)i (b)o(y)75 188 y Fv(group)f Fx(and)h(a)e(new)i(con)o(text.)k(No)14 b(cac)o(hed)h(information)f(propagates)f(from)h Fv(comm)8 b Fx(to)14 b Fv(new)o(comm)m Fx(.)j(The)75 244 y(function)f(returns)f Fi(MPI)p 487 244 13 2 v 14 w(COMM)p 631 244 V 15 w(NULL)h Fx(to)e(pro)q(cesses)i(that)e(are)h(not)g(in)h Fv(group)p Fx(.)k(The)15 b(call)h(is)g(erroneous)75 301 y(if)k(not)f(all)h Fv(group)f Fx(argumen)o(ts)g(ha)o(v)o(e)f(the)i(same)f(v)m(alue,)i(or)e (if)h Fv(group)f Fx(is)h(not)f(a)f(subset)i(of)f(the)g(group)75 357 y(asso)q(ciated)d(with)f Fv(comm)n Fx(.)i(Note)e(that)g(the)g(call) i(is)f(to)e(b)q(e)i(executed)h(b)o(y)e(all)h(pro)q(cesses)g(in)g Fv(comm)n Fx(,)c(ev)o(en)75 414 y(if)k(they)f(do)g(not)g(b)q(elong)h (to)f(the)g(new)g(group.)20 b(This)c(call)g(applies)h(only)f(to)e(in)o (tra-comm)o(unicators.)189 516 y Fl(R)n(ationale.)58 b Fx(The)19 b(requiremen)o(t)g(that)f(the)g(en)o(tire)h(group)f(of)g Fv(comm)12 b Fx(participate)19 b(in)g(the)g(call)189 572 y(stems)14 b(from)h(the)g(follo)o(wing)h(considerations:)243 665 y Ft(\017)23 b Fx(It)e(allo)o(ws)g(the)g(implemen)o(tation)h(to)e (la)o(y)o(er)h Fv(MPI)p 1161 665 14 2 v 16 w(COMM)p 1319 665 V 16 w(CREA)l(TE)h Fx(on)f(top)f(of)h(regular)289 721 y(collectiv)o(e)c(comm)o(unications.)243 793 y Ft(\017)23 b Fx(It)12 b(pro)o(vides)h(additional)h(safet)o(y)l(,)e(in)h (particular)g(in)g(the)g(case)f(where)h(partially)g(o)o(v)o(erlapping) 289 849 y(groups)h(are)h(used)h(to)f(create)g(new)g(comm)o(unicators.) 243 921 y Ft(\017)23 b Fx(It)11 b(p)q(ermits)g(implemen)o(tations)i (sometimes)e(to)f(a)o(v)o(oid)h(comm)o(unication)h(related)f(to)g(con)o (text)289 978 y(creation.)189 1070 y(\()p Fl(End)k(of)i(r)n(ationale.)p Fx(\))189 1172 y Fl(A)n(dvic)n(e)g(to)i(users.)53 b Fv(MPI)p 647 1172 V 16 w(COMM)p 805 1172 V 16 w(CREA)l(TE)19 b Fx(pro)o(vides)f(a)f(means)g(to)g(subset)h(a)f(group)g(of)g(pro-)189 1228 y(cesses)e(for)f(the)g(purp)q(ose)i(of)e(separate)g(MIMD)g (computation,)g(with)h(separate)f(comm)o(unication)189 1285 y(space.)33 b Fv(new)o(comm)n Fx(,)18 b(whic)o(h)i(emerges)g(from) f Fv(MPI)p 1072 1285 V 16 w(COMM)p 1230 1285 V 16 w(CREA)l(TE)i Fx(can)f(b)q(e)g(used)h(in)f(subse-)189 1341 y(quen)o(t)15 b(calls)i(to)e Fv(MPI)p 558 1341 V 16 w(COMM)p 716 1341 V 16 w(CREA)l(TE)i Fx(\(or)d(other)i(comm)o(unicator)f(constructors\))f (further)i(to)189 1398 y(sub)q(divide)21 b(a)d(computation)h(in)o(to)g (parallel)h(sub-computations.)31 b(A)18 b(more)h(general)g(service)g (is)189 1454 y(pro)o(vided)d(b)o(y)f Fv(MPI)p 524 1454 V 16 w(COMM)p 682 1454 V 16 w(SPLIT)p Fx(,)f(b)q(elo)o(w.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))189 1556 y Fl(A)n(dvic)n(e)32 b(to)i(implementors.)151 b Fx(Since)35 b(all)g(pro)q(cesses)f(calling)h Fv(MPI)p 1536 1556 V 16 w(COMM)p 1694 1556 V 17 w(DUP)e Fx(or)189 1612 y Fv(MPI)p 274 1612 V 15 w(COMM)p 431 1612 V 17 w(CREA)l(TE)21 b Fx(pro)o(vide)f(the)g(same)g Fv(group)g Fx(argumen)o(t,)f(it)i(is)f(theoretically)h(p)q(ossible)189 1669 y(to)13 b(agree)g(on)g(a)h(group-wide)g(unique)h(con)o(text)e (with)h(no)g(comm)o(unication.)20 b(Ho)o(w)o(ev)o(er,)13 b(lo)q(cal)h(exe-)189 1725 y(cution)f(of)f(these)h(functions)g (requires)h(use)f(of)f(a)g(larger)h(con)o(text)f(name)h(space)f(and)h (reduces)h(error)189 1782 y(c)o(hec)o(king.)20 b(Implemen)o(tations)c (ma)o(y)e(strik)o(e)h(v)m(arious)g(compromises)g(b)q(et)o(w)o(een)g (these)f(con\015icting)189 1838 y(goals,)g(suc)o(h)i(as)e(bulk)j(allo)q (cation)f(of)f(m)o(ultiple)i(con)o(texts)d(in)i(one)g(collectiv)o(e)h (op)q(eration.)189 1913 y(Imp)q(ortan)o(t:)28 b(If)20 b(new)g(comm)o(unicators)g(are)f(created)h(without)g(sync)o(hronizing)i (the)e(pro)q(cesses)189 1969 y(in)o(v)o(olv)o(ed)11 b(then)g(the)g (comm)o(unication)g(system)f(should)i(b)q(e)f(able)g(to)f(cop)q(e)h (with)g(messages)f(arriving)189 2026 y(in)16 b(a)g(con)o(text)f(that)g (has)g(not)h(y)o(et)f(b)q(een)i(allo)q(cated)f(at)f(the)h(receiving)i (pro)q(cess.)j(\()p Fl(End)16 b(of)h(advic)n(e)189 2082 y(to)f(implementors.)p Fx(\))75 2278 y Fv(MPI)p 160 2278 V 16 w(COMM)p 318 2278 V 16 w(SPLIT\(com)o(m)n(,)11 b(colo)o(r,)j(k)o (ey)l(,)h(new)o(comm)n(\))117 2356 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 2430 y(IN)155 b Fv(colo)o(r)495 b Fk(con)o(trol)14 b(of)f(subset)i(assignmen)o(t)e (\(in)o(teger\))117 2505 y(IN)155 b Fv(k)o(ey)525 b Fk(con)o(trol)14 b(of)f(rank)h(assigmen)o(t)f(\(in)o(teger\))117 2579 y(OUT)108 b Fv(new)o(comm)393 b Fk(new)15 b(comm)o(uni)o(cator)d (\(handle\))75 2704 y Fu(int)23 b(MPI)p 245 2704 15 2 v 17 w(Comm)p 358 2704 V 17 w(split\(MPI)p 591 2704 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 2704 V 16 w(Comm)g(*newcomm\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 145 153 145 152 bop 75 -100 a Fm(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b Fx(145)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(SPLIT\(COMM,)23 b(COLOR,)g(KEY,)g (NEWCOMM,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g (NEWCOMM,)g(IERROR)75 188 y Fx(This)c(function)f(partitions)g(the)h (group)e(asso)q(ciated)h(with)h Fv(comm)12 b Fx(in)o(to)18 b(disjoin)o(t)g(subgroups,)h(one)f(for)75 244 y(eac)o(h)g(v)m(alue)g (of)f Fv(colo)o(r)p Fx(.)26 b(Eac)o(h)17 b(subgroup)h(con)o(tains)f (all)i(pro)q(cesses)f(of)f(the)g(same)g(color.)27 b(Within)19 b(eac)o(h)75 301 y(subgroup,)24 b(the)f(pro)q(cesses)g(are)f(rank)o(ed) h(in)g(the)g(order)g(de\014ned)h(b)o(y)e(the)h(v)m(alue)h(of)e(the)h (argumen)o(t)75 357 y Fv(k)o(ey)p Fx(,)d(with)g(ties)g(brok)o(en)g (according)g(to)f(their)h(rank)f(in)i(the)f(old)g(group.)33 b(A)19 b(new)h(comm)o(unicator)f(is)75 414 y(created)14 b(for)f(eac)o(h)h(subgroup)g(and)g(returned)g(in)g Fv(new)o(comm)n Fx(.)i(A)e(pro)q(cess)g(ma)o(y)f(supply)i(the)f(color)g(v)m(alue)75 470 y Fv(MPI)p 160 470 14 2 v 16 w(UNDEFINED)p Fx(,)19 b(in)i(whic)o(h)g(case)e Fv(new)o(comm)c Fx(returns)20 b Fi(MPI)p 1200 470 13 2 v 14 w(COMM)p 1344 470 V 15 w(NULL)p Fx(.)g(This)g(is)h(a)e(collectiv)o(e)75 527 y(call,)d(but)f(eac)o(h)h(pro)q(cess)f(is)h(p)q(ermitted)g(to)e(pro)o (vide)i(di\013eren)o(t)f(v)m(alues)i(for)d Fv(colo)o(r)g Fx(and)i Fv(k)o(ey)p Fx(.)166 583 y(A)47 b(call)i(to)e Fv(MPI)p 536 583 14 2 v 16 w(COMM)p 694 583 V 16 w(CREA)l(TE\(comm)n(,) 52 b(group,)k(new)o(comm)m(\))44 b Fx(is)k(equiv)m(alen)o(t)i(to)75 640 y(a)15 b(call)i(to)d Fv(MPI)p 337 640 V 16 w(COMM)p 495 640 V 17 w(SPLIT\(com)o(m)m(,)e(colo)o(r,)i(k)o(ey)l(,)h(new)o (comm)n(\))o Fx(,)d(where)k(all)g(mem)o(b)q(ers)g(of)f Fv(group)g Fx(pro-)75 696 y(vide)i Fv(colo)o(r)e Fx(=)30 b(0)15 b(and)h Fv(k)o(ey)h Fx(=)32 b(rank)15 b(in)i Fv(group)p Fx(,)f(and)g(all)h(pro)q(cesses)f(that)f(are)h(not)f(mem)o(b)q(ers)i (of)e Fv(group)75 753 y Fx(pro)o(vide)k Fv(colo)o(r)e Fx(=)37 b Fv(MPI)p 506 753 V 16 w(UNDEFINED)p Fx(.)18 b(The)g(function)h Fv(MPI)p 1166 753 V 16 w(COMM)p 1324 753 V 16 w(SPLIT)f Fx(allo)o(ws)h(more)e(general)75 809 y(partitioning)22 b(of)e(a)g(group)h(in)o(to)g(one)g(or)f(more)g (subgroups)h(with)g(optional)h(reordering.)36 b(This)22 b(call)75 865 y(applies)17 b(only)f(in)o(tra-comm)o(unicators.)189 951 y Fl(A)n(dvic)n(e)e(to)h(users.)39 b Fx(This)15 b(is)f(an)g (extremely)h(p)q(o)o(w)o(erful)f(mec)o(hanism)h(for)e(dividing)j(a)e (single)h(com-)189 1007 y(m)o(unicating)h(group)e(of)h(pro)q(cesses)g (in)o(to)g Fp(k)h Fx(subgroups,)f(with)g Fp(k)h Fx(c)o(hosen)g (implicitly)i(b)o(y)d(the)g(user)189 1064 y(\(b)o(y)f(the)h(n)o(um)o(b) q(er)g(of)g(colors)g(asserted)f(o)o(v)o(er)g(all)i(the)f(pro)q (cesses\).)20 b(Eac)o(h)14 b(resulting)i(comm)o(unica-)189 1120 y(tor)g(will)i(b)q(e)f(non-o)o(v)o(erlapping.)26 b(Suc)o(h)17 b(a)g(division)i(could)e(b)q(e)h(useful)g(for)e (de\014ning)i(a)f(hierarc)o(h)o(y)189 1177 y(of)d(computations,)h(suc)o (h)h(as)e(for)h(m)o(ultigrid,)h(or)f(linear)h(algebra.)189 1249 y(Multiple)k(calls)f(to)f Fv(MPI)p 623 1249 V 16 w(COMM)p 781 1249 V 16 w(SPLIT)g Fx(can)h(b)q(e)g(used)g(to)f(o)o(v)o (ercome)f(the)i(requiremen)o(t)g(that)189 1305 y(an)o(y)d(call)h(ha)o (v)o(e)f(no)g(o)o(v)o(erlap)g(of)g(the)g(resulting)i(comm)o(unicators)e (\(eac)o(h)g(pro)q(cess)g(is)h(of)f(only)h(one)189 1362 y(color)i(p)q(er)g(call\).)32 b(In)19 b(this)g(w)o(a)o(y)l(,)g(m)o (ultiple)i(o)o(v)o(erlapping)e(comm)o(unication)h(structures)e(can)h(b) q(e)189 1418 y(created.)h(Creativ)o(e)14 b(use)i(of)f(the)g Fv(colo)o(r)f Fx(and)h Fv(k)o(ey)h Fx(in)g(suc)o(h)f(splitting)i(op)q (erations)e(is)h(encouraged.)189 1490 y(Note)g(that,)g(for)h(a)f (\014xed)i(color,)f(the)g(k)o(eys)f(need)i(not)e(b)q(e)i(unique.)26 b(It)17 b(is)h Fv(MPI)p 1551 1490 V 16 w(COMM)p 1709 1490 V 16 w(SPLIT)p Fx('s)189 1547 y(resp)q(onsibilit)o(y)f(to)c(sort)h (pro)q(cesses)h(in)g(ascending)h(order)e(according)h(to)e(this)i(k)o (ey)l(,)g(and)f(to)g(break)189 1603 y(ties)19 b(in)h(a)e(consisten)o(t) i(w)o(a)o(y)l(.)30 b(If)19 b(all)h(the)f(k)o(eys)g(are)f(sp)q (eci\014ed)k(in)d(the)g(same)g(w)o(a)o(y)l(,)g(then)g(all)h(the)189 1660 y(pro)q(cesses)15 b(in)g(a)f(giv)o(en)h(color)g(will)h(ha)o(v)o(e) e(the)g(relativ)o(e)h(rank)g(order)f(as)g(they)g(did)i(in)f(their)g (paren)o(t)189 1716 y(group.)k(\(In)d(general,)f(they)g(will)i(ha)o(v)o (e)e(di\013eren)o(t)g(ranks.\))189 1788 y(Essen)o(tially)l(,)f(making)e (the)h(k)o(ey)f(v)m(alue)i(zero)f(for)e(all)j(pro)q(cesses)f(of)f(a)g (giv)o(en)h(color)g(means)f(that)g(one)189 1845 y(do)q(esn't)j(really)h (care)f(ab)q(out)g(the)g(rank-order)g(of)g(the)g(pro)q(cesses)h(in)g (the)f(new)h(comm)o(unicator.)189 1917 y(\()p Fl(End)f(of)i(advic)n(e)f (to)g(users.)p Fx(\))75 2036 y Fj(5.4.3)49 b(Comm)o(unicato)o(r)13 b(Destructo)o(rs)75 2216 y Fv(MPI)p 160 2216 V 16 w(COMM)p 318 2216 V 16 w(FREE\(comm)n(\))117 2293 y Fk(INOUT)62 b Fv(comm)466 b Fk(comm)o(unicator)11 b(to)j(b)q(e)h(destro)o(y)o(ed)g (\(handle\))75 2418 y Fu(int)23 b(MPI)p 245 2418 15 2 v 17 w(Comm)p 358 2418 V 17 w(free\(MPI)p 567 2418 V 16 w(Comm)g(*comm\))75 2504 y(MPI)p 150 2504 V 17 w(COMM)p 263 2504 V 16 w(FREE\(COMM,)g(IERROR\))170 2561 y(INTEGER)g(COMM,)h (IERROR)166 2647 y Fx(This)13 b(collectiv)o(e)g(op)q(eration)g(marks)e (the)h(comm)o(unication)h(ob)s(ject)f(for)f(deallo)q(cation.)20 b(The)13 b(handle)75 2704 y(is)e(set)g(to)f Fi(MPI)p 311 2704 13 2 v 14 w(COMM)p 455 2704 V 15 w(NULL)p Fx(.)h(An)o(y)g(p)q (ending)h(op)q(erations)f(that)f(use)h(this)g(comm)o(unicator)g(will)h (complete)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 146 154 146 153 bop 75 -100 a Fx(146)123 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fx(normally;)j(the)f(ob)s(ject)g(is)g(actually)h(deallo)q(cated)h (only)e(if)h(there)f(are)g(no)g(other)g(activ)o(e)g(references)h(to)75 102 y(it.)28 b(This)18 b(call)h(applies)g(to)e(in)o(tra-)h(and)f(in)o (ter-comm)o(unicators.)28 b(The)18 b(delete)g(callbac)o(k)h(functions)g (for)75 158 y(all)d(cac)o(hed)g(attributed)f(\(see)g(section)h(5.7\))e (are)h(called)i(in)f(arbitrary)e(order.)189 264 y Fl(A)n(dvic)n(e)j(to) h(implementors.)52 b Fx(A)17 b(reference-coun)o(t)h(mec)o(hanism)f(ma)o (y)g(b)q(e)h(used:)24 b(the)17 b(reference)189 321 y(coun)o(t)d(is)h (incremen)o(ted)h(b)o(y)e(eac)o(h)h(call)g(to)f Fv(MPI)p 1000 321 14 2 v 16 w(COMM)p 1158 321 V 17 w(DUP)p Fx(,)g(and)g (decremen)o(ted)i(b)o(y)e(eac)o(h)h(call)189 377 y(to)j Fv(MPI)p 333 377 V 16 w(COMM)p 491 377 V 16 w(FREE)p Fx(.)h(The)h(ob)s(ject)e(is)i(ultimately)g(deallo)q(cated)g(when)g(the) f(coun)o(t)g(reac)o(hes)189 434 y(zero.)189 509 y(Though)10 b(collectiv)o(e,)j(it)d(is)h(an)o(ticipated)g(that)f(this)g(op)q (eration)h(will)h(normally)f(b)q(e)g(implemen)o(ted)h(to)189 565 y(b)q(e)h(lo)q(cal,)h(though)f(the)f(debugging)i(v)o(ersion)f(of)f (an)h Fv(MPI)f Fx(library)i(migh)o(t)e(c)o(ho)q(ose)h(to)f(sync)o (hronize.)189 622 y(\()p Fl(End)j(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 765 y Fn(5.5)59 b(Motivating)19 b(Examples)75 868 y Fj(5.5.1)49 b(Current)15 b(Practice)g(#1)75 954 y Fx(Example)h(#1a:)147 1048 y Fu(main\(int)22 b(argc,)i(char)f (**argv\))147 1104 y({)194 1161 y(int)h(me,)f(size;)194 1217 y(...)194 1273 y(MPI_Init)g(\()h(&argc,)f(&argv)g(\);)194 1330 y(MPI_Comm_rank)f(\(MPI_COMM_WORLD,)g(&me\);)194 1386 y(MPI_Comm_size)g(\(MPI_COMM_WORLD,)g(&size\);)194 1499 y(\(void\)printf)h(\("Process)f(\045d)i(size)f(\045d\\n",)g(me,)h (size\);)194 1556 y(...)194 1612 y(MPI_Finalize\(\);)147 1669 y(})75 1762 y Fx(Example)19 b(#1a)f(is)h(a)f(do-nothing)h(program) e(that)h(initializes)k(itself)d(legally)l(,)i(and)d(refers)g(to)g(the)h (the)75 1819 y(\\all")f(comm)o(unicator,)g(and)g(prin)o(ts)g(a)f (message.)27 b(It)18 b(terminates)g(itself)h(legally)g(to)q(o.)27 b(This)18 b(example)75 1875 y(do)q(es)e(not)e(imply)j(that)d Fv(MPI)h Fx(supp)q(orts)g Fu(printf)p Fx(-lik)o(e)h(comm)o(unication)g (itself.)75 1932 y(Example)g(#1b)f(\(supp)q(osing)h(that)e Fu(size)h Fx(is)h(ev)o(en\):)170 2026 y Fu(main\(int)23 b(argc,)g(char)h(**argv\))170 2082 y({)242 2139 y(int)f(me,)h(size;)242 2195 y(int)f(SOME_TAG)g(=)h(0;)242 2251 y(...)242 2308 y(MPI_Init\(&argc,)e(&argv\);)242 2421 y(MPI_Comm_rank\(MPI_COMM_WOR)o (LD,)f(&me\);)71 b(/*)23 b(local)h(*/)242 2477 y (MPI_Comm_size\(MPI_COMM_WOR)o(LD,)d(&size\);)i(/*)g(local)h(*/)242 2590 y(if\(\(me)f(\045)h(2\))f(==)h(0\))242 2647 y({)314 2703 y(/*)f(send)h(unless)f(highest-numbered)e(process)i(*/)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 147 155 147 154 bop 75 -100 a Fm(5.5.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b Fx(147)314 45 y Fu(if\(\(me)23 b(+)g(1\))h(<)g (size\))385 102 y(MPI_Send\(...,)e(me)i(+)g(1,)f(SOME_TAG,)g (MPI_COMM_WORLD\);)242 158 y(})242 214 y(else)314 271 y(MPI_Recv\(...,)f(me)h(-)h(1,)g(SOME_TAG,)e(MPI_COMM_WORLD\);)242 384 y(...)242 440 y(MPI_Finalize\(\);)170 497 y(})75 591 y Fx(Example)16 b(#1b)f(sc)o(hematically)i(illustrates)g(message)e (exc)o(hanges)h(b)q(et)o(w)o(een)g(\\ev)o(en")f(and)h(\\o)q(dd")f(pro-) 75 647 y(cesses)h(in)g(the)f(\\all")g(comm)o(unicator.)75 769 y Fj(5.5.2)49 b(Current)15 b(Practice)g(#2)147 855 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 911 y({)194 967 y(int)h(me,)f(count;)194 1024 y(void)h(*data;)194 1080 y(...)194 1193 y(MPI_Init\(&argc,)e(&argv\);)194 1250 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&me\);)194 1363 y(if\(me)i(==)h(0\))194 1419 y({)290 1476 y(/*)f(get)h(input,)f (create)g(buffer)g(``data'')g(*/)290 1532 y(...)194 1588 y(})194 1701 y(MPI_Bcast\(data,)f(count,)h(MPI_BYTE,)g(0,)g (MPI_COMM_WORLD\);)194 1814 y(...)194 1927 y(MPI_Finalize\(\);)147 1984 y(})75 2090 y Fx(This)16 b(example)g(illustrates)g(the)f(use)h(of) f(a)g(collectiv)o(e)h(comm)o(unication.)75 2212 y Fj(5.5.3)49 b(\(App)o(ro)o(ximate\))14 b(Current)h(Practice)g(#3)123 2297 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))123 2354 y({)170 2410 y(int)h(me,)f(count,)g(count2;)170 2467 y(void)h(*send_buf,)e(*recv_buf,)h(*send_buf2,)f(*recv_buf2;)170 2523 y(MPI_Group)h(MPI_GROUP_WORLD,)f(grprem;)170 2580 y(MPI_Comm)h(commslave;)170 2636 y(static)g(int)h(ranks[])f(=)h({0};) 170 2693 y(...)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 148 156 148 155 bop 75 -100 a Fx(148)123 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)170 45 y Fu(MPI_Init\(&argc,)22 b(&argv\);)170 102 y (MPI_Comm_group\(MPI_COMM_WORLD)o(,)f(&MPI_GROUP_WORLD\);)170 158 y(MPI_Comm_rank\(MPI_COMM_WORLD,)f(&me\);)47 b(/*)24 b(local)f(*/)170 271 y(MPI_Group_excl\(MPI_GROUP_WORL)o(D,)e(1,)j (ranks,)f(&grprem\);)46 b(/*)24 b(local)f(*/)170 327 y(MPI_Comm_create\(MPI_COMM_WORL)o(D,)e(grprem,)i(&commslave\);)170 440 y(if\(me)h(!=)f(0\))170 497 y({)218 553 y(/*)h(compute)f(on)g (slave)g(*/)218 610 y(...)218 666 y(MPI_Reduce\(send_buf,recv_bu)o (ff,count)o(,)e(MPI_INT,)i(MPI_SUM,)g(1,)g(commslave\);)218 723 y(...)170 779 y(})170 835 y(/*)h(zero)f(falls)h(through)e (immediately)h(to)g(this)h(reduce,)f(others)g(do)g(later...)g(*/)170 892 y(MPI_Reduce\(send_buf2,)e(recv_buff2,)i(count2,)433 948 y(MPI_INT,)g(MPI_SUM,)f(0,)i(MPI_COMM_WORLD\);)170 1061 y(MPI_Comm_free\(&commslave\);)170 1118 y (MPI_Group_free\(&MPI_GROUP_WOR)o(LD\);)170 1174 y (MPI_Group_free\(&grprem\);)170 1231 y(MPI_Finalize\(\);)123 1287 y(})75 1389 y Fx(This)18 b(example)h(illustrates)g(ho)o(w)e(a)h (group)f(consisting)i(of)e(all)i(but)f(the)g(zeroth)f(pro)q(cess)h(of)g (the)g(\\all")75 1446 y(group)13 b(is)h(created,)f(and)h(then)g(ho)o(w) f(a)g(comm)o(unicator)g(is)h(formed)f(\()g Fv(comm)m(slave)p Fx(\))d(for)j(that)g(new)h(group.)75 1502 y(The)f(new)h(comm)o (unicator)e(is)i(used)g(in)g(a)e(collectiv)o(e)j(call,)g(and)e(all)h (pro)q(cesses)f(execute)h(a)f(collectiv)o(e)i(call)75 1559 y(in)j(the)33 b Fi(MPI)p 303 1559 13 2 v 15 w(COMM)p 448 1559 V 14 w(W)o(ORLD)18 b Fx(con)o(text.)25 b(This)18 b(example)h(illustrates)f(ho)o(w)f(the)g(t)o(w)o(o)f(comm)o(unicators) 75 1615 y(\(that)c(inheren)o(tly)k(p)q(ossess)d(distinct)i(con)o (texts\))d(protect)h(comm)o(unication.)20 b(That)13 b(is,)h(comm)o (unication)75 1672 y(in)30 b Fi(MPI)p 219 1672 V 14 w(COMM)p 363 1672 V 15 w(W)o(ORLD)15 b Fx(is)h(insulated)g(from)f(comm)o (unication)h(in)31 b Fv(comm)m(slave)p Fx(,)12 b(and)j(vice)i(v)o (ersa.)166 1728 y(In)h(summary)l(,)g(\\group)f(safet)o(y")f(is)i(ac)o (hiev)o(ed)h(via)f(comm)o(unicators)f(b)q(ecause)i(distinct)g(con)o (texts)75 1785 y(within)d(comm)o(unicators)f(are)g(enforced)h(to)e(b)q (e)i(unique)h(on)e(an)o(y)g(pro)q(cess.)75 1906 y Fj(5.5.4)49 b(Example)15 b(#4)75 1992 y Fx(The)g(follo)o(wing)g(example)g(is)g (mean)o(t)f(to)f(illustrate)j(\\safet)o(y")d(b)q(et)o(w)o(een)h(p)q (oin)o(t-to-p)q(oin)o(t)i(and)e(collectiv)o(e)75 2048 y(comm)o(unication.)20 b Fv(MPI)13 b Fx(guaran)o(tees)f(that)h(a)g (single)h(comm)o(unicator)f(can)h(do)f(safe)g(p)q(oin)o(t-to-p)q(oin)o (t)h(and)75 2105 y(collectiv)o(e)j(comm)o(unication.)147 2195 y Fu(#define)22 b(TAG_ARBITRARY)h(12345)147 2252 y(#define)f(SOME_COUNT)166 b(50)147 2365 y(main\(int)22 b(argc,)i(char)f(**argv\))147 2421 y({)194 2478 y(int)h(me;)194 2534 y(MPI_Request)f(request[2];)194 2591 y(MPI_Status)g(status[2];)194 2647 y(MPI_Group)g(MPI_GROUP_WORLD,)f(subgroup;)194 2704 y(int)i(ranks[])f(=)g({2,)h(4,)f(6,)h(8};)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 149 157 149 156 bop 75 -100 a Fm(5.5.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b Fx(149)194 45 y Fu(MPI_Comm)23 b(the_comm;)194 102 y(...)194 158 y(MPI_Init\(&argc,)f(&argv\);)194 214 y(MPI_Comm_group\(MPI_COMM_WORL)o(D,)f(&MPI_GROUP_WORLD\);)194 327 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g(4,)j(ranks,)f(&subgroup\);) f(/*)i(local)f(*/)194 384 y(MPI_Group_rank\(subgroup,)e(&me\);)119 b(/*)23 b(local)g(*/)194 497 y(MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e (subgroup,)i(&the_comm\);)194 610 y(if\(me)g(!=)h(MPI_UNDEFINED\))194 666 y({)290 723 y(MPI_Irecv\(buff1,)d(count,)i(MPI_DOUBLE,)g (MPI_ANY_SOURCE,)f(TAG_ARBITRARY,)719 779 y(the_comm,)h(request\);)290 835 y(MPI_Isend\(buff2,)e(count,)i(MPI_DOUBLE,)g(\(me+1\)\0454,)f (TAG_ARBITRARY,)719 892 y(the_comm,)h(request+1\);)194 948 y(})194 1061 y(for\(i)g(=)h(0;)g(i)g(<)f(SOME_COUNT,)g(i++\))242 1118 y(MPI_Reduce\(...,)f(the_comm\);)194 1174 y(MPI_Waitall\(2,)g (request,)h(status\);)194 1287 y(MPI_Comm_free\(t&he_comm\);)194 1344 y(MPI_Group_free\(&MPI_GROUP_WO)o(RLD\);)194 1400 y(MPI_Group_free\(&subgroup\);)194 1456 y(MPI_Finalize\(\);)147 1513 y(})75 1632 y Fj(5.5.5)49 b(Lib)o(ra)o(ry)16 b(Example)f(#1)75 1718 y Fx(The)g(main)h(program:)147 1800 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 1857 y({)194 1913 y(int)h(done)f(=)h(0;) 194 1970 y(user_lib_t)f(*libh_a,)g(*libh_b;)194 2026 y(void)h(*dataset1,)e(*dataset2;)194 2083 y(...)194 2139 y(MPI_Init\(&argc,)g(&argv\);)194 2195 y(...)194 2252 y(init_user_lib\(MPI_COMM_WORLD)o(,)f(&libh_a\);)194 2308 y(init_user_lib\(MPI_COMM_WORLD)o(,)g(&libh_b\);)194 2365 y(...)194 2421 y(user_start_op\(libh_a,)g(dataset1\);)194 2478 y(user_start_op\(libh_b,)g(dataset2\);)194 2534 y(...)194 2591 y(while\(!done\))194 2647 y({)266 2704 y(/*)i(work)h(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 150 158 150 157 bop 75 -100 a Fx(150)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)266 45 y Fu(...)266 102 y(MPI_Reduce\(...,)22 b(MPI_COMM_WORLD\);)266 158 y(...)266 214 y(/*)h(see)h(if)g(done)f(*/)266 271 y(...)194 327 y(})194 384 y(user_end_op\(libh_a\);)194 440 y(user_end_op\(libh_b\);)194 553 y(uninit_user_lib\(libh_a\);)194 610 y(uninit_user_lib\(libh_b\);)194 666 y(MPI_Finalize\(\);)147 723 y(})75 800 y Fx(The)15 b(user)h(library)g(initialization)i(co)q (de:)147 878 y Fu(void)23 b(init_user_lib\(MPI_Comm)e(*comm,)i (user_lib_t)f(**handle\))147 935 y({)194 991 y(user_lib_t)h(*save;)194 1104 y(user_lib_initsave\(&save\);)e(/*)i(local)h(*/)194 1160 y(MPI_Comm_dup\(comm,)e(&\(save)h(->)g(comm\)\);)194 1273 y(/*)h(other)f(inits)g(*/)194 1330 y(...)194 1443 y(*handle)g(=)h(save;)147 1499 y(})75 1577 y Fx(User)15 b(start-up)g(co)q(de:)147 1655 y Fu(void)23 b (user_start_op\(user_lib_t)d(*handle,)j(void)h(*data\))147 1711 y({)194 1768 y(MPI_Irecv\()f(...,)g(handle->comm,)f(&\(handle)h (->)h(irecv_handle\))e(\);)194 1824 y(MPI_Isend\()h(...,)g (handle->comm,)f(&\(handle)h(->)h(isend_handle\))e(\);)147 1881 y(})75 1958 y Fx(User)15 b(comm)o(unication)h(clean-up)h(co)q(de:) 147 2036 y Fu(void)23 b(user_end_op\(user_lib_t)e(*handle\))147 2093 y({)194 2149 y(MPI_Status)i(*status;)194 2205 y(MPI_Wait\(handle)f (->)i(isend_handle,)e(status\);)194 2262 y(MPI_Wait\(handle)g(->)i (irecv_handle,)e(status\);)147 2318 y(})75 2396 y Fx(User)15 b(ob)s(ject)g(clean-up)i(co)q(de:)147 2474 y Fu(void)23 b(uninit_user_lib\(user_lib_)o(t)e(*handle\))147 2530 y({)194 2587 y(MPI_Comm_free\(&\(handle)g(->)j(comm\)\);)194 2643 y(free\(handle\);)147 2700 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 151 159 151 158 bop 75 -100 a Fm(5.5.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b Fx(151)75 45 y Fj(5.5.6)49 b(Lib)o(ra)o(ry)16 b(Example)f(#2)75 131 y Fx(The)g(main)h(program:)147 225 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 281 y({)194 338 y(int)h(ma,)f(mb;)194 394 y(MPI_Group)g(MPI_GROUP_WORLD,)f (group_a,)g(group_b;)194 451 y(MPI_Comm)h(comm_a,)g(comm_b;)194 563 y(static)g(int)h(list_a[])f(=)g({0,)h(1};)75 620 y(#if)47 b(defined\(EXAMPLE_2B\))22 b(|)h(defined\(EXAMPLE_2C\))194 676 y(static)g(int)h(list_b[])f(=)g({0,)h(2)f(,3};)75 733 y(#else/*)g(EXAMPLE_2A)f(*/)194 789 y(static)h(int)h(list_b[])f(=)g ({0,)h(2};)75 846 y(#endif)194 902 y(int)g(size_list_a)e(=)i (sizeof\(list_a\)/sizeof\(int)o(\);)194 959 y(int)g(size_list_b)e(=)i (sizeof\(list_b\)/sizeof\(int)o(\);)194 1072 y(...)194 1128 y(MPI_Init\(&argc,)e(&argv\);)194 1184 y (MPI_Comm_group\(MPI_COMM_WORL)o(D,)f(&MPI_GROUP_WORLD\);)194 1297 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g(size_list_a,)h(list_a,)h (&group_a\);)194 1354 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)e (size_list_b,)h(list_b,)h(&group_b\);)194 1467 y (MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e(group_a,)i(&comm_a\);)194 1523 y(MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e(group_b,)i(&comm_b\);)194 1636 y(MPI_Comm_rank\(comm_a,)e(&ma\);)194 1693 y (MPI_Comm_rank\(comm_b,)g(&mb\);)194 1805 y(if\(ma)i(!=)h (MPI_UNDEFINED\))266 1862 y(lib_call\(comm_a\);)194 1975 y(if\(mb)f(!=)h(MPI_UNDEFINED\))194 2031 y({)242 2088 y(lib_call\(comm_b\);)242 2144 y(lib_call\(comm_b\);)194 2201 y(})194 2314 y(MPI_Comm_free\(&comm_a\);)194 2370 y(MPI_Comm_free\(&comm_b\);)194 2426 y(MPI_Group_free\(&group_a\);)194 2483 y(MPI_Group_free\(&group_b\);)194 2539 y (MPI_Group_free\(&MPI_GROUP_WO)o(RLD\);)194 2596 y(MPI_Finalize\(\);) 147 2652 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 152 160 152 159 bop 75 -100 a Fx(152)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fx(The)g(library:)147 139 y Fu(void)23 b(lib_call\(MPI_Comm)e (comm\))147 195 y({)194 252 y(int)j(me,)f(done)h(=)f(0;)194 308 y(MPI_Comm_rank\(comm,)f(&me\);)194 365 y(if\(me)h(==)h(0\))266 421 y(while\(!done\))266 478 y({)337 534 y(MPI_Recv\(...,)f (MPI_ANY_SOURCE,)e(MPI_ANY_TAG,)i(comm\);)337 591 y(...)266 647 y(})194 703 y(else)194 760 y({)242 816 y(/*)h(work)f(*/)242 873 y(MPI_Send\(...,)f(0,)i(ARBITRARY_TAG,)e(comm\);)242 929 y(....)194 986 y(})75 1042 y(#ifdef)h(EXAMPLE_2C)194 1099 y(/*)h(include)f(\(resp,)g(exclude\))g(for)g(safety)g(\(resp,)g (no)h(safety\):)f(*/)194 1155 y(MPI_Barrier\(comm\);)75 1212 y(#endif)147 1268 y(})75 1362 y Fx(The)13 b(ab)q(o)o(v)o(e)f (example)h(is)h(really)f(three)g(examples,)g(dep)q(ending)i(on)e (whether)f(or)g(not)h(one)f(includes)j(rank)75 1418 y(3)f(in)h Fv(list)p 222 1418 14 2 v 17 w(b)p Fx(,)g(and)f(whether)h(or)f(not)f(a) i(sync)o(hronize)g(is)g(included)i(in)e Fv(lib)p 1290 1418 V 17 w(call)p Fx(.)k(This)c(example)g(illustrates)75 1475 y(that,)i(despite)h(con)o(texts,)f(subsequen)o(t)h(calls)h(to)d Fv(lib)p 984 1475 V 17 w(call)h Fx(with)h(the)f(same)g(con)o(text)g (need)h(not)f(b)q(e)h(safe)75 1531 y(from)g(one)h(another)g(\(collo)q (quially)l(,)j(\\bac)o(k-masking"\).)30 b(Safet)o(y)18 b(is)i(realized)g(if)g(the)37 b Fv(MPI)p 1684 1531 V 16 w(Ba)o(rrier)17 b Fx(is)75 1588 y(added.)34 b(What)19 b(this)h(demonstrates)f(is)h(that)f(libraries)i(ha)o(v)o(e)f(to)f(b)q (e)h(written)g(carefully)l(,)i(ev)o(en)e(with)75 1644 y(con)o(texts.)j(When)17 b(rank)f(3)g(is)h(excluded,)i(then)d(the)h (sync)o(hronize)g(is)g(not)g(needed)g(to)f(get)g(safet)o(y)g(from)75 1701 y(bac)o(k)f(masking.)166 1757 y(Algorithms)i(lik)o(e)h(\\reduce")e (and)h(\\allreduce")h(ha)o(v)o(e)e(strong)g(enough)h(source)g (selectivit)o(y)h(prop-)75 1813 y(erties)c(so)g(that)g(they)g(are)f (inheren)o(tly)j(ok)m(a)o(y)e(\(no)f(bac)o(kmasking\),)h(pro)o(vided)h (that)e Fv(MPI)h Fx(pro)o(vides)g(basic)75 1870 y(guaran)o(tees.)21 b(So)16 b(are)g(m)o(ultiple)i(calls)f(to)e(a)h(t)o(ypical)h (tree-broadcast)e(algorithm)h(with)g(the)g(same)g(ro)q(ot)75 1926 y(or)h(di\013eren)o(t)h(ro)q(ots)f(\(see)h([28)o(]\).)27 b(Here)18 b(w)o(e)f(rely)h(on)g(t)o(w)o(o)f(guaran)o(tees)f(of)i Fv(MPI)p Fx(:)f(pairwise)h(ordering)g(of)75 1983 y(messages)f(b)q(et)o (w)o(een)h(pro)q(cesses)g(in)g(the)g(same)f(con)o(text,)g(and)h(source) f(selectivit)o(y)i(|)f(deleting)h(either)75 2039 y(feature)c(remo)o(v)o (es)f(the)i(guaran)o(tee)e(that)g(bac)o(kmasking)i(cannot)f(o)q(ccur.) 166 2096 y(Algorithms)d(that)g(try)g(to)f(do)h(non-deterministic)j (broadcasts)c(or)h(other)g(calls)h(that)f(include)j(wild-)75 2152 y(card)g(op)q(erations)g(will)i(not)e(generally)h(ha)o(v)o(e)f (the)g(go)q(o)q(d)g(prop)q(erties)h(of)f(the)g(deterministic)i (implemen-)75 2209 y(tations)g(of)h(\\reduce,")g(\\allreduce,")i(and)e (\\broadcast.")27 b(Suc)o(h)18 b(algorithms)g(w)o(ould)g(ha)o(v)o(e)g (to)f(utilize)75 2265 y(the)e(monotonically)h(increasing)h(tags)d (\(within)i(a)f(comm)o(unicator)g(scop)q(e\))g(to)g(k)o(eep)g(things)h (straigh)o(t.)166 2322 y(All)k(of)e(the)h(foregoing)f(is)h(a)g(supp)q (osition)h(of)e(\\collectiv)o(e)i(calls")f(implemen)o(ted)i(with)e(p)q (oin)o(t-to-)75 2378 y(p)q(oin)o(t)g(op)q(erations.)30 b Fv(MPI)18 b Fx(implemen)o(tations)i(ma)o(y)e(or)g(ma)o(y)g(not)g (implemen)o(t)i(collectiv)o(e)h(calls)e(using)75 2434 y(p)q(oin)o(t-to-p)q(oin)o(t)e(op)q(erations.)22 b(These)16 b(algorithms)g(are)f(used)i(to)e(illustrate)i(the)f(issues)h(of)e (correctness)75 2491 y(and)g(safet)o(y)l(,)g(indep)q(enden)o(t)i(of)e (ho)o(w)g Fv(MPI)f Fx(implemen)o(ts)j(its)e(collectiv)o(e)i(calls.)k (See)16 b(also)f(section)h(5.8.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 153 161 153 160 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fx(153)75 45 y Fn(5.6)59 b(Inter-Communication)75 147 y Fx(This)18 b(section)h(in)o(tro)q(duces)f(the)g(concept)g(of)f(in)o (ter-comm)o(unication)i(and)f(describ)q(es)i(the)d(p)q(ortions)h(of)75 203 y Fv(MPI)j Fx(that)g(supp)q(ort)h(it.)40 b(It)22 b(describ)q(es)h(supp)q(ort)f(for)f(writing)i(programs)d(that)h(con)o (tain)h(user-lev)o(el)75 259 y(serv)o(ers.)166 316 y(All)c(p)q(oin)o (t-to-p)q(oin)o(t)g(comm)o(unication)g(describ)q(ed)h(th)o(us)d(far)h (has)g(in)o(v)o(olv)o(ed)g(comm)o(unication)h(b)q(e-)75 372 y(t)o(w)o(een)c(pro)q(cesses)g(that)f(are)h(mem)o(b)q(ers)g(of)f (the)h(same)g(group.)19 b(This)c(t)o(yp)q(e)f(of)f(comm)o(unication)i (is)f(called)75 429 y(\\in)o(tra-comm)o(unication")j(and)g(the)h(comm)o (unicator)e(used)i(is)f(called)i(an)e(\\in)o(tra-comm)o(unicator,")f (as)75 485 y(w)o(e)f(ha)o(v)o(e)g(noted)g(earlier)h(in)g(the)f(c)o (hapter.)166 542 y(In)c(mo)q(dular)g(and)f(m)o(ulti-discipl)q(inary)j (applications,)g(di\013eren)o(t)d(pro)q(cess)h(groups)f(execute)h (distinct)75 598 y(mo)q(dules)i(and)f(pro)q(cesses)h(within)g (di\013eren)o(t)f(mo)q(dules)h(comm)o(unicate)f(with)h(one)f(another)f (in)i(a)f(pip)q(eline)75 655 y(or)j(a)f(more)h(general)h(mo)q(dule)g (graph.)j(In)d(these)f(applications,)i(the)e(most)f(natural)h(w)o(a)o (y)f(for)h(a)g(pro)q(cess)75 711 y(to)g(sp)q(ecify)i(a)e(target)f(pro)q (cess)i(is)g(b)o(y)g(the)g(rank)f(of)g(the)h(target)e(pro)q(cess)i (within)h(the)e(target)g(group.)20 b(In)75 768 y(applications)13 b(that)e(con)o(tain)h(in)o(ternal)h(user-lev)o(el)g(serv)o(ers,)f(eac)o (h)f(serv)o(er)h(ma)o(y)f(b)q(e)h(a)f(pro)q(cess)h(group)g(that)75 824 y(pro)o(vides)17 b(services)f(to)g(one)g(or)g(more)f(clien)o(ts,)i (and)g(eac)o(h)f(clien)o(t)h(ma)o(y)f(b)q(e)g(a)g(pro)q(cess)h(group)e (that)h(uses)75 880 y(the)h(services)h(of)f(one)g(or)g(more)g(serv)o (ers.)25 b(It)17 b(is)h(again)f(most)f(natural)i(to)e(sp)q(ecify)j(the) e(target)f(pro)q(cess)75 937 y(b)o(y)d(rank)g(within)i(the)e(target)f (group)i(in)g(these)f(applications.)21 b(This)14 b(t)o(yp)q(e)g(of)e (comm)o(unication)j(is)e(called)75 993 y(\\in)o(ter-comm)o(unication") 18 b(and)g(the)g(comm)o(unicator)f(used)h(is)g(called)h(an)f(\\in)o (ter-comm)o(unicator,")f(as)75 1050 y(in)o(tro)q(duced)f(earlier.)166 1106 y(An)d(in)o(ter-comm)o(unication)g(is)g(a)f(p)q(oin)o(t-to-p)q (oin)o(t)h(comm)o(unication)g(b)q(et)o(w)o(een)g(pro)q(cesses)f(in)h (di\013er-)75 1163 y(en)o(t)f(groups.)19 b(The)12 b(group)g(con)o (taining)h(a)f(pro)q(cess)h(that)e(initiates)j(an)e(in)o(ter-comm)o (unication)i(op)q(eration)75 1219 y(is)i(called)i(the)d(\\lo)q(cal)i (group,")e(that)g(is,)h(the)g(sender)g(in)h(a)e(send)i(and)f(the)g (receiv)o(er)g(in)h(a)e(receiv)o(e.)23 b(The)75 1276 y(group)16 b(con)o(taining)g(the)g(target)f(pro)q(cess)h(is)g(called)i (the)e(\\remote)f(group,")g(that)g(is,)h(the)g(receiv)o(er)h(in)f(a)75 1332 y(send)f(and)f(the)g(sender)h(in)g(a)f(receiv)o(e.)20 b(As)14 b(in)h(in)o(tra-comm)o(unication,)g(the)f(target)f(pro)q(cess)h (is)h(sp)q(eci\014ed)75 1389 y(using)k(a)g Fv(\(comm)m(unicato)o(r,)d (rank\))j Fx(pair.)30 b(Unlik)o(e)21 b(in)o(tra-comm)o(unication,)e (the)g(rank)g(is)g(relativ)o(e)g(to)f(a)75 1445 y(second,)d(remote)g (group.)166 1501 y(All)e(in)o(ter-comm)o(unicator)f(constructors)g(are) f(blo)q(c)o(king)j(and)e(require)h(that)e(the)h(lo)q(cal)h(and)g (remote)75 1558 y(groups)i(b)q(e)h(disjoin)o(t)g(in)g(order)f(to)f(a)o (v)o(oid)h(deadlo)q(c)o(k.)166 1614 y(Here)f(is)h(a)f(summary)g(of)g (the)g(prop)q(erties)h(of)f(in)o(ter-comm)o(unication)i(and)e(in)o (ter-comm)o(unicators:)143 1702 y Ft(\017)23 b Fx(The)c(syn)o(tax)g(of) g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unication)g(is)g(the)f(same)g (for)g(b)q(oth)h(in)o(ter-)f(and)h(in)o(tra-)189 1758 y(comm)o(unication.)26 b(The)17 b(same)g(comm)o(unicator)g(can)g(b)q(e) g(used)h(b)q(oth)f(for)g(send)g(and)h(for)e(receiv)o(e)189 1815 y(op)q(erations.)143 1906 y Ft(\017)23 b Fx(A)14 b(target)g(pro)q(cess)h(is)g(addressed)h(b)o(y)f(its)g(rank)f(in)i(the) f(remote)f(group,)g(b)q(oth)h(for)f(sends)i(and)f(for)189 1963 y(receiv)o(es.)143 2054 y Ft(\017)23 b Fx(Comm)o(unications)12 b(using)i(an)e(in)o(ter-comm)o(unicator)h(are)f(guaran)o(teed)g(not)g (to)g(con\015ict)h(with)g(an)o(y)189 2110 y(comm)o(unications)i(that)g (use)h(a)e(di\013eren)o(t)i(comm)o(unicator.)143 2202 y Ft(\017)23 b Fx(An)15 b(in)o(ter-comm)o(unicator)g(cannot)g(b)q(e)h (used)g(for)f(collectiv)o(e)i(comm)o(unication.)143 2293 y Ft(\017)23 b Fx(A)15 b(comm)o(unicator)g(will)i(pro)o(vide)e(either)h (in)o(tra-)f(or)g(in)o(ter-comm)o(unication,)h(nev)o(er)f(b)q(oth.)75 2380 y(The)i(routine)f Fv(MPI)p 412 2380 14 2 v 16 w(COMM)p 570 2380 V 17 w(TEST)p 701 2380 V 16 w(INTER)h Fx(ma)o(y)e(b)q(e)i (used)g(to)f(determine)h(if)g(a)f(comm)o(unicator)g(is)h(an)75 2437 y(in)o(ter-)12 b(or)f(in)o(tra-comm)o(unicator.)19 b(In)o(ter-comm)o(unicators)11 b(can)h(b)q(e)h(used)f(as)f(argumen)o (ts)g(to)g(some)h(of)f(the)75 2493 y(other)i(comm)o(unicator)h(access)g (routines.)20 b(In)o(ter-comm)o(unicators)13 b(cannot)h(b)q(e)g(used)h (as)e(input)i(to)e(some)75 2550 y(of)i(the)g(constructor)f(routines)i (for)f(in)o(tra-comm)o(unicators)f(\(for)h(instance,)g Fv(MPI)p 1476 2550 V 16 w(COMM)p 1634 2550 V 17 w(CREA)l(TE)p Fx(\).)189 2647 y Fl(A)n(dvic)n(e)g(to)i(implementors.)41 b Fx(F)l(or)15 b(the)g(purp)q(ose)h(of)f(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication,)g(comm)o(u-)189 2704 y(nicators)f(can)g(b)q(e)h (represen)o(ted)g(in)g(eac)o(h)f(pro)q(cess)g(b)o(y)g(a)g(tuple)i (consisting)f(of:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 154 162 154 161 bop 75 -100 a Fx(154)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)189 45 y Fq(group)189 118 y(send)p 295 118 16 2 v 18 w(con)o(text)189 191 y(receiv)o(e)p 350 191 V 18 w(con)o(text)189 264 y(source)189 358 y Fx(F)l(or)9 b(in)o(ter-comm)o(unicators,)i Fq(group)g Fx(describ)q(es)h(the)e(remote)g(group,)h(and)f Fq(source)g Fx(is)h(the)f(rank)g(of)189 415 y(the)k(pro)q(cess)h(in)g (the)g(lo)q(cal)g(group.)20 b(F)l(or)14 b(in)o(tra-comm)o(unicators,)f Fq(group)i Fx(is)g(the)f(comm)o(unicator)189 471 y(group)20 b(\(remote=lo)q(cal\),)j Fq(source)d Fx(is)i(the)f(rank)g(of)f(the)i (pro)q(cess)f(in)h(this)f(group,)h(and)f Fq(send)189 527 y(con)o(text)e Fx(and)g Fq(receiv)o(e)h(con)o(text)f Fx(are)g(iden)o(tical.)32 b(A)19 b(group)g(is)g(represen)o(ted)g(b)o(y) g(a)g(rank-to-)189 584 y(absolute-address)c(translation)h(table.)189 659 y(The)11 b(in)o(ter-comm)o(unicator)h(cannot)g(b)q(e)g(discussed)h (sensibly)g(without)f(considering)h(pro)q(cesses)f(in)189 716 y(b)q(oth)f(the)h(lo)q(cal)g(and)g(remote)e(groups.)19 b(Imagine)12 b(a)f(pro)q(cess)g Fq(P)g Fx(in)i(group)e Ft(P)t Fx(,)g(whic)o(h)h(has)f(an)h(in)o(ter-)189 772 y(comm)o(unicator)18 b Fq(C)527 779 y Fh(P)556 772 y Fx(,)i(and)f(a)f(pro)q(cess)h Fq(Q)g Fx(in)h(group)e Ft(Q)p Fx(,)i(whic)o(h)f(has)g(an)g(in)o(ter-comm)o(unicator)189 828 y Fq(C)227 835 y Fh(Q)257 828 y Fx(.)h(Then)243 922 y Ft(\017)j Fq(C)327 929 y Fh(P)356 922 y Fq(.group)14 b Fx(describ)q(es)j(the)f(group)e Ft(Q)i Fx(and)f Fq(C)1097 929 y Fh(Q)1128 922 y Fq(.group)f Fx(describ)q(es)j(the)e(group)g Ft(P)t Fx(.)243 995 y Ft(\017)23 b Fq(C)327 1002 y Fh(P)356 995 y Fq(.send)p 477 995 V 18 w(con)o(text)17 b(=)h Fx(C)770 1002 y Fh(Q)800 995 y Fq(.receiv)o(e)p 976 995 V 18 w(con)o(text)d Fx(and)h(the)f(con)o(text)g(is)g(unique)i(in)f Ft(Q)p Fx(;)289 1052 y Fq(C)327 1059 y Fh(P)356 1052 y Fq(.receiv)o(e)p 532 1052 V 18 w(con)o(text)h(=)35 b(C)847 1059 y Fh(Q)877 1052 y Fq(.send)p 998 1052 V 18 w(con)o(text)16 b Fx(and)f(this)h(con)o (text)e(is)i(unique)h(in)f Ft(P)t Fx(.)243 1125 y Ft(\017)23 b Fq(C)327 1132 y Fh(P)356 1125 y Fq(.source)14 b Fx(is)i(rank)f(of)f Fq(P)h Fx(in)h Ft(P)j Fx(and)c Fq(C)1013 1132 y Fh(Q)1044 1125 y Fq(.source)f Fx(is)i(rank)f(of)g Fq(Q)g Fx(in)h Ft(Q)p Fx(.)189 1219 y(Assume)j(that)f Fq(P)g Fx(sends)i(a)e(message)g (to)h Fq(Q)g Fx(using)g(the)g(in)o(ter-comm)o(unicator.)31 b(Then)19 b Fq(P)g Fx(uses)189 1275 y(the)g Fq(group)f Fx(table)i(to)e(\014nd)h(the)g(absolute)g(address)g(of)g Fq(Q)p Fx(;)f Fq(source)h Fx(and)g Fq(send)p 1611 1275 V 18 w(con)o(text)g Fx(are)189 1332 y(app)q(ended)d(to)f(the)g (message.)189 1407 y(Assume)22 b(that)f Fq(Q)h Fx(p)q(osts)f(a)h (receiv)o(e)g(with)h(an)e(explicit)j(source)e(argumen)o(t)f(using)i (the)f(in)o(ter-)189 1463 y(comm)o(unicator.)c(Then)c Fq(Q)f Fx(matc)o(hes)g Fq(receiv)o(e)p 1006 1463 V 18 w(con)o(text)g Fx(to)f(the)i(message)e(con)o(text)h(and)g(source)189 1520 y(argumen)o(t)h(to)h(the)g(message)g(source.)189 1595 y(The)g(same)g(algorithm)g(is)h(appropriate)f(for)g(in)o(tra-comm) o(unicators)f(as)h(w)o(ell.)189 1670 y(In)g(order)g(to)f(supp)q(ort)h (in)o(ter-comm)o(unicator)g(accessors)g(and)g(constructors,)f(it)h(is)g (necessary)h(to)189 1726 y(supplemen)o(t)j(this)f(mo)q(del)h(with)f (additional)i(structures,)e(that)f(store)g(information)h(ab)q(out)g (the)189 1783 y(lo)q(cal)i(comm)o(unication)f(group,)h(and)f (additional)h(safe)f(con)o(texts.)30 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to)g(imple-)189 1839 y(mentors.)p Fx(\))75 1961 y Fj(5.6.1)49 b(Inter-comm)n(unicato)o(r)13 b(Accesso)o(rs)75 2141 y Fv(MPI)p 160 2141 14 2 v 16 w(COMM)p 318 2141 V 16 w(TEST)p 448 2141 V 17 w(INTER\(comm)m(,)f (\015ag\))117 2218 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))117 2294 y(OUT)108 b Fv(\015ag)518 b Fk(\(logical\))75 2418 y Fu(int)23 b(MPI)p 245 2418 15 2 v 17 w(Comm)p 358 2418 V 17 w(test)p 471 2418 V 16 w(inter\(MPI)p 703 2418 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 2504 y(MPI)p 150 2504 V 17 w(COMM)p 263 2504 V 16 w(TEST)p 375 2504 V 17 w(INTER\(COMM,)f(FLAG,)i(IERROR\))170 2561 y(INTEGER)f(COMM,)h (IERROR)170 2617 y(LOGICAL)f(FLAG)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 155 163 155 162 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fx(155)75 45 y(This)20 b(lo)q(cal)g(routine)g(allo)o(ws)f(the)g (calling)i(pro)q(cess)f(to)e(determine)i(if)g(a)f(comm)o(unicator)g(is) g(an)h(in)o(ter-)75 102 y(comm)o(unicator)g(or)h(an)f(in)o(tra-comm)o (unicator.)36 b(It)21 b(returns)g Fi(true)g Fx(if)g(it)g(is)h(an)e(in)o (ter-comm)o(unicator,)75 158 y(otherwise)15 b Fi(false)p Fx(.)166 214 y(When)j(an)f(in)o(ter-comm)o(unicator)h(is)g(used)g(as)f (an)g(input)i(argumen)o(t)d(to)h(the)h(comm)o(unicator)f(ac-)75 271 y(cessors)d(describ)q(ed)i(ab)q(o)o(v)o(e)e(under)h(in)o(tra-comm)o (unication,)f(the)h(follo)o(wing)f(table)h(describ)q(es)h(b)q(eha)o (vior.)p 270 365 1410 2 v 269 422 2 57 v 652 405 a Fv(MPI)p 737 405 14 2 v 16 w(COMM)p 895 405 V 17 w(*)e Fx(F)l(unction)j(Beha)o (vior)p 1679 422 2 57 v 269 478 V 655 461 a(\(in)f(In)o(ter-Comm)o (unication)f(Mo)q(de\))p 1679 478 V 270 480 1410 2 v 270 490 V 269 546 2 57 v 295 529 a Fv(MPI)p 380 529 14 2 v 16 w(COMM)p 538 529 V 16 w(SIZE)p 729 546 2 57 v 111 w Fx(returns)g(the)g(size)h(of)f(the)g(lo)q(cal)i(group.)p 1679 546 V 269 603 V 295 586 a Fv(MPI)p 380 586 14 2 v 16 w(COMM)p 538 586 V 16 w(GROUP)p 729 603 2 57 v 52 w Fx(returns)e(the)g(lo)q(cal)h(group.)p 1679 603 V 269 659 V 295 642 a Fv(MPI)p 380 642 14 2 v 16 w(COMM)p 538 642 V 16 w(RANK)p 729 659 2 57 v 81 w Fx(returns)f(the)g(rank)g(in)h (the)f(lo)q(cal)i(group)p 1679 659 V 270 661 1410 2 v 75 746 a(F)l(urthermore,)d(the)g(op)q(eration)g Fv(MPI)p 715 746 14 2 v 16 w(COMM)p 873 746 V 17 w(COMP)l(ARE)h Fx(is)g(v)m(alid)h(for)e(in)o(ter-comm)o(unicators.)19 b(Both)75 803 y(comm)o(unicators)13 b(m)o(ust)h(b)q(e)g(either)g(in)o (tra-)g(or)f(in)o(ter-comm)o(unicators,)h(or)f(else)i Fi(MPI)p 1515 803 13 2 v 14 w(UNEQUAL)e Fx(results.)75 859 y(Both)22 b(corresp)q(onding)g(lo)q(cal)h(and)f(remote)f(groups)h (m)o(ust)f(compare)g(correctly)h(to)f(get)h(the)g(results)75 916 y Fi(MPI)p 152 916 V 14 w(CONGRUENT)c Fx(and)g Fi(MPI)p 603 916 V 15 w(SIMILAR)p Fx(.)f(In)i(particular,)h(it)e(is)h(p)q (ossible)i(for)d Fi(MPI)p 1515 916 V 14 w(SIMILAR)g Fx(to)g(result)75 972 y(b)q(ecause)e(either)g(the)f(lo)q(cal)i(or)d(remote)h(groups)g(w)o (ere)g(similar)h(but)f(not)g(iden)o(tical.)166 1029 y(The)23 b(follo)o(wing)g(accessors)f(pro)o(vide)h(consisten)o(t)f(access)h(to)f (the)g(remote)g(group)g(of)g(an)h(in)o(ter-)75 1085 y(comm)o(unicator:) 166 1142 y(The)15 b(follo)o(wing)h(are)f(all)h(lo)q(cal)h(op)q (erations.)75 1293 y Fv(MPI)p 160 1293 14 2 v 16 w(COMM)p 318 1293 V 16 w(REMOTE)p 521 1293 V 17 w(SIZE\(comm)m(,)12 b(size\))117 1370 y Fk(IN)155 b Fv(comm)466 b Fk(in)o(ter-comm)o (unicator)11 b(\(handle\))117 1445 y(OUT)108 b Fv(size)519 b Fk(n)o(um)o(b)q(er)19 b(of)g(pro)q(cesses)j(in)d(the)h(remote)f (group)g(of)38 b Fi(comm)905 1501 y Fk(\(in)o(teger\))75 1626 y Fu(int)23 b(MPI)p 245 1626 15 2 v 17 w(Comm)p 358 1626 V 17 w(remote)p 519 1626 V 16 w(size\(MPI)p 727 1626 V 16 w(Comm)h(comm,)f(int)g(*size\))75 1712 y(MPI)p 150 1712 V 17 w(COMM)p 263 1712 V 16 w(REMOTE)p 423 1712 V 17 w(SIZE\(COMM,)f(SIZE,)h(IERROR\))170 1769 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 1950 y Fv(MPI)p 160 1950 14 2 v 16 w(COMM)p 318 1950 V 16 w(REMOTE)p 521 1950 V 17 w(GROUP\(comm)o(,)11 b(group\))117 2027 y Fk(IN)155 b Fv(comm)466 b Fk(in)o(ter-comm)o(unicator)11 b(\(handle\))117 2102 y(OUT)108 b Fv(group)479 b Fk(remote)14 b(group)f(corresp)q(onding)i(to)f Fi(comm)8 b Fk(\(handle\))75 2227 y Fu(int)23 b(MPI)p 245 2227 15 2 v 17 w(Comm)p 358 2227 V 17 w(remote)p 519 2227 V 16 w(group\(MPI)p 751 2227 V 16 w(Comm)g(comm,)h(MPI)p 1102 2227 V 16 w(Group)g(*group\)) 75 2313 y(MPI)p 150 2313 V 17 w(COMM)p 263 2313 V 16 w(REMOTE)p 423 2313 V 17 w(GROUP\(COMM,)e(GROUP,)h(IERROR\))170 2369 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)189 2506 y Fl(R)n(ationale.)83 b Fx(Symmetric)23 b(access)g(to)f(b)q(oth)g(the)h(lo)q(cal)g(and)g (remote)f(groups)g(of)g(an)h(in)o(ter-)189 2562 y(comm)o(unicator)17 b(is)h(imp)q(ortan)o(t,)g(so)f(this)h(function,)h(as)f(w)o(ell)g(as)g Fv(MPI)p 1407 2562 14 2 v 16 w(COMM)p 1565 2562 V 16 w(REMOTE)p 1768 2562 V 17 w(SIZE)189 2619 y Fx(ha)o(v)o(e)c(b)q(een)j (pro)o(vided.)j(\()p Fl(End)c(of)g(r)n(ationale.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 156 164 156 163 bop 75 -100 a Fx(156)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fj(5.6.2)49 b(Inter-comm)n(unicato)o(r)13 b(Op)q(erations)75 131 y Fx(This)87 b(section)g(in)o(tro)q(duces)g(four)e(blo)q(c)o(king)j (in)o(ter-comm)o(unicator)e(op)q(erations.)75 187 y Fv(MPI)p 160 187 14 2 v 16 w(INTERCOMM)p 450 187 V 16 w(CREA)l(TE)12 b Fx(is)e(used)h(to)f(bind)h(t)o(w)o(o)e(in)o(tra-comm)o(unicators)h (in)o(to)g(an)g(in)o(ter-comm)o(unicator;)75 244 y(the)23 b(function)h Fv(MPI)p 432 244 V 16 w(INTERCOMM)p 722 244 V 17 w(MERGE)g Fx(creates)e(an)i(in)o(tra-comm)o(unicator)e(b)o(y)i (merging)f(the)75 300 y(lo)q(cal)f(and)e(remote)g(groups)g(of)g(an)g (in)o(ter-comm)o(unicator.)36 b(The)20 b(functions)p Fv(MPI)p 1516 300 V 17 w(COMM)p 1675 300 V 16 w(DUP)h Fx(and)75 357 y Fv(MPI)p 160 357 V 16 w(COMM)p 318 357 V 16 w(FREE)p Fx(,)g(in)o(tro)q(duced)h(previously)l(,)h(duplicate)g (and)e(free)f(an)h(in)o(ter-comm)o(unicator,)h(re-)75 413 y(sp)q(ectiv)o(ely)l(.)166 470 y(Ov)o(erlap)h(of)f(lo)q(cal)h(and)g (remote)f(groups)g(that)g(are)g(b)q(ound)h(in)o(to)g(an)f(in)o (ter-comm)o(unicator)g(is)75 526 y(prohibited.)h(If)15 b(there)h(is)g(o)o(v)o(erlap,)f(then)h(the)f(program)g(is)h(erroneous)f (and)h(is)g(lik)o(ely)h(to)e(deadlo)q(c)o(k.)22 b(\(If)75 583 y(a)16 b(pro)q(cess)g(is)g(m)o(ultithreaded,)h(and)f Fv(MPI)f Fx(calls)i(blo)q(c)o(k)g(only)f(a)g(thread,)f(rather)g(than)h (a)g(pro)q(cess,)g(then)75 639 y(\\dual)f(mem)o(b)q(ership")g(can)f(b)q (e)g(supp)q(orted.)20 b(It)14 b(is)h(then)f(the)g(user's)g(resp)q (onsibilit)o(y)j(to)c(mak)o(e)h(sure)g(that)75 695 y(calls)i(on)f(b)q (ehalf)i(of)e(the)g(t)o(w)o(o)f(\\roles")h(of)f(a)h(pro)q(cess)h(are)f (executed)h(b)o(y)f(t)o(w)o(o)f(indep)q(enden)o(t)j(threads.\))166 752 y(The)12 b(function)h Fv(MPI)p 516 752 V 16 w(INTERCOMM)p 806 752 V 17 w(CREA)l(TE)g Fx(can)g(b)q(e)f(used)h(to)f(create)g(an)g (in)o(ter-comm)o(unicator)75 808 y(from)19 b(t)o(w)o(o)g(existing)i(in) o(tra-comm)o(unicators,)g(in)g(the)f(follo)o(wing)h(situation:)30 b(A)o(t)20 b(least)g(one)h(selected)75 865 y(mem)o(b)q(er)f(from)g(eac) o(h)g(group)g(\(the)g(\\group)g(leader"\))g(has)g(the)h(abilit)o(y)g (to)f(comm)o(unicate)g(with)h(the)75 921 y(selected)d(mem)o(b)q(er)f (from)g(the)g(other)f(group;)h(that)g(is,)g(a)g(\\p)q(eer")g(comm)o (unicator)f(exists)i(where)f(b)q(oth)75 978 y(leaders)12 b(b)q(elong,)g(and)g(eac)o(h)f(leader)h(kno)o(ws)e(the)i(rank)e(of)h (the)g(other)g(leader)h(in)g(this)g(p)q(eer)f(comm)o(unicator)75 1034 y(\(the)18 b(t)o(w)o(o)e(leaders)j(could)g(b)q(e)g(the)f(same)f (pro)q(cess\).)28 b(F)l(urthermore,)18 b(mem)o(b)q(ers)g(of)g(eac)o(h)g (group)f(kno)o(w)75 1091 y(the)e(rank)g(of)g(their)h(leader.)166 1147 y(Construction)10 b(of)h(an)f(in)o(ter-comm)o(unicator)h(from)f(t) o(w)o(o)f(in)o(tra-comm)o(unicators)h(requires)h(separate)75 1204 y(collectiv)o(e)16 b(op)q(erations)e(in)h(the)f(lo)q(cal)h(group)e (and)i(in)f(the)g(remote)g(group,)f(as)h(w)o(ell)h(as)f(a)f(p)q(oin)o (t-to-p)q(oin)o(t)75 1260 y(comm)o(unication)j(b)q(et)o(w)o(een)f(a)g (pro)q(cess)h(in)g(the)f(lo)q(cal)h(group)f(and)h(a)f(pro)q(cess)g(in)h (the)f(remote)g(group.)166 1316 y(In)g(standard)g Fv(MPI)f Fx(implemen)o(tations)i(\(with)f(static)g(pro)q(cess)g(allo)q(cation)h (at)f(initialization\),)i(the)75 1373 y Fv(MPI)p 160 1373 V 16 w(COMM)p 318 1373 V 16 w(W)o(ORLD)j Fx(comm)o(unicator)f (\(or)f(preferably)i(a)f(dedicated)i(duplicate)g(thereof)t(\))e(can)g (b)q(e)75 1429 y(this)e(p)q(eer)g(comm)o(unicator.)24 b(In)18 b(dynamic)f Fv(MPI)f Fx(implemen)o(tations,)i(where,)f(for)f (example,)h(a)g(pro)q(cess)75 1486 y(ma)o(y)c(spa)o(wn)g(new)h(c)o (hildren)h(pro)q(cesses)f(during)h(an)e Fv(MPI)g Fx(execution,)i(the)f (paren)o(t)f(pro)q(cess)h(ma)o(y)f(b)q(e)h(the)75 1542 y(\\bridge")i(b)q(et)o(w)o(een)g(the)g(old)g(comm)o(unication)h(univ)o (erse)g(and)f(the)f(new)h(comm)o(unication)h(w)o(orld)f(that)75 1599 y(includes)h(the)f(paren)o(t)f(and)g(its)g(c)o(hildren.)166 1655 y(The)25 b(application)h(top)q(ology)e(functions)h(describ)q(ed)h (in)g(c)o(hapter)e(6)g(do)g(not)g(apply)i(to)e(in)o(ter-)75 1712 y(comm)o(unicators.)18 b(Users)11 b(that)g(require)h(this)g (capabilit)o(y)h(should)f(utilize)i Fv(MPI)p 1418 1712 V 16 w(INTERCOMM)p 1708 1712 V 16 w(MERGE)75 1768 y Fx(to)g(build)j(an) e(in)o(tra-comm)o(unicator,)f(then)i(apply)f(the)g(graph)g(or)g (cartesian)g(top)q(ology)f(capabilities)k(to)75 1825 y(that)i(in)o(tra-comm)o(unicator,)h(creating)f(an)h(appropriate)f(top) q(ology-orien)o(ted)h(in)o(tra-comm)o(unicator.)75 1881 y(Alternativ)o(ely)l(,)f(it)e(ma)o(y)f(b)q(e)h(reasonable)g(to)g (devise)h(one's)e(o)o(wn)g(application)j(top)q(ology)d(mec)o(hanisms)75 1937 y(for)e(this)g(case,)g(without)g(loss)h(of)e(generalit)o(y)l(.) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 157 165 157 164 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fx(157)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(INTERCOMM)p 450 45 V 16 w(CREA)l(TE\(lo)q(cal)p 741 45 V 17 w(comm)n(,)44 b(lo)q(cal)p 1022 45 V 16 w(leader,)k(p)q(eer)p 1291 45 V 17 w(comm)m(,)d(remote)p 1618 45 V 14 w(leader,)i(tag,)75 102 y(newintercomm)n(\))117 179 y Fk(IN)155 b Fv(lo)q(cal)p 409 179 V 17 w(comm)361 b Fk(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e (\(handle\))117 254 y(IN)155 b Fv(lo)q(cal)p 409 254 V 17 w(leader)370 b Fk(rank)14 b(of)f(lo)q(cal)g(group)h(leader)g(in)g Fi(lo)q(cal)p 1517 254 13 2 v 15 w(comm)8 b Fk(\(in)o(teger\))117 329 y(IN)155 b Fv(p)q(eer)p 401 329 14 2 v 18 w(comm)368 b Fk(\\p)q(eer")15 b(in)o(tra-comm)n(unicator;)c(signi\014can)o(t)i (only)g(at)g(the)h Fi(lo-)905 385 y(cal)p 955 385 13 2 v 15 w(leader)i Fk(\(handle\))117 461 y(IN)155 b Fv(remote)p 455 461 14 2 v 14 w(leader)327 b Fk(rank)12 b(of)f(remote)g(group)h (leader)g(in)23 b Fi(p)q(eer)p 1548 461 13 2 v 16 w(comm)n Fk(;)8 b(signi\014can)o(t)905 517 y(only)13 b(at)h(the)h Fi(lo)q(cal)p 1201 517 V 15 w(leader)g Fk(\(in)o(teger\))117 592 y(IN)155 b Fv(tag)526 b Fk(\\safe")14 b(tag)f(\(in)o(teger\))117 667 y(OUT)108 b Fv(newintercomm)306 b Fk(new)15 b(in)o(ter-comm)o (unicator)c(\(handle\))75 792 y Fu(int)23 b(MPI)p 245 792 15 2 v 17 w(Intercomm)p 478 792 V 16 w(create\(MPI)p 734 792 V 16 w(Comm)g(local)p 989 792 V 17 w(comm,)g(int)g(local)p 1364 792 V 17 w(leader,)393 848 y(MPI)p 468 848 V 17 w(Comm)g(peer)p 700 848 V 17 w(comm,)g(int)h(remote)p 1100 848 V 16 w(leader,)f(int)g(tag,)393 905 y(MPI)p 468 905 V 17 w(Comm)g(*newintercomm\))75 991 y(MPI)p 150 991 V 17 w(INTERCOMM)p 383 991 V 16 w(CREATE\(LOCAL)p 687 991 V 15 w(COMM,)g(LOCAL)p 965 991 V 17 w(LEADER,)g(PEER)p 1269 991 V 16 w(COMM,)h(REMOTE)p 1573 991 V 16 w(LEADER,)f(TAG,)393 1048 y(NEWINTERCOMM,)f(IERROR\))170 1104 y(INTEGER)h(LOCAL)p 484 1104 V 17 w(COMM,)g(LOCAL)p 764 1104 V 17 w(LEADER,)f(PEER)p 1067 1104 V 17 w(COMM,)h(REMOTE)p 1371 1104 V 17 w(LEADER,)g(TAG,)170 1161 y(NEWINTERCOMM,)g(IERROR)75 1247 y Fx(This)d(call)h(creates)e(an)h (in)o(ter-comm)o(unicator.)33 b(It)20 b(is)g(collectiv)o(e)i(o)o(v)o (er)d(the)g(union)i(of)e(the)h(lo)q(cal)h(and)75 1303 y(remote)d(groups.)28 b(Pro)q(cesses)18 b(should)h(pro)o(vide)g(iden)o (tical)h Fv(lo)q(cal)p 1197 1303 14 2 v 16 w(comm)12 b Fx(and)18 b Fv(lo)q(cal)p 1525 1303 V 17 w(leader)g Fx(argumen)o(ts)75 1360 y(within)e(eac)o(h)g(group.)j(Wildcards)e(are)d (not)h(p)q(ermitted)h(for)f Fv(remo)o(te)p 1255 1360 V 14 w(leader,)g(lo)q(cal)p 1497 1360 V 16 w(leader)p Fx(,)g(and)h Fv(tag)p Fx(.)166 1416 y(This)g(call)g(uses)f(p)q(oin)o (t-to-p)q(oin)o(t)h(comm)o(unication)g(with)f(comm)o(unicator)g Fv(p)q(eer)p 1542 1416 V 17 w(comm)m Fx(,)c(and)16 b(with)75 1473 y(tag)e Fv(tag)g Fx(b)q(et)o(w)o(een)h(the)g(leaders.)20 b(Th)o(us,)15 b(care)f(m)o(ust)g(b)q(e)h(tak)o(en)f(that)g(there)h(b)q (e)g(no)g(p)q(ending)h(comm)o(uni-)75 1529 y(cation)f(on)g Fv(p)q(eer)p 357 1529 V 18 w(comm)9 b Fx(that)14 b(could)j(in)o (terfere)e(with)h(this)f(comm)o(unication.)189 1636 y Fl(A)n(dvic)n(e)j(to)i(users.)60 b Fx(W)l(e)19 b(recommend)g(using)g(a) f(dedicated)i(p)q(eer)f(comm)o(unicator,)g(suc)o(h)g(as)f(a)189 1692 y(duplicate)e(of)e(MPI)p 528 1692 V 16 w(COMM)p 696 1692 V 16 w(W)o(ORLD,)h(to)e(a)o(v)o(oid)i(trouble)g(with)f(p)q (eer)h(comm)o(unicators.)20 b(\()p Fl(End)189 1748 y(of)c(advic)n(e)g (to)h(users.)p Fx(\))75 1949 y Fv(MPI)p 160 1949 V 16 w(INTERCOMM)p 450 1949 V 16 w(MERGE\(intercomm)o(,)11 b(high,)16 b(newintracomm)n(\))117 2026 y Fk(IN)155 b Fv(intercomm)380 b Fk(In)o(ter-Comm)o(unicator)11 b(\(handle\))117 2102 y(IN)155 b Fv(high)507 b Fk(\(logical\))117 2177 y(OUT)108 b Fv(newintracomm)304 b Fk(new)15 b(in)o(tra-comm)n(unicator) c(\(handle\))75 2301 y Fu(int)23 b(MPI)p 245 2301 15 2 v 17 w(Intercomm)p 478 2301 V 16 w(merge\(MPI)p 710 2301 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 2358 y(MPI)p 468 2358 V 17 w(Comm)g(*newintracomm\))75 2444 y(MPI)p 150 2444 V 17 w(INTERCOMM)p 383 2444 V 16 w(MERGE\(INTERCOMM,)e (HIGH,)j(INTRACOMM,)e(IERROR\))170 2501 y(INTEGER)h(INTERCOMM,)g (INTRACOMM,)f(IERROR)170 2557 y(LOGICAL)h(HIGH)75 2643 y Fx(This)e(function)f(creates)g(an)g(in)o(tra-comm)o(unicator)g(from)f (the)h(union)h(of)f(the)g(t)o(w)o(o)e(groups)i(that)f(are)75 2700 y(asso)q(ciated)f(with)g Fv(intercomm)n Fx(.)24 b(All)19 b(pro)q(cesses)f(should)h(pro)o(vide)f(the)g(same)f Fv(high)h Fx(v)m(alue)h(within)g(eac)o(h)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 158 166 158 165 bop 75 -100 a Fx(158)123 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)383 4 y 18945146 4357381 0 0 24865505 5920358 startTexFig 383 4 a %%BeginDocument: context-fig-1.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def /col8 {.68 .85 .9 setrgbcolor} bind def /col9 {0 .39 0 setrgbcolor} bind def /col10 {.65 .17 .17 setrgbcolor} bind def /col11 {1 .51 0 setrgbcolor} bind def /col12 {.63 .13 .94 setrgbcolor} bind def /col13 {1 .75 .8 setrgbcolor} bind def /col14 {.7 .13 .13 setrgbcolor} bind def /col15 {1 .84 0 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -90.0 198.0 translate 0.900 -0.900 scale 0.500 setlinewidth n 199 219 m 199 119 l 99 119 l 99 219 l clp gs col-1 s gr n 359 219 m 359 119 l 259 119 l 259 219 l clp gs col-1 s gr n 519 219 m 519 119 l 419 119 l 419 219 l clp gs col-1 s gr 1.000 setlinewidth n 220.000 173.000 m 204.000 169.000 l 220.000 165.000 l gs 2 setlinejoin col-1 s gr n 204 169 m 254 169 l gs col-1 s gr n 238.000 165.000 m 254.000 169.000 l 238.000 173.000 l gs 2 setlinejoin col-1 s gr n 380.000 173.000 m 364.000 169.000 l 380.000 165.000 l gs 2 setlinejoin col-1 s gr n 364 169 m 414 169 l gs col-1 s gr n 398.000 165.000 m 414.000 169.000 l 398.000 173.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 279 174 m gs 1 -1 scale (Group 1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 439 174 m gs 1 -1 scale (Group 2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 174 m gs 1 -1 scale (Group 0) col-1 show gr $F2psEnd %%EndDocument endTexFig 644 378 a Fx(Figure)g(5.1:)k(Three-group)c(pip)q(eline)q(.)75 509 y(of)h(the)g(t)o(w)o(o)e(groups.)22 b(If)16 b(pro)q(cesses)h(in)g (one)f(group)f(pro)o(vided)i(the)f(v)m(alue)h Fv(high)g(=)f(false)g Fx(and)h(pro)q(cesses)75 566 y(in)h(the)g(other)f(group)g(pro)o(vided)i (the)e(v)m(alue)i Fv(high)f(=)g(true)g Fx(then)g(the)f(union)i(orders)e (the)g(\\lo)o(w")g(group)75 622 y(b)q(efore)g(the)g(\\high")g(group.)23 b(If)17 b(all)h(pro)q(cesses)f(pro)o(vided)g(the)g(same)g Fv(high)g Fx(argumen)o(t)f(then)g(the)h(order)75 679 y(of)g(the)h(union)h(is)f(arbitrary)l(.)27 b(This)18 b(call)h(is)f(blo)q(c)o(king)h(and)f(collectiv)o(e)i(within)f(the)e (union)i(of)f(the)f(t)o(w)o(o)75 735 y(groups.)189 839 y Fl(A)n(dvic)n(e)27 b(to)h(implementors.)118 b Fx(The)29 b(implemen)o(tation)g(of)f Fv(MPI)p 1405 839 14 2 v 16 w(INTERCOMM)p 1695 839 V 17 w(MERGE)p Fx(,)189 895 y Fv(MPI)p 274 895 V 15 w(COMM)p 431 895 V 17 w(FREE)k Fx(and)g Fv(MPI)p 776 895 V 16 w(COMM)p 934 895 V 16 w(DUP)g Fx(are)f(similar)i(to)e(the)h(implemen)o(tation)h(of)189 952 y Fv(MPI)p 274 952 V 15 w(INTERCOMM)p 563 952 V 17 w(CREA)l(TE)p Fx(,)11 b(except)g(that)e(con)o(texts)h(priv)m(ate)g(to)g (the)g(input)h(in)o(ter-comm)o(unicator)189 1008 y(are)f(used)i(for)f (comm)o(unication)g(b)q(et)o(w)o(een)h(group)f(leaders)g(rather)g(than) g(con)o(texts)f(inside)j(a)e(bridge)189 1065 y(comm)o(unicator.)19 b(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 1186 y Fj(5.6.3)49 b(Inter-Comm)n(unication)14 b(Examples)75 1272 y Fv(Example)f(1:)19 b(Three-Group)e(\\Pip)q(eline") 75 1358 y Fx(Groups)f(0)g(and)h(1)f(comm)o(unicate.)24 b(Groups)16 b(1)h(and)f(2)g(comm)o(unicate.)24 b(Therefore,)17 b(group)f(0)g(requires)75 1414 y(one)f(in)o(ter-comm)o(unicator,)f (group)g(1)g(requires)i(t)o(w)o(o)d(in)o(ter-comm)o(unicators,)h(and)h (group)f(2)g(requires)h(1)75 1471 y(in)o(ter-comm)o(unicator.)147 1574 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 1631 y({)194 1687 y(MPI_Comm)71 b(myComm;)166 b(/*)23 b(intra-communicator)f(of)h(local)h(sub-group)e(*/)194 1744 y(MPI_Comm)71 b(myFirstComm;)46 b(/*)23 b(inter-communicator)f(*/) 194 1800 y(MPI_Comm)71 b(mySecondComm;)22 b(/*)h(second)h (inter-communicator)d(\(group)i(1)h(only\))f(*/)194 1857 y(int)h(membershipKey;)194 1913 y(int)g(rank;)194 2026 y(MPI_Init\(&argc,)e(&argv\);)194 2083 y(MPI_Comm_rank\(MPI_COMM_WORLD) o(,)f(&rank\);)194 2195 y(/*)j(User)f(code)h(must)f(generate)g (membershipKey)f(in)h(the)h(range)f([0,)g(1,)h(2])g(*/)194 2252 y(membershipKey)e(=)i(rank)f(\045)h(3;)194 2365 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/)194 2421 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e(membershipKey,)h(rank,)h (&myComm\);)194 2534 y(/*)h(Build)f(inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 2591 y(if)i(\(membershipKey)e (==)h(0\))194 2647 y({)501 b(/*)24 b(Group)f(0)h(communicates)e(with)h (group)h(1.)f(*/)242 2704 y(MPI_Intercomm_create\()e(myComm,)i(0,)h (MPI_COMM_WORLD,)d(1,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 159 167 159 166 bop 75 -100 a Fm(5.6.)29 b(INTER-COMMUNICA)l(TION)1052 b Fx(159)383 4 y 18945146 5115185 0 0 28417720 7696465 startTexFig 383 4 a %%BeginDocument: context-fig-2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def /col8 {.68 .85 .9 setrgbcolor} bind def /col9 {0 .39 0 setrgbcolor} bind def /col10 {.65 .17 .17 setrgbcolor} bind def /col11 {1 .51 0 setrgbcolor} bind def /col12 {.63 .13 .94 setrgbcolor} bind def /col13 {1 .75 .8 setrgbcolor} bind def /col14 {.7 .13 .13 setrgbcolor} bind def /col15 {1 .84 0 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -63.0 198.0 translate 0.900 -0.900 scale 0.500 setlinewidth n 199 219 m 199 119 l 99 119 l 99 219 l clp gs col-1 s gr n 359 219 m 359 119 l 259 119 l 259 219 l clp gs col-1 s gr n 519 219 m 519 119 l 419 119 l 419 219 l clp gs col-1 s gr 1.000 setlinewidth n 220.000 173.000 m 204.000 169.000 l 220.000 165.000 l gs 2 setlinejoin col-1 s gr n 204 169 m 254 169 l gs col-1 s gr n 238.000 165.000 m 254.000 169.000 l 238.000 173.000 l gs 2 setlinejoin col-1 s gr n 380.000 173.000 m 364.000 169.000 l 380.000 165.000 l gs 2 setlinejoin col-1 s gr n 364 169 m 414 169 l gs col-1 s gr n 398.000 165.000 m 414.000 169.000 l 398.000 173.000 l gs 2 setlinejoin col-1 s gr n 78.000 165.000 m 94.000 169.000 l 78.000 173.000 l gs 2 setlinejoin col-1 s gr n 94 169 m 69 169 l 69 89 l 549 89 l 549 169 l 524 169 l gs col-1 s gr n 540.000 173.000 m 524.000 169.000 l 540.000 165.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 279 174 m gs 1 -1 scale (Group 1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 439 174 m gs 1 -1 scale (Group 2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 174 m gs 1 -1 scale (Group 0) col-1 show gr $F2psEnd %%EndDocument endTexFig 682 426 a Fx(Figure)16 b(5.2:)i(Three-group)e(ring.)743 559 y Fu(1,)24 b(&myFirstComm\);)194 615 y(})194 672 y(else)g(if)f(\(membershipKey)f(==)i(1\))194 728 y({)334 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)g(2.)f(*/) 242 785 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,) 743 841 y(1,)j(&myFirstComm\);)242 898 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 954 y(12,)j(&mySecondComm\);) 194 1011 y(})194 1067 y(else)g(if)f(\(membershipKey)f(==)i(2\))194 1123 y({)501 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)h(1.)f (*/)242 1180 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,) d(1,)743 1236 y(12,)j(&myFirstComm\);)194 1293 y(})194 1406 y(/*)g(Do)f(work)h(...)f(*/)194 1519 y(switch\(membershipKey\))45 b(/*)24 b(free)f(communicators)f(appropriately)g(*/)194 1575 y({)194 1632 y(case)i(1:)266 1688 y(MPI_Comm_free\(&mySecondCo)o (mm\);)194 1744 y(case)g(0:)194 1801 y(case)g(2:)266 1857 y(MPI_Comm_free\(&myFirstCom)o(m\);)266 1914 y(break;)194 1970 y(})194 2083 y(MPI_Finalize\(\);)147 2140 y(})75 2260 y Fv(Example)13 b(2:)19 b(Three-Group)e(\\Ring")75 2346 y Fx(Groups)12 b(0)g(and)h(1)f(comm)o(unicate.)19 b(Groups)13 b(1)f(and)g(2)h(comm)o(unicate.)19 b(Groups)12 b(0)g(and)h(2)f(comm)o(unicate.)75 2402 y(Therefore,)j(eac)o(h)g (requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators.)147 2508 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 2565 y({)194 2621 y(MPI_Comm)71 b(myComm;)142 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)f(*/)194 2678 y(MPI_Comm)71 b(myFirstComm;)22 b(/*)i(inter-communicators)d(*/) -32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 160 168 160 167 bop 75 -100 a Fx(160)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)194 45 y Fu(MPI_Comm)71 b(mySecondComm;)194 102 y(MPI_Status)23 b(status;)194 158 y(int)h(membershipKey;)194 214 y(int)g(rank;)194 327 y(MPI_Init\(&argc,)e(&argv\);)194 384 y (MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&rank\);)194 440 y(...)194 553 y(/*)j(User)f(code)h(must)f(generate)g(membershipKey)f (in)h(the)h(range)f([0,)g(1,)h(2])g(*/)194 610 y(membershipKey)e(=)i (rank)f(\045)h(3;)194 723 y(/*)g(Build)f(intra-communicator)e(for)j (local)f(sub-group)g(*/)194 779 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e (membershipKey,)h(rank,)h(&myComm\);)194 892 y(/*)h(Build)f (inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 948 y(if)i(\(membershipKey)e(==)h(0\))194 1005 y({)310 b(/*)24 b(Group)f(0)h(communicates)e(with)h(groups)g(1)h(and)g(2.)f(*/) 242 1061 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d (1,)743 1118 y(1,)j(&myFirstComm\);)242 1174 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 1231 y(2,)j(&mySecondComm\);) 194 1287 y(})194 1344 y(else)g(if)f(\(membershipKey)f(==)i(1\))194 1400 y({)215 b(/*)24 b(Group)f(1)g(communicates)g(with)g(groups)g(0)h (and)f(2.)h(*/)242 1456 y(MPI_Intercomm_create\()d(myComm,)i(0,)h (MPI_COMM_WORLD,)d(0,)743 1513 y(1,)j(&myFirstComm\);)242 1569 y(MPI_Intercomm_create\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 1626 y(12,)j(&mySecondComm\);)194 1682 y(})194 1739 y(else)g(if)f (\(membershipKey)f(==)i(2\))194 1795 y({)191 b(/*)24 b(Group)f(2)h(communicates)e(with)h(groups)g(0)h(and)f(1.)h(*/)242 1852 y(MPI_Intercomm_create\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,)743 1908 y(2,)j(&myFirstComm\);)242 1965 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 2021 y(12,)j(&mySecondComm\);) 194 2077 y(})194 2190 y(/*)g(Do)f(some)h(work)f(...)h(*/)194 2303 y(/*)g(Then)f(free)h(communicators)e(before)h(terminating...)f(*/) 194 2360 y(MPI_Comm_free\(&myFirstComm\);)194 2416 y (MPI_Comm_free\(&mySecondComm\))o(;)194 2473 y (MPI_Comm_free\(&myComm\);)194 2529 y(MPI_Finalize\(\);)147 2586 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 161 169 161 168 bop 75 -100 a Fm(5.6.)29 b(INTER-COMMUNICA)l(TION)1052 b Fx(161)75 45 y Fv(Example)13 b(3:)19 b(Building)e(Name)12 b(Service)j(fo)o(r)f(Intercomm)n(unication)75 131 y Fx(The)i(follo)o (wing)g(pro)q(cedures)h(exemplify)g(the)f(pro)q(cess)g(b)o(y)f(whic)o (h)i(a)e(user)h(could)h(create)e(name)h(service)75 187 y(for)g(building)j(in)o(tercomm)o(unicators)e(via)g(a)f(rendezv)o(ous)h (in)o(v)o(olving)h(a)f(serv)o(er)f(comm)o(unicator,)g(and)h(a)75 244 y(tag)d(name)h(selected)i(b)o(y)e(b)q(oth)g(groups.)166 300 y(After)k(all)h Fv(MPI)f Fx(pro)q(cesses)h(execute)g Fv(MPI)p 910 300 14 2 v 15 w(INIT)p Fx(,)f(ev)o(ery)g(pro)q(cess)h (calls)g(the)f(example)h(function,)75 357 y Fv(Init)p 141 357 V 17 w(server\(\))p Fx(,)c(de\014ned)i(b)q(elo)o(w.)26 b(Then,)18 b(if)g(the)f Fv(new)p 975 357 V 17 w(w)o(o)o(rld)f Fx(returned)h(is)h(NULL,)g(the)f(pro)q(cess)g(getting)75 413 y(NULL)e(is)g(required)h(to)e(implemen)o(t)i(a)e(serv)o(er)g (function,)h(in)g(a)f(reactiv)o(e)h(lo)q(op,)g Fv(Do)p 1502 413 V 16 w(server\(\))p Fx(.)j(Ev)o(ery)o(one)75 470 y(else)g(just)f(do)q(es)g(their)h(prescrib)q(ed)g(computation,)f (using)h Fv(new)p 1168 470 V 17 w(w)o(o)o(rld)e Fx(as)h(the)g(new)h (e\013ectiv)o(e)f(\\global")75 526 y(comm)o(unicator.)k(One)16 b(designated)h(pro)q(cess)e(calls)i Fv(Undo)p 1076 526 V 17 w(Server\(\))e Fx(to)g(get)g(rid)h(of)g(the)f(serv)o(er)h(when)g (it)75 583 y(is)g(not)f(needed)h(an)o(y)f(longer.)166 639 y(F)l(eatures)g(of)g(this)g(approac)o(h)g(include:)143 745 y Ft(\017)23 b Fx(Supp)q(ort)15 b(for)g(m)o(ultiple)i(name)e(serv)o (ers)143 839 y Ft(\017)23 b Fx(Abilit)o(y)16 b(to)f(scop)q(e)h(the)f (name)g(serv)o(ers)g(to)f(sp)q(eci\014c)k(pro)q(cesses)143 933 y Ft(\017)23 b Fx(Abilit)o(y)16 b(to)f(mak)o(e)g(suc)o(h)g(serv)o (ers)g(come)g(and)g(go)g(as)g(desired.)75 1039 y Fu(#define)23 b(INIT_SERVER_TAG_1)e(666)75 1096 y(#define)i(UNDO_SERVER_TAG_1)93 b(777)75 1209 y(static)23 b(int)g(server_key_val;)75 1321 y(/*)h(for)f(attribute)g(management)f(for)i(server_comm,)46 b(copy)23 b(callback:)g(*/)75 1378 y(void)g(handle_copy_fn\(MPI_Comm)e (*oldcomm,)i(int)g(*keyval,)g(void)g(*extra_state,)75 1434 y(void)g(*attribute_val_in,)f(void)h(**attribute_val_out,)e(int)j (*flag\))75 1491 y({)147 1547 y(/*)f(copy)g(the)h(handle)f(*/)147 1604 y(*attribute_val_out)e(=)j(attribute_val_in;)147 1660 y(*flag)f(=)g(1;)h(/*)g(indicate)e(that)i(copy)f(to)h(happen)f(*/) 75 1717 y(})75 1830 y(int)g(Init_server\(peer_comm,)e(rank_of_server,)h (server_comm,)g(new_world\))75 1886 y(MPI_Comm)h(peer_comm;)75 1942 y(int)g(rank_of_server;)75 1999 y(MPI_Comm)g(*server_comm;)75 2055 y(MPI_Comm)g(*new_world;)94 b(/*)23 b(new)h(effective)e(world,)h (sans)h(server)f(*/)75 2112 y({)170 2168 y(MPI_Comm)g(temp_comm,)g (lone_comm;)170 2225 y(MPI_Group)g(peer_group,)f(temp_group;)170 2281 y(int)i(rank_in_peer_comm,)d(size,)i(color,)h(key)f(=)h(0;)170 2338 y(int)g(peer_leader,)e(peer_leader_rank_in_temp_co)o(mm;)170 2451 y(MPI_Comm_rank\(peer_comm,)f(&rank_in_peer_comm\);)170 2507 y(MPI_Comm_size\(peer_comm,)g(&size\);)170 2620 y(if)j(\(\(size)f(<)h(2\))f(||)h(\(0)g(>)f(rank_of_server\))f(||)i (\(rank_of_server)e(>=)h(size\)\))266 2676 y(return)g (\(MPI_ERR_OTHER\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 162 170 162 169 bop 75 -100 a Fx(162)123 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)170 102 y Fu(/*)24 b(create)f(two)h(communicators,)d(by)j(splitting)f (peer_comm)242 158 y(into)g(the)h(server)f(process,)g(and)g(everyone)g (else)g(*/)170 271 y(peer_leader)g(=)h(\(rank_of_server)d(+)j(1\))g (\045)f(size;)47 b(/*)24 b(arbitrary)f(choice)g(*/)170 384 y(if)h(\(\(color)f(=)h(\(rank_in_peer_comm)d(==)j (rank_of_server\)\)\))170 440 y({)266 497 y(MPI_Comm_split\(peer_comm,) c(color,)j(key,)h(&lone_comm\);)266 610 y(MPI_Intercomm_create\(lone)o (_comm,)d(0,)i(peer_comm,)g(peer_leader,)719 666 y(INIT_SERVER_TAG_1,)f (server_comm\);)266 779 y(MPI_Comm_free\(&lone_comm\))o(;)266 835 y(*new_world)g(=)i(\(MPI_Comm\))f(0;)170 892 y(})170 948 y(else)170 1005 y({)266 1061 y(MPI_Comm_Split\(peer_comm,)d(color,) j(key,)h(&temp_comm\);)266 1174 y(MPI_Comm_group\(peer_comm,)c (&peer_group\);)266 1231 y(MPI_Comm_group\(temp_comm,)g(&temp_group\);) 266 1287 y(MPI_Group_translate_ranks)o(\(peer_gr)o(oup,)h(1,)i (&peer_leader,)886 1344 y(temp_group,)g(&peer_leader_rank_in_tem)o (p_comm\))o(;)266 1456 y(MPI_Intercomm_create\(temp)o(_comm,)e (peer_leader_rank_in_tem)o(p_comm,)719 1513 y(peer_comm,)i (rank_of_server,)719 1569 y(INIT_SERVER_TAG_1,)f(server_comm\);)266 1682 y(/*)h(attach)h(new_world)e(communication)g(attribute)h(to)g (server_comm:)g(*/)266 1795 y(/*)g(CRITICAL)g(SECTION)g(FOR)h (MULTITHREADING)e(*/)266 1852 y(if\(server_keyval)g(==)h (MPI_KEYVAL_INVALID\))266 1908 y({)361 1965 y(/*)h(acquire)f(the)g (process-local)f(name)i(for)f(the)h(server)f(keyval)g(*/)361 2021 y(MPI_Attr_keyval_create\(handle)o(_copy_f)o(n,)e(NULL,)i (&server_keyval,)f(NULL\);)266 2077 y(})266 2190 y(*new_world)g(=)i (temp_comm;)266 2303 y(/*)f(Cache)h(handle)f(of)g(intra-communicator)f (on)h(inter-communicator:)f(*/)266 2360 y(MPI_Attr_put\(server_comm,)e (server_keyval,)i(\(void)h(*\)\(*new_world\)\);)170 2416 y(})170 2529 y(return)g(\(MPI_SUCCESS\);)75 2586 y(})166 2692 y Fx(The)15 b(actual)h(serv)o(er)f(pro)q(cess)g(w)o(ould)h(commit) f(to)f(running)j(the)e(follo)o(wing)h(co)q(de:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 163 171 163 170 bop 75 -100 a Fm(5.6.)29 b(INTER-COMMUNICA)l(TION)1052 b Fx(163)75 45 y Fu(int)23 b(Do_server\(server_comm\))75 102 y(MPI_Comm)g(server_comm;)75 158 y({)170 214 y(void)h (init_queue\(\);)170 271 y(int)g(en_queue\(\),)e(de_queue\(\);)h(/*)g (keep)h(triplets)e(of)i(integers)910 327 y(for)g(later)f(matching)g (\(fns)g(not)g(shown\))g(*/)170 440 y(MPI_Comm)g(comm;)170 497 y(MPI_Status)g(status;)170 553 y(int)h(client_tag,)e (client_source;)170 610 y(int)i(client_rank_in_new_world,)c (pairs_rank_in_new_world;)170 666 y(int)k(buffer[10],)e(count)h(=)h(1;) 170 779 y(void)g(*queue;)170 835 y(init_queue\(&queue\);)170 1005 y(for)g(\(;;\))170 1061 y({)266 1118 y(MPI_Recv\(buffer,)e(count,) h(MPI_INT,)f(MPI_ANY_SOURCE,)g(MPI_ANY_TAG,)481 1174 y(server_comm,)g(&status\);)h(/*)g(accept)g(from)g(any)1101 1231 y(client)g(*/)266 1344 y(/*)g(determine)g(client:)g(*/)266 1400 y(client_tag)f(=)i(status.MPI_TAG;)266 1456 y(client_source)e(=)i (status.MPI_SOURCE;)266 1513 y(client_rank_in_new_world)c(=)k (buffer[0];)266 1626 y(if)f(\(client_tag)g(==)g(UNDO_SERVER_TAG_1\))165 b(/*)24 b(client)f(that)1292 1682 y(terminates)g(server)g(*/)266 1739 y({)361 1795 y(while)h(\(de_queue\(queue,)d(MPI_ANY_TAG,)h (&pairs_rank_in_new_world,)743 1852 y(&pairs_rank_in_server\)\))457 1908 y(;)361 2021 y(MPI_Intercomm_free\(&server_co)o(mm\);)361 2077 y(break;)266 2134 y(})266 2247 y(if)h(\(de_queue\(queue,)f (client_tag,)h(&pairs_rank_in_new_worl)o(d,)648 2303 y(&pairs_rank_in_server\)\))266 2360 y({)361 2416 y(/*)h(matched)f (pair)g(with)g(same)h(tag,)f(tell)g(them)433 2473 y(about)g(each)g (other!)g(*/)361 2529 y(buffer[0])g(=)h(pairs_rank_in_new_world;)361 2586 y(MPI_Send\(buffer,)e(1,)i(MPI_INT,)e(client_src,)h(client_tag,)f (server_comm\);)361 2698 y(buffer[0])h(=)h(client_rank_in_new_world)o (;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 164 172 164 171 bop 75 -100 a Fx(164)123 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)361 45 y Fu(MPI_Send\(buffer,)22 b(1,)i(MPI_INT,)e(pairs_rank_in_server,)f (client_tag,)576 102 y(server_comm\);)266 158 y(})266 214 y(else)361 271 y(en_queue\(queue,)h(client_tag,)h(client_source,)e (client_rank_in_new_world\);)170 384 y(})75 440 y(})166 534 y Fx(A)d(particular)h(pro)q(cess)g(w)o(ould)f(b)q(e)h(resp)q (onsible)h(for)e(ending)h(the)g(serv)o(er)f(when)g(it)h(is)f(no)h (longer)75 591 y(needed.)i(Its)15 b(call)i(to)d Fv(Undo)p 553 591 14 2 v 18 w(server)h Fx(w)o(ould)g(terminate)g(serv)o(er)g (function.)75 684 y Fu(int)23 b(Undo_server\(server_comm\))116 b(/*)24 b(example)f(client)g(that)g(ends)h(server)f(*/)75 741 y(MPI_Comm)g(*server_comm;)75 797 y({)170 854 y(int)h(buffer)f(=)h (0;)170 910 y(MPI_Send\(&buffer,)e(1,)i(MPI_INT,)e(0,)i (UNDO_SERVER_TAG_1,)d(*server_comm\);)170 967 y (MPI_Intercomm_free\(server_com)o(m\);)75 1023 y(})166 1117 y Fx(The)15 b(follo)o(wing)g(is)g(a)f(blo)q(c)o(king)i (name-service)f(for)f(in)o(ter-comm)o(unication,)i(with)f(same)f(seman) o(tic)75 1173 y(restrictions)k(as)g Fv(MPI)p 457 1173 V 16 w(Intercomm)p 679 1173 V 10 w(create)p Fx(,)h(but)f(simpli\014ed)i (syn)o(tax.)27 b(It)18 b(uses)g(the)g(functionalit)o(y)h(just)75 1230 y(de\014ned)e(to)d(create)h(the)g(name)h(service.)75 1324 y Fu(int)23 b(Intercomm_name_create\(local_c)o(omm,)e (server_comm,)h(tag,)h(comm\))75 1380 y(MPI_Comm)g(local_comm,)f (server_comm;)75 1437 y(int)h(tag;)75 1493 y(MPI_Comm)g(*comm;)75 1549 y({)170 1606 y(int)h(error;)170 1662 y(int)g(found;)71 b(/*)23 b(attribute)g(acquisition)f(mgmt)i(for)f(new_world)g(comm)g(in) h(server_comm)e(*/)170 1719 y(void)i(*val;)170 1832 y(MPI_Comm)f (new_world;)170 1945 y(int)h(buffer[10],)e(rank;)170 2001 y(int)i(local_leader)e(=)i(0;)170 2114 y (MPI_Attr_get\(server_comm,)d(server_keyval,)h(&val,)h(&found\);)170 2170 y(new_world)g(=)h(\(MPI_Comm\)val;)e(/*)h(retrieve)g(cached)g (handle)g(*/)170 2283 y(MPI_Comm_rank\(server_comm,)e(&rank\);)47 b(/*)23 b(rank)g(in)h(local)f(group)g(*/)170 2396 y(if)h(\(rank)f(==)h (local_leader\))170 2453 y({)266 2509 y(buffer[0])f(=)g(rank;)266 2566 y(MPI_Send\(&buffer,)e(1,)j(MPI_INT,)f(0,)g(tag,)h(server_comm\);) 266 2622 y(MPI_Recv\(&buffer,)d(1,)j(MPI_INT,)f(0,)g(tag,)h (server_comm\);)170 2679 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 165 173 165 172 bop 75 -100 a Fm(5.7.)34 b(CA)o(CHING)1407 b Fx(165)170 102 y Fu(error)24 b(=)f(MPI_Intercomm_create\(local_le)o (ader,)e(local_comm,)h(buffer[0],)g(new_world,)624 158 y(tag,)h(comm\);)170 271 y(return\(error\);)75 327 y(})75 470 y Fn(5.7)59 b(Caching)75 571 y Fv(MPI)18 b Fx(pro)o(vides)g(a)g (\\cac)o(hing")g(facilit)o(y)h(that)f(allo)o(ws)g(an)g(application)i (to)d(attac)o(h)g(arbitrary)h(pieces)h(of)75 628 y(information,)24 b(called)g Fq(attributes)p Fx(,)g(to)e(comm)o(unicators.)40 b(More)22 b(precisely)l(,)k(the)c(cac)o(hing)h(facilit)o(y)75 684 y(allo)o(ws)15 b(a)g(p)q(ortable)h(library)g(to)e(do)i(the)f(follo) o(wing:)143 775 y Ft(\017)23 b Fx(pass)10 b(information)g(b)q(et)o(w)o (een)g(calls)i(b)o(y)e(asso)q(ciating)g(it)h(with)f(an)g Fv(MPI)g Fx(in)o(tra-)g(or)f(in)o(ter-comm)o(unicator,)143 867 y Ft(\017)23 b Fx(quic)o(kly)16 b(retriev)o(e)g(that)e (information,)h(and)143 960 y Ft(\017)23 b Fx(b)q(e)18 b(guaran)o(teed)e(that)h(out-of-date)g(information)g(is)h(nev)o(er)g (retriev)o(ed,)f(ev)o(en)h(if)g(the)f(comm)o(uni-)189 1016 y(cator)d(is)i(freed)f(and)h(its)f(handle)h(subsequen)o(tly)h (reused)f(b)o(y)f Fv(MPI)p Fx(.)166 1106 y(The)i(cac)o(hing)h (capabilities,)h(in)f(some)e(form,)g(are)h(required)h(b)o(y)e(built-in) k Fv(MPI)c Fx(routines)h(suc)o(h)g(as)75 1163 y(collectiv)o(e)22 b(comm)o(unication)e(and)g(application)i(top)q(ology)l(.)33 b(De\014ning)21 b(an)f(in)o(terface)g(to)f(these)h(capa-)75 1219 y(bilities)h(as)c(part)h(of)f(the)h Fv(MPI)g Fx(standard)f(is)i(v) m(aluable)h(b)q(ecause)f(it)f(p)q(ermits)h(routines)f(lik)o(e)h (collectiv)o(e)75 1276 y(comm)o(unication)h(and)f(application)h(top)q (ologies)g(to)e(b)q(e)h(implemen)o(ted)i(as)d(p)q(ortable)i(co)q(de,)g (and)f(also)75 1332 y(b)q(ecause)h(it)g(mak)o(es)e Fv(MPI)h Fx(more)g(extensible)i(b)o(y)e(allo)o(wing)h(user-written)g(routines)f (to)g(use)g(standard)75 1389 y Fv(MPI)c Fx(calling)i(sequences.)189 1490 y Fl(A)n(dvic)n(e)d(to)i(users.)40 b Fx(The)15 b(comm)o(unicator)f Fi(MPI)p 1009 1490 13 2 v 15 w(COMM)p 1154 1490 V 14 w(SELF)h Fx(is)g(a)f(suitable)i(c)o(hoice)g(for)e(p)q(osting)189 1547 y(pro)q(cess-lo)q(cal)21 b(attributes,)e(via)h(this)g (attributing-cac)o(hing)g(mec)o(hanism.)33 b(\()p Fl(End)20 b(of)g(advic)n(e)g(to)189 1603 y(users.)p Fx(\))75 1724 y Fj(5.7.1)49 b(F)o(unctionalit)o(y)75 1810 y Fx(A)o(ttributes)15 b(are)g(attac)o(hed)g(to)g(comm)o(unicators.)20 b(A)o(ttributes)c(are)f (lo)q(cal)h(to)f(the)h(pro)q(cess)f(and)h(sp)q(eci\014c)75 1866 y(to)h(the)i(comm)o(unicator)e(to)h(whic)o(h)h(they)f(are)g(attac) o(hed.)28 b(A)o(ttributes)18 b(are)f(not)h(propagated)g(b)o(y)g Fv(MPI)75 1923 y Fx(from)k(one)h(comm)o(unicator)f(to)g(another)g (except)h(when)g(the)g(comm)o(unicator)f(is)i(duplicated)g(using)75 1979 y Fv(MPI)p 160 1979 14 2 v 16 w(COMM)p 318 1979 V 16 w(DUP)e Fx(\(and)f(ev)o(en)h(then)f(the)h(application)h(m)o(ust)e (giv)o(e)g(sp)q(eci\014c)i(p)q(ermission)g(through)75 2036 y(callbac)o(k)16 b(functions)g(for)f(the)g(attribute)g(to)g(b)q(e) h(copied\).)189 2137 y Fl(A)n(dvic)n(e)e(to)i(implementors.)39 b Fx(A)o(ttributes)15 b(are)f(scalar)g(v)m(alues,)h(equal)g(in)g(size)h (to,)d(or)h(larger)g(than)189 2194 y(a)i(C-language)g(p)q(oin)o(ter.)24 b(A)o(ttributes)17 b(can)f(alw)o(a)o(ys)g(hold)h(an)g Fv(MPI)e Fx(handle.)25 b(\()p Fl(End)17 b(of)g(advic)n(e)h(to)189 2250 y(implementors.)p Fx(\))166 2351 y(The)12 b(cac)o(hing)g(in)o (terface)g(de\014ned)g(here)g(represen)o(ts)g(that)e(attributes)i(b)q (e)g(stored)f(b)o(y)g Fv(MPI)g Fx(opaquely)75 2408 y(within)16 b(a)f(comm)o(unicator.)20 b(Accessor)15 b(functions)h(include)i(the)d (follo)o(wing:)143 2498 y Ft(\017)23 b Fx(obtain)16 b(a)f(k)o(ey)h(v)m (alue)h(\(used)f(to)g(iden)o(tify)h(an)f(attribute\);)f(the)h(user)g (sp)q(eci\014es)i(\\callbac)o(k")e(func-)189 2555 y(tions)h(b)o(y)g (whic)o(h)h Fv(MPI)f Fx(informs)g(the)h(application)h(when)f(the)f (comm)o(unicator)g(is)h(destro)o(y)o(ed)e(or)189 2611 y(copied.)143 2704 y Ft(\017)23 b Fx(store)14 b(and)i(retriev)o(e)f (the)g(v)m(alue)i(of)d(an)i(attribute;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 166 174 166 173 bop 75 -100 a Fx(166)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)189 45 y Fl(A)n(dvic)n(e)10 b(to)i(implementors.)37 b Fx(Cac)o(hing)10 b(and)h(callbac)o(k)g(functions)g(are)f(only)g(called)i(sync)o (hronously)l(,)189 102 y(in)k(resp)q(onse)f(to)g(explicit)j (application)e(requests.)k(This)c(a)o(v)o(oid)f(problems)h(that)f (result)g(from)g(re-)189 158 y(p)q(eated)h(crossings)h(b)q(et)o(w)o (een)f(user)h(and)f(system)g(space.)24 b(\(This)16 b(sync)o(hronous)g (calling)i(rule)f(is)g(a)189 214 y(general)e(prop)q(ert)o(y)g(of)g Fv(MPI)p Fx(.\))189 290 y(The)20 b(c)o(hoice)h(of)f(k)o(ey)g(v)m(alues) i(is)f(under)g(con)o(trol)f(of)f Fv(MPI)p Fx(.)h(This)h(allo)o(ws)f Fv(MPI)g Fx(to)f(optimize)j(its)189 346 y(implemen)o(tation)14 b(of)e(attribute)h(sets.)19 b(It)12 b(also)h(a)o(v)o(oids)g(con\015ict) h(b)q(et)o(w)o(een)f(indep)q(enden)o(t)i(mo)q(dules)189 403 y(cac)o(hing)h(information)f(on)g(the)g(same)g(comm)o(unicators.) 189 478 y(A)g(m)o(uc)o(h)g(smaller)h(in)o(terface,)f(consisting)h(of)e (just)h(a)g(callbac)o(k)h(facilit)o(y)l(,)g(w)o(ould)f(allo)o(w)g(the)g (en)o(tire)189 534 y(cac)o(hing)22 b(facilit)o(y)g(to)f(b)q(e)h (implemen)o(ted)h(b)o(y)e(p)q(ortable)h(co)q(de.)39 b(Ho)o(w)o(ev)o (er,)21 b(with)h(the)f(minimal)189 591 y(callbac)o(k)i(in)o(terface,)g (some)e(form)h(of)f(table)h(searc)o(hing)g(is)h(implied)h(b)o(y)e(the)g (need)g(to)g(handle)189 647 y(arbitrary)10 b(comm)o(unicator.)18 b(In)11 b(con)o(trast,)f(the)h(more)f(complete)h(in)o(terface)g (de\014ned)h(here)f(p)q(ermits)189 703 y(rapid)17 b(access)f(to)g (attributes)h(through)f(the)h(use)f(of)h(p)q(oin)o(ters)g(in)g(comm)o (unicators)f(\(to)g(\014nd)h(the)189 760 y(attribute)22 b(table\))g(and)g(clev)o(erly)i(c)o(hosen)e(k)o(ey)g(v)m(alues)i(\(to)d (retriev)o(e)h(individual)k(attributes\).)189 816 y(In)18 b(ligh)o(t)g(of)f(the)h(e\016ciency)h(\\hit")e(inheren)o(t)i(in)f(the)f (minimal)j(in)o(terface,)e(the)f(more)g(complete)189 873 y(in)o(terface)e(de\014ned)i(here)e(is)h(seen)g(to)e(b)q(e)i(sup)q (erior.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 979 y Fv(MPI)f Fx(pro)o(vides)g(the)h(follo)o(wing)f(services) i(related)e(to)g(cac)o(hing.)20 b(They)c(are)f(all)h(pro)q(cess)f(lo)q (cal.)75 1130 y Fv(MPI)p 160 1130 14 2 v 16 w(KEYV)l(AL)p 346 1130 V 17 w(CREA)l(TE\(cop)o(y)p 636 1130 V 17 w(fn,)g(delete)p 828 1130 V 18 w(fn,)g(k)o(eyval,)g(extra)p 1150 1130 V 16 w(state\))117 1207 y Fk(IN)155 b Fv(cop)o(y)p 407 1207 V 17 w(fn)447 b Fk(Cop)o(y)14 b(callbac)o(k)f(function)g(for)h Fi(k)o(eyval)117 1282 y Fk(IN)155 b Fv(delete)p 431 1282 V 18 w(fn)422 b Fk(Delete)15 b(callbac)o(k)e(function)g(for)h Fi(k)o(eyval)117 1358 y Fk(OUT)108 b Fv(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue)f(for)h(future)g(access)i(\(in)o(teger\))117 1433 y(IN)155 b Fv(extra)p 416 1433 V 17 w(state)384 b Fk(Extra)14 b(state)h(for)f(callbac)o(k)f(functions)75 1557 y Fu(int)23 b(MPI)p 245 1557 15 2 v 17 w(Keyval)p 406 1557 V 17 w(create\(MPI)p 663 1557 V 15 w(Copy)p 774 1557 V 17 w(function)g(*copy)p 1126 1557 V 16 w(fn,)h(MPI)p 1310 1557 V 17 w(Delete)p 1471 1557 V 16 w(function)393 1614 y(*delete)p 564 1614 V 16 w(fn,)g(int)f(*keyval,)g(void*)g(extra)p 1249 1614 V 17 w(state\))75 1700 y(MPI)p 150 1700 V 17 w(KEYVAL)p 311 1700 V 16 w(CREATE\(COPY)p 591 1700 V 16 w(FN,)g(DELETE)p 846 1700 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1269 1700 V 16 w(STATE,)h(IERROR\))170 1756 y(EXTERNAL)f(COPY)p 484 1756 V 17 w(FN,)g(DELETE)p 740 1756 V 17 w(FN)170 1813 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 1813 V 17 w(STATE,)g(IERROR)166 1899 y Fx(Generates)18 b(a)g(new)g(attribute)h(k)o(ey)l(.)29 b(Keys)19 b(are)f(lo)q(cally)i(unique)g(in)f(a)f(pro)q(cess,)h(and)f (opaque)h(to)75 1956 y(user,)g(though)g(they)f(are)h(explicitly)i (stored)d(in)i(in)o(tegers.)30 b(Once)19 b(allo)q(cated,)i(the)d(k)o (ey)h(v)m(alue)h(can)e(b)q(e)75 2012 y(used)e(to)e(asso)q(ciate)i (attributes)e(and)i(access)f(them)g(on)h(an)o(y)e(lo)q(cally)j (de\014ned)g(comm)o(unicator.)166 2069 y(The)10 b Fv(cop)o(y)p 343 2069 14 2 v 17 w(fn)g Fx(function)h(is)g(in)o(v)o(ok)o(ed)f(when)h (a)f(comm)o(unicator)f(is)i(duplicated)h(b)o(y)e Fv(MPI)p 1617 2069 V 16 w(COMM)p 1775 2069 V 17 w(DUP)p Fx(.)75 2125 y Fv(cop)o(y)p 164 2125 V 16 w(fn)16 b Fx(should)g(b)q(e)g(of)f(t) o(yp)q(e)g Fi(MPI)p 667 2125 13 2 v 14 w(Cop)o(y)p 768 2125 V 15 w(function)p Fx(,)h(whic)o(h)g(is)g(de\014ned)g(as)f(follo)o (ws:)75 2219 y Fu(typedef)23 b(int)g(\(MPI\\_Copy\\_function\))75 2275 y(\(MPI_Comm)g(*oldcomm,)f(int)i(*keyval,)e(void)i(*extra_state,) 505 2332 y(void)f(*attribute_val_in,)e(void)j(**attribute_val_out,)d (int)i(*flag\))166 2426 y Fx(An)15 b(example)h(of)f(suc)o(h)h(a)f (function)h(declaration)g(is)f(as)g(follo)o(ws:)75 2520 y Fu(int)23 b(Copy_function\(MPI_Comm)e(*oldcomm,)i(int)g(*keyval,)g (void)g(*extra_state,)505 2576 y(void)g(*attribute_val_in,)e(void)j (**attribute_val_out,)d(int)i(*flag\))166 2670 y Fx(The)15 b(F)o(OR)l(TRAN)h(protot)o(yp)q(e)f(for)f(this)i(function)g(is)g(as)f (follo)o(ws:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 167 175 167 174 bop 75 -100 a Fm(5.7.)34 b(CA)o(CHING)1407 b Fx(167)75 45 y Fu(FUNCTION)23 b(COPY)p 389 45 15 2 v 16 w(FN\(OLDCOMM,)g(KEYVAL,)g(EXTRA)p 1003 45 V 16 w(STATE,)g (ATTRIBUTE)p 1402 45 V 16 w(VAL)p 1490 45 V 17 w(IN,)393 102 y(ATTRIBUTE)p 612 102 V 16 w(VAL)p 700 102 V 17 w(OUT,)g(FLAG\))170 158 y(INTEGER)g(OLDCOMM,)g(KEYVAL,)g(EXTRA)p 890 158 V 17 w(STATE,)g(ATTRIBUTE)p 1290 158 V 16 w(VAL)p 1378 158 V 16 w(IN,)170 214 y(ATTRIBUTE)p 389 214 V 16 w(VAL)p 477 214 V 17 w(OUT)170 271 y(LOGICAL)g(FLAG)166 357 y Fx(The)13 b(cop)o(y)g(callbac)o(k)i(function)f(is)f(in)o(v)o(ok)o(ed)h (for)e(eac)o(h)h(k)o(ey)g(v)m(alue)i(in)f Fv(oldcomm)7 b Fx(in)14 b(arbitrary)f(order.)75 414 y(Eac)o(h)18 b(call)i(to)e(the)g (cop)o(y)g(callbac)o(k)i(is)f(made)f(with)h(a)f(k)o(ey)g(v)m(alue)i (and)e(its)h(corresp)q(onding)g(attribute.)75 470 y(If)f(it)g(returns)g Fi(\015ag)e(=)g(0)p Fx(,)i(then)g(the)g(attribute)f(is)h(deleted)h(in)g (the)f(duplicated)i(comm)o(unicator.)27 b(Oth-)75 527 y(erwise)17 b(\()p Fi(\015ag)e(=)h(1)p Fx(\),)g(the)g(new)h(attribute)g (v)m(alue)h(is)f(set)g(via)g Fv(attribute)p 1274 527 14 2 v 18 w(val)p 1346 527 V 16 w(out)p Fx(.)25 b(The)17 b(function)g(returns)75 583 y Fi(MPI)p 152 583 13 2 v 14 w(SUCCESS)d Fx(on)h(success)h(and)f(an)g(error)f(co)q(de)i(on)f (failure)h(\(in)g(whic)o(h)g(case)f Fv(MPI)p 1525 583 14 2 v 16 w(COMM)p 1683 583 V 16 w(DUP)g Fx(will)75 640 y(fail\).)166 696 y Fv(cop)o(y)p 255 696 V 16 w(fn)g Fx(ma)o(y)g(b)q(e)g(sp)q(eci\014ed)i(as)d Fv(MPI)p 803 696 V 16 w(NULL)p 932 696 V 16 w(FN)h Fx(from)f(either)i(C)e(or)g(F)o (OR)l(TRAN,)i(in)f(whic)o(h)h(case)75 753 y(no)k(cop)o(y)h(callbac)o(k) g(o)q(ccurs)g(for)e Fv(k)o(eyval)p Fx(;)k Fv(MPI)p 891 753 V 16 w(NULL)p 1020 753 V 16 w(FN)d Fx(is)h(a)f(function)h(that)f (do)q(es)g(nothing)h(other)75 809 y(than)c(returning)g Fi(\015ag)e(=)h(0)p Fx(.)24 b(In)18 b(C,)e(the)h(NULL)h(function)f(p)q (oin)o(ter)g(has)g(the)g(same)f(b)q(eha)o(vior)i(as)e(using)75 865 y Fv(MPI)p 160 865 V 16 w(NULL)p 289 865 V 16 w(FN)p Fx(.)i(As)h(a)f(further)h(con)o(v)o(enience,)h Fv(MPI)p 1010 865 V 16 w(DUP)p 1119 865 V 17 w(FN)e Fx(is)h(a)g(simple-minded)i (cop)o(y)e(callbac)o(k)75 922 y(a)o(v)m(ailable)g(from)d(C)h(and)h(F)o (OR)l(TRAN;)f(it)h(sets)e Fi(\015ag)g(=)g(1)p Fx(,)h(and)g(returns)g (the)h(v)m(alue)g(of)f Fv(attribute)p 1756 922 V 18 w(val)p 1828 922 V 16 w(in)75 978 y Fx(in)f Fv(attribute)p 294 978 V 18 w(val)p 366 978 V 16 w(out)p Fx(.)166 1035 y(Note)f(that)h (the)g(C)f(v)o(ersion)h(of)g(this)g Fv(MPI)p 885 1035 V 16 w(COMM)p 1043 1035 V 16 w(DUP)g Fx(assumes)g(that)f(the)h(callbac) o(k)h(functions)75 1091 y(follo)o(w)11 b(the)g(C)g(protot)o(yp)q(e,)f (while)j(the)e(corresp)q(onding)h(F)o(OR)l(TRAN)f(v)o(ersion)g(assumes) g(the)g(F)o(OR)l(TRAN)75 1148 y(protot)o(yp)q(e.)189 1245 y Fl(A)n(dvic)n(e)19 b(to)i(users.)69 b Fx(A)20 b(v)m(alid)i(cop)o(y)d(function)i(is)g(one)f(that)f(completely)i (duplicates)h(the)e(in-)189 1302 y(formation)d(b)o(y)i(making)f(a)g (full)i(duplicate)g(cop)o(y)e(of)g(the)g(data)g(structures)g(implied)j (b)o(y)d(an)g(at-)189 1358 y(tribute;)13 b(another)g(migh)o(t)f(just)g (mak)o(e)h(another)f(reference)h(to)f(that)g(data)g(structure,)h(while) h(using)189 1415 y(a)i(reference-coun)o(t)i(mec)o(hanism.)27 b(Other)17 b(t)o(yp)q(es)g(of)g(attributes)g(migh)o(t)g(not)g(cop)o(y)g (at)f(all)i(\(they)189 1471 y(migh)o(t)d(b)q(e)h(sp)q(eci\014c)h(to)d Fv(oldcomm)c Fx(only\).)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (users.)p Fx(\))166 1569 y(Analogous)g(to)g Fv(cop)o(y)p 534 1569 V 17 w(fn)h Fx(is)g(a)f(callbac)o(k)i(deletion)g(function,)f (de\014ned)h(as)e(follo)o(ws.)24 b(The)17 b Fv(delete)p 1823 1569 V 18 w(fn)75 1625 y Fx(function)h(is)g(in)o(v)o(ok)o(ed)g (when)g(a)g(comm)o(unicator)f(is)h(deleted)h(b)o(y)e Fv(MPI)p 1289 1625 V 16 w(COMM)p 1447 1625 V 16 w(FREE)h Fx(or)g(when)g(a)f(call)75 1682 y(is)j(made)f(explicitly)k(to)18 b Fv(MPI)p 596 1682 V 16 w(A)l(TTR)p 729 1682 V 17 w(DELETE)p Fx(.)h Fv(delete)p 1058 1682 V 18 w(fn)h Fx(should)g(b)q(e)g(of)f(t)o (yp)q(e)h Fi(MPI)p 1584 1682 13 2 v 14 w(Delete)p 1707 1682 V 16 w(function)p Fx(,)75 1738 y(whic)o(h)c(is)g(de\014ned)g(as)f (follo)o(ws:)75 1836 y Fu(typedef)23 b(int)g (\(MPI\\_Delete\\_function\))75 1892 y(\(MPI_Comm)g(*comm,)g(int)g (*keyval,)g(void)g(*attribute\\_val,)f(void)h(*extra_state\);)166 1990 y Fx(An)15 b(example)h(of)f(suc)o(h)h(a)f(function)h(declaration)g (is)f(as)g(follo)o(ws:)75 2078 y Fu(int)23 b(Delete_function\(MPI_Comm) e(*comm,)i(int)h(*keyval,)e(void)i(*attribute\\_val,)552 2134 y(void)g(*extra_state\);)166 2222 y Fx(The)15 b(F)o(OR)l(TRAN)h (protot)o(yp)q(e)f(for)f(this)i(function)g(is)g(as)f(follo)o(ws:)75 2278 y Fu(FUNCTION)23 b(DELETE)p 437 2278 15 2 v 16 w(FN\(COMM,)g (KEYVAL,)g(ATTRIBUTE)p 1075 2278 V 16 w(VAL,)g(EXTRA)p 1330 2278 V 17 w(STATE\))170 2335 y(INTEGER)g(COMM,)h(KEYVAL,)e (ATTRIBUTE)p 914 2335 V 16 w(VAL,)i(EXTRA)p 1170 2335 V 16 w(STATE)166 2421 y Fx(This)16 b(function)h(is)f(called)h(b)o(y)f Fv(MPI)p 773 2421 14 2 v 16 w(COMM)p 931 2421 V 16 w(FREE)g Fx(and)g Fv(MPI)p 1243 2421 V 16 w(A)l(TTR)p 1376 2421 V 17 w(DELETE)g Fx(to)f(do)h(whatev)o(er)75 2478 y(is)h(needed)h(to)e (remo)o(v)o(e)g(an)g(attribute.)24 b(It)17 b(ma)o(y)e(b)q(e)j(sp)q (eci\014ed)g(as)f(the)f(n)o(ull)i(function)g(p)q(oin)o(ter)f(in)g(C)f (or)75 2534 y(as)f Fv(MPI)p 216 2534 V 16 w(NULL)p 345 2534 V 16 w(FN)h Fx(from)f(either)h(C)f(or)g(F)o(OR)l(TRAN,)h(in)h (whic)o(h)f(case)g(no)f(delete)i(callbac)o(k)g(o)q(ccurs)f(for)75 2591 y Fv(k)o(eyval)p Fx(.)166 2647 y(The)10 b(sp)q(ecial)i(k)o(ey)e(v) m(alue)i Fi(MPI)p 664 2647 13 2 v 14 w(KEYV)m(AL)p 835 2647 V 14 w(INV)m(ALID)c Fx(is)j(nev)o(er)f(returned)h(b)o(y)f Fv(MPI)p 1493 2647 14 2 v 16 w(KEYV)l(AL)p 1679 2647 V 17 w(CREA)l(TE)p Fx(.)75 2704 y(Therefore,)15 b(it)g(can)g(b)q(e)h (used)g(for)f(static)g(initialization)j(of)d(k)o(ey)g(v)m(alues.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 168 176 168 175 bop 75 -100 a Fx(168)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(KEYV)l(AL)p 346 45 V 17 w(FREE\(k)o(eyval\))117 122 y Fk(INOUT)62 b Fv(k)o(eyval)471 b Fk(F)m(rees)15 b(the)g(in)o(teger)f(k)o(ey)g(v)n(alue)f(\(in)o (teger\))75 247 y Fu(int)23 b(MPI)p 245 247 15 2 v 17 w(Keyval)p 406 247 V 17 w(free\(int)f(*keyval\))75 333 y(MPI)p 150 333 V 17 w(KEYVAL)p 311 333 V 16 w(FREE\(KEYVAL,)g (IERROR\))170 390 y(INTEGER)h(KEYVAL,)g(IERROR)166 476 y Fx(F)l(rees)37 b(an)g(extan)o(t)g(attribute)g(k)o(ey)l(.)87 b(This)37 b(function)i(sets)e(the)g(v)m(alue)i(of)e Fv(k)o(eyval)g Fx(to)75 533 y Fi(MPI)p 152 533 13 2 v 14 w(KEYV)m(AL)p 323 533 V 14 w(INV)m(ALID)p Fx(.)15 b(Note)h(that)f(it)i(is)g(not)f (erroneous)g(to)f(free)i(an)f(attribute)g(k)o(ey)g(that)g(is)h(in)g (use,)75 589 y(b)q(ecause)e(the)e(actual)h(free)f(do)q(es)h(not)f (transpire)h(un)o(til)h(after)e(all)h(references)g(\(in)g(other)f(comm) o(unicators)75 645 y(on)20 b(the)g(pro)q(cess\))f(to)h(the)g(k)o(ey)f (ha)o(v)o(e)h(b)q(een)h(freed.)34 b(These)20 b(references)g(need)h(to)e (b)q(e)i(explictly)h(freed)75 702 y(b)o(y)d(the)h(program,)f(either)h (via)f(calls)i(to)e Fv(MPI)p 887 702 14 2 v 15 w(A)l(TTR)p 1019 702 V 18 w(DELETE)g Fx(that)f(free)i(one)f(attribute)h(instance,) 75 758 y(or)c(b)o(y)g(calls)h(to)e Fv(MPI)p 439 758 V 16 w(COMM)p 597 758 V 17 w(FREE)h Fx(that)g(free)g(all)h(attribute)f (instances)h(asso)q(ciated)f(with)g(the)h(freed)75 815 y(comm)o(unicator.)189 921 y Fl(A)n(dvic)n(e)h(to)h(implementors.)30 b Fx(The)18 b(function)i Fv(MPI)p 1068 921 V 16 w(NULL)p 1197 921 V 16 w(FN)e Fx(need)h(not)f(b)q(e)h(aliased)h(to)e Fu(\(void)189 978 y(\(*\)\)0)i Fx(in)j(C,)e(though)g(this)h(is)g (\014ne.)40 b(It)22 b(could)g(b)q(e)g(a)g(legitimately)h(callable)g (function)g(that)189 1034 y(pro\014les)15 b(and)g(so)g(on.)k(F)l(or)14 b(F)o(OR)l(TRAN,)i(it)f(is)g(most)f(con)o(v)o(enien)o(t)i(to)e(ha)o(v)o (e)g Fv(MPI)p 1576 1034 V 16 w(NULL)p 1705 1034 V 16 w(FN)h Fx(b)q(e)g(a)189 1090 y(legitimate)h(do-nothing)g(function)g (call.\()p Fl(End)g(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 1291 y Fv(MPI)p 160 1291 V 16 w(A)l(TTR)p 293 1291 V 17 w(PUT\(comm)m(,)12 b(k)o(eyval,)i(attribute)p 868 1291 V 18 w(val\))117 1368 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)7 b(to)i(whic)o(h)g(attribute)h(will)e(b)q(e)h (attac)o(hed)h(\(han-)905 1425 y(dle\))117 1500 y(IN)155 b Fv(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue,)f(as)h(returned)h(b)o(y) 905 1557 y Fi(MPI)p 982 1557 13 2 v 15 w(KEYV)m(AL)p 1154 1557 V 13 w(CREA)m(TE)d Fk(\(in)o(teger\))117 1632 y(IN)155 b Fv(attribute)p 484 1632 14 2 v 18 w(val)352 b Fk(attribute)15 b(v)n(alue)75 1756 y Fu(int)23 b(MPI)p 245 1756 15 2 v 17 w(Attr)p 358 1756 V 17 w(put\(MPI)p 543 1756 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 1756 V 16 w(val\))75 1843 y(MPI)p 150 1843 V 17 w(ATTR)p 263 1843 V 16 w(PUT\(COMM,)g(KEYVAL,)g(ATTRIBUTE)p 925 1843 V 16 w(VAL,)g(IERROR\))170 1899 y(INTEGER)g(COMM,)h(KEYVAL,)e (ATTRIBUTE)p 914 1899 V 16 w(VAL,)i(IERROR)166 1985 y Fx(This)14 b(function)g(stores)f(the)g(stipulated)i(attribute)e(v)m (alue)i Fv(attribute)p 1331 1985 14 2 v 18 w(val)e Fx(for)g(subsequen)o (t)h(retriev)m(al)75 2042 y(b)o(y)c Fv(MPI)p 218 2042 V 16 w(A)l(TTR)p 351 2042 V 17 w(GET)p Fx(.)g(If)h(the)f(v)m(alue)h(is) g(already)f(presen)o(t,)h(then)f(the)h(outcome)e(is)i(as)f(if)h Fv(MPI)p 1627 2042 V 15 w(A)l(TTR)p 1759 2042 V 17 w(DELETE)75 2098 y Fx(w)o(as)17 b(\014rst)h(called)i(to)e(delete)h(the)f(previous)h (v)m(alue)h(\(and)e(the)g(callbac)o(k)i(function)f Fv(delete)p 1635 2098 V 17 w(fn)g Fx(w)o(as)e(exe-)75 2155 y(cuted\),)d(and)h(a)e (new)i(v)m(alue)g(w)o(as)f(next)g(stored.)19 b(The)14 b(call)i(is)e(erroneous)g(if)h(there)f(is)h(no)f(k)o(ey)g(with)h(v)m (alue)75 2211 y Fv(k)o(eyval)p Fx(;)g(in)h(particular)f Fi(MPI)p 558 2211 13 2 v 15 w(KEYV)m(AL)p 730 2211 V 13 w(INV)m(ALID)f Fx(is)i(an)f(erroneous)g(k)o(ey)g(v)m(alue.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 169 177 169 176 bop 75 -100 a Fm(5.7.)29 b(CA)o(CHING)1412 b Fx(169)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(A)l(TTR)p 293 45 V 17 w(GET\(comm)m(,)12 b(k)o(eyval,)i(attribute)p 865 45 V 18 w(val,)h(\015ag\))117 122 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed) h(\(handle\))117 197 y(IN)155 b Fv(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue)f(\(in)o(teger\))117 273 y(OUT)108 b Fv(attribute)p 484 273 V 18 w(val)352 b Fk(attribute)15 b(v)n(alue,)d(unless)j Fi(\015ag)f Fk(=)g(false)117 348 y(OUT)108 b Fv(\015ag)518 b Fb(true)14 b Fk(if)h(an)f(attribute)i(v)n(alue)e(w)o(as)h(extracted;) i Fb(false)d Fk(if)g(no)905 404 y(attribute)h(is)e(asso)q(ciated)i (with)f(the)g(k)o(ey)75 529 y Fu(int)23 b(MPI)p 245 529 15 2 v 17 w(Attr)p 358 529 V 17 w(get\(MPI)p 543 529 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void)g(**attribute)p 1491 529 V 16 w(val,)g(int)h(*flag\))75 615 y(MPI)p 150 615 V 17 w(ATTR)p 263 615 V 16 w(GET\(COMM,)f(KEYVAL,)g(ATTRIBUTE)p 925 615 V 16 w(VAL,)g(FLAG,)g(IERROR\))170 672 y(INTEGER)g(COMM,)h (KEYVAL,)e(ATTRIBUTE)p 914 672 V 16 w(VAL,)i(IERROR)170 728 y(LOGICAL)f(FLAG)166 814 y Fx(Retriev)o(es)c(attribute)g(v)m(alue)h (b)o(y)f(k)o(ey)l(.)30 b(The)19 b(call)h(is)f(erroneous)g(if)g(there)g (is)g(no)g(k)o(ey)g(with)g(v)m(alue)75 871 y Fv(k)o(eyval)p Fx(.)30 b(On)19 b(the)g(other)f(hand,)h(the)g(call)h(is)f(correct)f(if) h(the)g(k)o(ey)f(v)m(alue)i(exists,)g(but)e(no)h(attribute)f(is)75 927 y(attac)o(hed)e(on)h Fu(comm)f Fx(for)g(that)g(k)o(ey;)g(in)i(suc)o (h)f(case,)g(the)f(call)i(returns)f Fu(flag)23 b(=)h(false)p Fx(.)f(In)17 b(particular)75 984 y Fi(MPI)p 152 984 13 2 v 14 w(KEYV)m(AL)p 323 984 V 14 w(INV)m(ALID)d Fx(is)i(an)f (erroneous)g(k)o(ey)g(v)m(alue.)75 1135 y Fv(MPI)p 160 1135 14 2 v 16 w(A)l(TTR)p 293 1135 V 17 w(DELETE\(com)o(m)n(,)c(k)o (eyval\))117 1212 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed)h(\(handle\))117 1287 y(IN)155 b Fv(k)o(eyval)471 b Fk(The)15 b(k)o(ey)e(v)n(alue)h(of)f (the)i(deleted)f(attribute)h(\(in)o(teger\))75 1412 y Fu(int)23 b(MPI)p 245 1412 15 2 v 17 w(Attr)p 358 1412 V 17 w(delete\(MPI)p 615 1412 V 16 w(Comm)g(comm,)g(int)h(keyval\))75 1498 y(MPI)p 150 1498 V 17 w(ATTR)p 263 1498 V 16 w(DELETE\(COMM,)f (KEYVAL,)g(IERROR\))170 1554 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)166 1641 y Fx(Delete)14 b(attribute)g(from)f(cac)o(he)h(b)o(y)f(k)o(ey)l(.) 20 b(This)14 b(function)g(in)o(v)o(ok)o(es)g(the)g(attribute)f(delete)i (function)75 1697 y Fv(delete)p 188 1697 14 2 v 18 w(fn)g Fx(sp)q(eci\014ed)j(when)d(the)h Fv(k)o(eyval)e Fx(w)o(as)h(created.) 166 1754 y(Whenev)o(er)h(a)g(comm)o(unicator)f(is)h(replicated)i(using) e(the)g(function)h Fv(MPI)p 1435 1754 V 16 w(COMM)p 1593 1754 V 16 w(DUP)p Fx(,)f(all)h(call-)75 1810 y(bac)o(k)h(cop)o(y)f (functions)i(for)e(attributes)g(that)g(are)h(curren)o(tly)g(set)f(are)h (in)o(v)o(ok)o(ed)g(\(in)g(arbitrary)f(order\).)75 1867 y(Whenev)o(er)23 b(a)f(comm)o(unicator)g(is)g(deleted)i(using)f(the)g (function)g Fv(MPI)p 1343 1867 V 15 w(COMM)p 1500 1867 V 17 w(FREE)g Fx(all)g(callbac)o(k)75 1923 y(delete)16 b(functions)g(for)f(attributes)g(that)f(are)h(curren)o(tly)h(set)f(are) g(in)o(v)o(ok)o(ed.)75 2045 y Fj(5.7.2)49 b(A)o(ttributes)14 b(Example)189 2131 y Fl(R)n(ationale.)40 b Fx(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 2237 y Fl(A)n(dvic)n(e)22 b(to)h(users.)83 b Fx(This)23 b(example)h(sho)o(ws)e(ho)o(w)g(to)f (write)i(a)f(collectiv)o(e)i(comm)o(unication)189 2293 y(op)q(eration)16 b(that)f(uses)i(cac)o(hing)g(to)e(b)q(e)i(more)f (e\016cien)o(t)h(after)e(the)i(\014rst)e(call.)24 b(The)17 b(co)q(ding)g(st)o(yle)189 2350 y(assumes)11 b(that)h Fv(MPI)f Fx(function)i(results)f(return)g(only)h(error)e(statuses.)18 b(\()p Fl(End)13 b(of)g(advic)n(e)g(to)h(users.)p Fx(\))189 2456 y Fl(A)n(dvic)n(e)h(to)i(implementors.)39 b Fx(\()p Fl(End)16 b(of)g(advic)n(e)h(to)f(implementors.)p Fx(\))147 2562 y Fu(/*)23 b(key)h(for)f(this)g(module's)g(stuff:)g(*/)147 2619 y(static)g(int)g(gop_key)g(=)h(MPI_KEYVAL_INVALID;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 170 178 170 177 bop 75 -100 a Fx(170)123 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)147 45 y Fu(typedef)22 b(struct)147 102 y({)218 158 y(int)i(ref_count;)237 b(/*)23 b(reference)g(count)g(*/)218 214 y(/*)h(other)f(stuff,)g (whatever)g(else)g(we)h(want)f(*/)147 271 y(})g(gop_stuff_type;)147 384 y(Efficient_Collective_Op)d(\(comm,)j(...\))147 440 y(MPI_Comm)f(comm;)147 497 y({)194 553 y(gop_stuff_type)g(*gop_stuff;) 194 610 y(MPI_Group)166 b(group;)194 666 y(int)310 b(foundflag;)194 779 y(MPI_Comm_group\(comm,)21 b(&group\);)194 892 y(if)j(\(gop_key)f (==)g(MPI_KEYVAL_INVALID\))117 b(/*)24 b(get)f(a)h(key)f(on)h(first)f (call)g(ever)h(*/)194 948 y({)242 1005 y(if)g(\()f(!)h (MPI_Attr_keyval_create\()d(gop_stuff_copier,)839 1061 y(gop_stuff_destructor,)839 1118 y(&gop_key,)h(\(void)h(*\)0\)\);)242 1174 y(/*)h(get)f(the)h(key)f(while)g(assigning)g(its)g(copy)h(and)f (delete)g(callback)314 1231 y(behavior.)f(*/)242 1344 y(MPI_Abort)h(\("Insufficient)f(keys)h(available"\);)194 1400 y(})194 1513 y(MPI_Attr_get)g(\(comm,)g(gop_key,)f(&gop_stuff,)h (&foundflag\);)194 1569 y(if)h(\(foundflag\))194 1626 y({)g(/*)g(This)f(module)g(has)g(executed)g(in)h(this)f(group)g (before.)314 1682 y(We)g(will)h(use)f(the)g(cached)g(information)g(*/) 194 1739 y(})194 1795 y(else)194 1852 y({)h(/*)g(This)f(is)h(a)f(group) g(that)h(we)f(have)h(not)f(yet)h(cached)f(anything)f(in.)314 1908 y(We)h(will)h(now)f(do)h(so.)242 1965 y(*/)242 2077 y(/*)g(First,)f(allocate)f(storage)h(for)h(the)f(stuff)g(we)h(want,)314 2134 y(and)f(initialize)g(the)g(reference)g(count)g(*/)242 2247 y(gop_stuff)g(=)g(\(gop_stuff_type)f(*\))i(malloc)f (\(sizeof\(gop_stuff_type\)\);)242 2303 y(if)h(\(gop_stuff)e(==)i (NULL\))f({)h(/*)f(abort)g(on)h(out-of-memory)e(error)h(*/)h(})242 2416 y(gop_stuff)f(->)g(ref_count)g(=)h(1;)242 2529 y(/*)g(Second,)f (fill)g(in)g(*gop_stuff)g(with)g(whatever)g(we)h(want.)314 2586 y(This)f(part)g(isn't)g(shown)h(here)f(*/)242 2698 y(/*)h(Third,)f(store)g(gop_stuff)f(as)i(the)g(attribute)e(value)h(*/) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 171 179 171 178 bop 75 -100 a Fm(5.8.)29 b(F)o(ORMALIZING)11 b(THE)g(LOOSEL)l(Y)i(SYNCHR)o(ONOUS)g(MODEL)e(\(USA)o(GE,)f(SAFETY\))p Fx(171)242 45 y Fu(MPI_Attr_put)22 b(\()i(comm,)f(gop_key,)g (gop_stuff\);)194 102 y(})194 158 y(/*)h(Then,)f(in)h(any)f(case,)g (use)h(contents)f(of)g(*gop_stuff)266 214 y(to)g(do)h(the)g(global)f (op)g(...)h(*/)147 271 y(})147 384 y(/*)f(The)h(following)e(routine)h (is)h(called)f(by)g(MPI)h(when)f(a)h(group)f(is)h(freed)f(*/)147 497 y(gop_stuff_destructor)e(\(comm,)i(keyval,)g(gop_stuff,)f(extra\)) 147 553 y(MPI_Comm)g(comm;)147 610 y(int)h(keyval;)147 666 y(gop_stuff_type)f(*gop_stuff;)147 723 y(void)h(*extra;)147 779 y({)194 835 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h(--) f(programming)g(error)g(*/)g(})194 948 y(/*)h(The)f(group's)g(being)g (freed)h(removes)f(one)g(reference)g(to)g(gop_stuff)g(*/)194 1005 y(gop_stuff)g(->)h(ref_count)e(-=)i(1;)194 1118 y(/*)g(If)f(no)h(references)f(remain,)f(then)i(free)f(the)h(storage)e (*/)194 1174 y(if)i(\(gop_stuff)e(->)i(ref_count)f(==)g(0\))h({)242 1231 y(free\(\(void)f(*\)gop_stuff\);)194 1287 y(})147 1344 y(})147 1456 y(/*)g(The)h(following)e(routine)h(is)h(called)f(by)g (MPI)h(when)f(a)h(group)f(is)h(copied)f(*/)147 1513 y(gop_stuff_copier) e(\(comm,)i(keyval,)g(gop_stuff,)g(extra\))147 1569 y(MPI_Comm)f(comm;) 147 1626 y(int)h(keyval;)147 1682 y(gop_stuff_type)f(*gop_stuff;)147 1739 y(void)h(*extra;)147 1795 y({)194 1852 y(if)h(\(keyval)f(!=)g (gop_key\))g({)h(/*)f(abort)h(--)f(programming)g(error)g(*/)g(})194 1965 y(/*)h(The)f(new)h(group)f(adds)g(one)h(reference)e(to)i(this)f (gop_stuff)g(*/)194 2021 y(gop_stuff)g(->)h(ref_count)e(+=)i(1;)147 2077 y(})75 2221 y Fn(5.8)59 b(F)n(o)n(rmalizing)21 b(the)e(Lo)r(osely) g(Synchronous)g(Mo)r(del)g(\(Usage,)g(Safet)n(y\))75 2322 y Fx(In)i(this)h(section,)g(w)o(e)f(mak)o(e)f(further)h(statemen)o (ts)e(ab)q(out)i(the)g(lo)q(osely)h(sync)o(hronous)f(mo)q(del,)i(with) 75 2379 y(particular)16 b(atten)o(tion)f(to)f(in)o(tra-comm)o (unication.)75 2500 y Fj(5.8.1)49 b(Basic)17 b(Stateme)o(nts)75 2586 y Fx(When)e(a)g(caller)h(passes)e(a)h(comm)o(unicator)f(\(that)g (con)o(tains)h(a)g(con)o(text)f(and)h(group\))f(to)g(a)h(callee,)h (that)75 2643 y(comm)o(unicator)g(m)o(ust)h(b)q(e)g(free)g(of)f(side)i (e\013ects)e(throughout)g(execution)i(of)f(the)g(subprogram:)22 b(there)75 2699 y(should)16 b(b)q(e)f(no)g(activ)o(e)g(op)q(erations)g (on)g(that)f(comm)o(unicator)g(that)g(migh)o(t)h(in)o(v)o(olv)o(e)g (the)g(pro)q(cess.)20 b(This)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 172 180 172 179 bop 75 -100 a Fx(172)118 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fx(pro)o(vides)22 b(one)g(mo)q(del)h(in)f(whic)o(h)h(libraries)g (can)f(b)q(e)h(written,)g(and)f(w)o(ork)f(\\safely)l(.")39 b(F)l(or)21 b(libraries)75 102 y(so)e(designated,)h(the)g(callee)h(has) e(p)q(ermission)h(to)f(do)g(whatev)o(er)g(comm)o(unication)h(it)f(lik)o (es)h(with)g(the)75 158 y(comm)o(unicator,)e(and)h(under)g(the)f(ab)q (o)o(v)o(e)g(guaran)o(tee)g(kno)o(ws)g(that)f(no)h(other)g(comm)o (unications)h(will)75 214 y(in)o(terfere.)36 b(Since)22 b(w)o(e)f(p)q(ermit)g(go)q(o)q(d)f(implemen)o(tations)i(to)e(create)g (new)h(comm)o(unicators)f(without)75 271 y(sync)o(hronization)14 b(\(suc)o(h)g(as)f(b)o(y)g(preallo)q(cated)i(con)o(texts)e(on)g(comm)o (unicators\),)g(this)g(do)q(es)h(not)f(imp)q(ose)75 327 y(a)i(signi\014can)o(t)h(o)o(v)o(erhead.)166 384 y(This)k(form)f(of)g (safet)o(y)f(is)i(analogous)f(to)g(other)g(common)g(computer-science)j (usages,)e(suc)o(h)f(as)75 440 y(passing)c(a)f(descriptor)h(of)f(an)h (arra)o(y)e(to)h(a)g(library)i(routine.)k(The)15 b(library)g(routine)g (has)g(ev)o(ery)f(righ)o(t)h(to)75 497 y(exp)q(ect)h(suc)o(h)f(a)g (descriptor)h(to)f(b)q(e)g(v)m(alid)i(and)f(mo)q(di\014able.)75 618 y Fj(5.8.2)49 b(Mo)q(dels)17 b(of)g(Execution)75 704 y Fx(In)12 b(the)g(lo)q(osely)h(sync)o(hronous)f(mo)q(del,)h (transfer)e(of)h(con)o(trol)f(to)h(a)f Fq(parallel)16 b(pro)q(cedure)c Fx(is)g(e\013ected)g(b)o(y)75 761 y(ha)o(ving)f(eac)o (h)g(executing)g(pro)q(cess)g(in)o(v)o(ok)o(e)g(the)f(pro)q(cedure.)19 b(The)11 b(in)o(v)o(o)q(cation)g(is)g(a)g(collectiv)o(e)h(op)q (eration:)75 817 y(it)i(is)h(executed)g(b)o(y)f(all)h(pro)q(cesses)f (in)h(the)g(execution)g(group,)e(and)i(in)o(v)o(o)q(cations)f(are)g (similarly)i(ordered)75 874 y(at)f(all)h(pro)q(cesses.)k(Ho)o(w)o(ev)o (er,)14 b(the)h(in)o(v)o(o)q(cation)h(need)g(not)f(b)q(e)h(sync)o (hronized.)166 930 y(W)l(e)d(sa)o(y)g(that)g(a)g(parallel)i(pro)q (cedure)f(is)g Fl(active)f Fx(in)h(a)g(pro)q(cess)f(if)h(the)f(pro)q (cess)h(b)q(elongs)g(to)f(a)g(group)75 987 y(that)j(ma)o(y)h(collectiv) o(ely)i(execute)f(the)f(pro)q(cedure,)h(and)f(some)f(mem)o(b)q(er)h(of) g(that)f(group)h(is)g(curren)o(tly)75 1043 y(executing)c(the)f(pro)q (cedure)h(co)q(de.)19 b(If)13 b(a)e(parallel)j(pro)q(cedure)f(is)f (activ)o(e)g(in)h(a)f(pro)q(cess,)g(then)g(this)h(pro)q(cess)75 1099 y(ma)o(y)e(b)q(e)i(receiving)g(messages)e(p)q(ertaining)i(to)f (this)g(pro)q(cedure,)h(ev)o(en)f(if)g(it)g(do)q(es)h(not)e(curren)o (tly)h(execute)75 1156 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 1276 y Fv(Static)h(comm)m(unicato)o(r)d(allo)q(cation)75 1362 y Fx(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o(y)g (p)q(oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(o)q(cation)g(of) f(a)g(parallel)75 1418 y(pro)q(cedure)c(can)f(b)q(e)h(activ)o(e)f(at)f (an)o(y)h(pro)q(cess,)g(and)g(the)g(group)f(of)h(executing)h(pro)q (cesses)f(is)h(\014xed.)23 b(F)l(or)75 1475 y(example,)d(all)g(in)o(v)o (o)q(cations)f(of)g(parallel)h(pro)q(cedures)g(in)o(v)o(olv)o(e)f(all)h (pro)q(cesses,)g(pro)q(cesses)f(are)f(single-)75 1531 y(threaded,)d(and)h(there)f(are)g(no)g(recursiv)o(e)h(in)o(v)o(o)q (cations.)166 1588 y(In)k(suc)o(h)g(a)g(case,)g(a)g(comm)o(unicator)f (can)h(b)q(e)g(statically)h(allo)q(cated)f(to)g(eac)o(h)f(pro)q (cedure.)35 b(The)75 1644 y(static)18 b(allo)q(cation)h(can)f(b)q(e)h (done)g(in)g(a)e(pream)o(ble,)j(as)d(part)h(of)g(initialization)j(co)q (de.)29 b(If)18 b(the)g(parallel)75 1701 y(pro)q(cedures)g(can)f(b)q(e) g(organized)g(in)o(to)g(libraries,)h(so)f(that)f(only)h(one)g(pro)q (cedure)h(of)e(eac)o(h)h(library)h(can)75 1757 y(b)q(e)e(concurren)o (tly)g(activ)o(e)g(in)g(eac)o(h)f(pro)q(cessor,)g(then)h(it)g(is)g (su\016cien)o(t)g(to)f(allo)q(cate)h(one)f(comm)o(unicator)75 1813 y(p)q(er)h(library)l(.)75 1933 y Fv(Dynamic)c(comm)n(unicato)o(r)g (allo)q(cation)75 2019 y Fx(Calls)j(of)g(parallel)h(pro)q(cedures)g (are)e(w)o(ell-nested)i(if)g(a)e(new)h(parallel)h(pro)q(cedure)g(is)f (alw)o(a)o(ys)f(in)o(v)o(ok)o(ed)h(in)75 2076 y(a)j(subset)h(of)e(a)h (group)g(executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 2132 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 2189 y(In)i(suc)o(h)g(a)g(case,)f(a)h(new)g (comm)o(unicator)f(needs)h(to)f(b)q(e)i(dynamically)g(allo)q(cated)g (for)e(eac)o(h)h(new)75 2245 y(in)o(v)o(o)q(cation)11 b(of)f(a)g(parallel)i(pro)q(cedure.)19 b(The)11 b(allo)q(cation)g(is)g (done)g(b)o(y)f(the)h(caller.)19 b(A)11 b(new)f(comm)o(unicator)75 2302 y(can)k(b)q(e)h(generated)g(b)o(y)f(a)g(call)h(to)f Fv(MPI)p 746 2302 14 2 v 15 w(COMM)p 903 2302 V 17 w(DUP)p Fx(,)g(if)g(the)h(callee)g(execution)h(group)e(is)g(iden)o(tical)j(to) 75 2358 y(the)f(caller)h(execution)g(group,)e(or)g(b)o(y)h(a)f(call)i (to)e Fv(MPI)p 1005 2358 V 16 w(COMM)p 1163 2358 V 16 w(SPLIT)g Fx(if)i(the)f(caller)g(execution)h(group)75 2414 y(is)f(split)h(in)o(to)f(sev)o(eral)f(subgroups)h(executing)h (distinct)g(parallel)g(routines.)22 b(The)15 b(new)h(comm)o(unicator)75 2471 y(is)g(passed)f(as)g(an)g(argumen)o(t)g(to)f(the)h(in)o(v)o(ok)o (ed)h(routine.)166 2527 y(The)h(need)h(for)e(generating)h(a)f(new)h (comm)o(unicator)f(at)h(eac)o(h)f(in)o(v)o(o)q(cation)i(can)f(b)q(e)g (alleviated)h(or)75 2584 y(a)o(v)o(oided)e(altogether)e(in)j(some)e (cases:)20 b(If)15 b(the)h(execution)g(group)f(is)h(not)f(split,)h (then)f(one)h(can)f(allo)q(cate)75 2640 y(a)g(stac)o(k)g(of)g(comm)o (unicators)h(in)g(a)f(pream)o(ble,)h(and)g(next)g(manage)f(the)h(stac)o (k)f(in)h(a)f(w)o(a)o(y)g(that)g(mimics)75 2697 y(the)g(stac)o(k)g(of)f (recursiv)o(e)i(calls.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 173 181 173 180 bop 75 -100 a Fm(5.8.)29 b(F)o(ORMALIZING)11 b(THE)g(LOOSEL)l(Y)i(SYNCHR)o(ONOUS)g(MODEL)e(\(USA)o(GE,)f(SAFETY\))p Fx(173)166 45 y(One)17 b(can)f(also)g(tak)o(e)f(adv)m(an)o(tage)g(of)h (the)g(w)o(ell-ordering)h(prop)q(ert)o(y)f(of)f(comm)o(unication)i(to)e (a)o(v)o(oid)75 102 y(confusing)g(caller)g(and)f(callee)h(comm)o (unication,)f(ev)o(en)h(if)f(b)q(oth)g(use)g(the)g(same)f(comm)o (unicator.)19 b(T)l(o)14 b(do)75 158 y(so,)g(one)i(needs)g(to)e(abide)j (b)o(y)e(the)g(follo)o(wing)h(t)o(w)o(o)e(rules:)143 252 y Ft(\017)23 b Fx(messages)13 b(sen)o(t)g(b)q(efore)h(a)f(pro)q (cedure)h(call)h(\(or)e(b)q(efore)h(a)f(return)g(from)g(the)h(pro)q (cedure\))g(are)f(also)189 308 y(receiv)o(ed)j(b)q(efore)f(the)h(matc)o (hing)f(call)h(\(or)f(return\))g(at)f(the)h(receiving)i(end;)143 402 y Ft(\017)23 b Fx(messages)14 b(are)h(alw)o(a)o(ys)g(selected)h(b)o (y)f(source)g(\(no)g(use)h(is)g(made)f(of)g Fi(MPI)p 1430 402 13 2 v 14 w(ANY)p 1529 402 V 15 w(SOURCE)p Fx(\).)75 522 y Fv(The)h(General)f(case)75 608 y Fx(In)22 b(the)g(general)g (case,)h(there)e(ma)o(y)g(b)q(e)h(m)o(ultiple)i(concurren)o(tly)e (activ)o(e)g(in)o(v)o(o)q(cations)g(of)f(the)g(same)75 664 y(parallel)i(pro)q(cedure)g(within)g(the)f(same)f(group;)j(in)o(v)o (o)q(cations)e(ma)o(y)f(not)h(b)q(e)g(w)o(ell-nested.)41 b(A)22 b(new)75 721 y(comm)o(unicator)11 b(needs)i(to)e(b)q(e)i (created)e(for)h(eac)o(h)f(in)o(v)o(o)q(cation.)20 b(It)11 b(is)i(the)f(user's)f(resp)q(onsibilit)o(y)j(to)d(mak)o(e)75 777 y(sure)18 b(that,)f(should)h(t)o(w)o(o)e(distinct)j(parallel)g(pro) q(cedures)f(b)q(e)g(in)o(v)o(ok)o(ed)g(concurren)o(tly)g(on)f(o)o(v)o (erlapping)75 834 y(sets)e(of)g(pro)q(cesses,)g(then)g(comm)o(unicator) g(creation)h(b)q(e)f(prop)q(erly)i(co)q(ordinated.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 174 182 174 181 bop 75 356 a Fo(Chapter)34 b(6)75 564 y Fr(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)75 805 y Fn(6.1)59 b(Intro)r(duction)75 906 y Fx(This)22 b(c)o(hapter)g(discusses)h(the)f Fv(MPI)f Fx(top)q(ology)g(mec)o(hanism.)40 b(A)22 b(top)q(ology)f(is)h(an)g (extra,)g(optional)75 963 y(attribute)14 b(that)g(one)g(can)h(giv)o(e)g (to)e(an)i(in)o(tra-comm)o(unicator;)f(top)q(ologies)g(cannot)h(b)q(e)g (added)g(to)f(in)o(ter-)75 1019 y(comm)o(unicators.)19 b(A)d(top)q(ology)e(can)i(pro)o(vide)f(a)g(con)o(v)o(enien)o(t)h (naming)f(mec)o(hanism)h(for)e(the)i(pro)q(cesses)75 1076 y(of)j(a)f(group)h(\(within)h(a)e(comm)o(unicator\),)h(and)g (additionally)l(,)j(ma)o(y)c(assist)h(the)g(run)o(time)h(system)e(in)75 1132 y(mapping)e(the)f(pro)q(cesses)h(on)o(to)e(hardw)o(are.)166 1188 y(As)19 b(stated)g(in)h(c)o(hapter)f(5,)h(a)f(pro)q(cess)h(group)f (in)h Fv(MPI)e Fx(is)i(a)f(collection)i(of)e Fu(n)g Fx(pro)q(cesses.)33 b(Eac)o(h)75 1245 y(pro)q(cess)17 b(in)h(the)e(group)h(is)g(assigned)g (a)g(rank)f(b)q(et)o(w)o(een)h Fu(0)g Fx(and)g Fu(n-1)p Fx(.)24 b(In)17 b(man)o(y)f(parallel)i(applications)75 1301 y(a)c(linear)h(ranking)g(of)f(pro)q(cesses)h(do)q(es)f(not)g (adequately)h(re\015ect)g(the)f(logical)h(comm)o(unication)g(pattern)75 1358 y(of)21 b(the)g(pro)q(cesses)g(\(whic)o(h)h(is)g(usually)g (determined)g(b)o(y)f(the)h(underlying)h(problem)f(geometry)e(and)75 1414 y(the)f(n)o(umerical)h(algorithm)g(used\).)31 b(Often)20 b(the)f(pro)q(cesses)g(are)g(arranged)g(in)h(top)q(ological)f(patterns) 75 1471 y(suc)o(h)c(as)g(t)o(w)o(o-)e(or)i(three-dimensional)i(grids.)j (More)14 b(generally)l(,)i(the)f(logical)h(pro)q(cess)g(arrangemen)o(t) d(is)75 1527 y(describ)q(ed)19 b(b)o(y)f(a)f(graph.)25 b(In)18 b(this)g(c)o(hapter)f(w)o(e)g(will)i(refer)f(to)e(this)i (logical)g(pro)q(cess)g(arrangemen)o(t)e(as)75 1584 y(the)f(\\virtual)h (top)q(ology)l(.")166 1640 y(A)11 b(clear)g(distinction)i(m)o(ust)e(b)q (e)g(made)g(b)q(et)o(w)o(een)g(the)h(virtual)f(pro)q(cess)g(top)q (ology)g(and)g(the)g(top)q(ology)75 1697 y(of)j(the)h(underlying,)i(ph) o(ysical)f(hardw)o(are.)j(The)c(virtual)g(top)q(ology)f(can)h(b)q(e)h (exploited)g(b)o(y)f(the)f(system)75 1753 y(in)j(the)f(assignmen)o(t)h (of)e(pro)q(cesses)i(to)f(ph)o(ysical)h(pro)q(cessors,)f(if)h(this)g (helps)g(to)f(impro)o(v)o(e)g(the)g(comm)o(u-)75 1810 y(nication)i(p)q(erformance)f(on)g(a)g(giv)o(en)h(mac)o(hine.)26 b(Ho)o(w)17 b(this)g(mapping)h(is)g(done,)f(ho)o(w)o(ev)o(er,)g(is)g (outside)75 1866 y(the)e(scop)q(e)g(of)f Fv(MPI)p Fx(.)g(The)g (description)j(of)d(the)g(virtual)i(top)q(ology)l(,)e(on)h(the)f(other) g(hand,)h(dep)q(ends)h(only)75 1922 y(on)i(the)g(application,)i(and)e (is)h(mac)o(hine-indep)q(endent.)30 b(The)19 b(functions)f(that)g(are)f (prop)q(osed)i(in)g(this)75 1979 y(c)o(hapter)c(deal)h(only)g(with)f (mac)o(hine-indep)q(en)q(den)o(t)j(mapping.)189 2067 y Fl(R)n(ationale.)55 b Fx(Though)18 b(ph)o(ysical)h(mapping)g(is)f (not)f(discussed,)j(the)e(existence)h(of)e(the)h(virtual)189 2123 y(top)q(ology)d(information)i(ma)o(y)e(b)q(e)i(used)f(as)g(advice) h(b)o(y)f(the)g(run)o(time)h(system.)22 b(There)16 b(are)g(w)o(ell-)189 2179 y(kno)o(wn)h(tec)o(hniques)h(for)f(mapping)h(grid/torus)f (structures)g(to)g(hardw)o(are)g(top)q(ologies)h(suc)o(h)f(as)189 2236 y(h)o(yp)q(ercub)q(es)k(or)f(grids.)35 b(F)l(or)20 b(more)g(complicated)h(graph)f(structures)g(go)q(o)q(d)h(heuristics)g (often)189 2292 y(yield)e(nearly)e(optimal)h(results)g([20)o(].)25 b(On)18 b(the)f(other)g(hand,)h(if)g(there)f(is)h(no)f(w)o(a)o(y)f(for) h(the)g(user)189 2349 y(to)12 b(sp)q(ecify)j(the)f(logical)g(pro)q (cess)g(arrangemen)o(t)e(as)h(a)g(\\virtual)h(top)q(ology)l(,")f(a)g (random)g(mapping)189 2405 y(is)k(most)f(lik)o(ely)j(to)d(result.)26 b(On)17 b(some)g(mac)o(hines,)g(this)h(will)g(lead)g(to)e(unnecessary)i (con)o(ten)o(tion)189 2462 y(in)g(the)f(in)o(terconnection)i(net)o(w)o (ork.)25 b(Some)17 b(details)h(ab)q(out)f(predicted)i(and)e(measured)h (p)q(erfor-)189 2518 y(mance)d(impro)o(v)o(emen)o(ts)f(that)g(result)i (from)e(go)q(o)q(d)h(pro)q(cess-to-pro)q(cessor)f(mapping)i(on)f(mo)q (dern)189 2575 y(w)o(ormhole-routing)g(arc)o(hitectures)g(can)h(b)q(e)f (found)h(in)g([10)o(,)f(9].)189 2647 y(Besides)22 b(p)q(ossible)g(p)q (erformance)f(b)q(ene\014ts,)h(the)f(virtual)g(top)q(ology)g(can)g (function)g(as)f(a)h(con-)189 2704 y(v)o(enien)o(t,)16 b(pro)q(cess-naming)h(structure,)f(with)g(tremendous)g(b)q(ene\014ts)h (for)f(program)f(readabilit)o(y)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 175 183 175 182 bop 75 -100 a Fm(6.2.)34 b(VIR)l(TUAL)17 b(TOPOLOGIES)1096 b Fx(175)189 45 y(and)15 b(notational)g(p)q(o)o(w)o(er)g(in)h (message-passing)f(programming.)20 b(\()p Fl(End)15 b(of)h(r)n (ationale.)p Fx(\))75 188 y Fn(6.2)59 b(Virtual)20 b(T)-5 b(op)r(ologies)75 290 y Fx(The)22 b(comm)o(unication)h(pattern)e(of)g (a)h(set)g(of)f(pro)q(cesses)h(can)g(b)q(e)h(represen)o(ted)f(b)o(y)g (a)f(graph.)40 b(The)75 346 y(no)q(des)14 b(stand)f(for)f(the)i(pro)q (cesses,)f(and)g(the)h(edges)f(connect)h(pro)q(cesses)f(that)g(comm)o (unicate)g(with)h(eac)o(h)75 403 y(other.)37 b Fv(MPI)21 b Fx(pro)o(vides)h(message-passing)f(b)q(et)o(w)o(een)h(an)o(y)e(pair)i (of)f(pro)q(cesses)g(in)h(a)f(group.)38 b(There)75 459 y(is)23 b(no)g(requiremen)o(t)g(for)f(op)q(ening)i(a)e(c)o(hannel)i (explicitly)l(.)46 b(Therefore,)24 b(a)e(\\missing)h(link")h(in)g(the) 75 516 y(user-de\014ned)19 b(pro)q(cess)e(graph)g(do)q(es)g(not)g(prev) o(en)o(t)g(the)g(corresp)q(onding)h(pro)q(cesses)g(from)e(exc)o (hanging)75 572 y(messages.)26 b(It)17 b(means)g(rather)g(that)g(this)h (connection)g(is)g(neglected)h(in)f(the)f(virtual)h(top)q(ology)l(.)26 b(This)75 629 y(strategy)21 b(implies)k(that)c(the)i(top)q(ology)f(giv) o(es)h(no)f(con)o(v)o(enien)o(t)h(w)o(a)o(y)f(of)g(naming)g(this)h (path)o(w)o(a)o(y)e(of)75 685 y(comm)o(unication.)28 b(Another)17 b(p)q(ossible)j(consequence)f(is)f(that)f(an)g(automatic)g (mapping)i(to)q(ol)e(\(if)h(one)75 741 y(exists)12 b(for)g(the)g(run)o (time)h(en)o(vironmen)o(t\))f(will)i(not)e(tak)o(e)f(accoun)o(t)h(of)g (this)g(edge)h(when)f(mapping.)20 b(Edges)75 798 y(in)13 b(the)g(comm)o(unication)g(graph)g(are)f(not)g(w)o(eigh)o(ted,)h(so)f (that)g(pro)q(cesses)h(are)f(either)h(simply)h(connected)75 854 y(or)h(not)f(connected)j(at)d(all.)189 961 y Fl(R)n(ationale.)55 b Fx(Exp)q(erience)20 b(with)e(similar)h(tec)o(hniques)g(in)g(P)l(ARMA) o(CS)f([5,)f(8])g(sho)o(w)g(that)h(this)189 1017 y(information)i(is)h (usually)h(su\016cien)o(t)f(for)e(a)h(go)q(o)q(d)g(mapping.)36 b(Additionally)l(,)24 b(a)c(more)g(precise)189 1074 y(sp)q (eci\014cation)e(is)f(more)f(di\016cult)i(for)e(the)g(user)h(to)f(set)g (up,)h(and)g(it)f(w)o(ould)h(mak)o(e)f(the)h(in)o(terface)189 1130 y(functions)f(substan)o(tially)g(more)f(complicated.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))166 1236 y(Sp)q(ecifying)22 b(the)e(virtual)g(top)q(ology)g(in)g(terms)g(of)f(a) g(graph)h(is)g(su\016cien)o(t)h(for)e(all)i(applications.)75 1293 y(Ho)o(w)o(ev)o(er,)12 b(in)i(man)o(y)e(applications)j(the)e (graph)g(structure)g(is)g(regular,)g(and)g(the)g(detailed)i(set-up)e (of)g(the)75 1349 y(graph)d(w)o(ould)i(b)q(e)f(incon)o(v)o(enien)o(t)h (for)e(the)h(user)g(and)g(migh)o(t)g(b)q(e)g(less)g(e\016cien)o(t)h(at) e(run)h(time.)19 b(A)10 b(large)h(frac-)75 1406 y(tion)j(of)g(all)h (parallel)h(applications)g(use)e(pro)q(cess)h(top)q(ologies)g(lik)o(e)g (rings,)f(t)o(w)o(o-)f(or)h(higher-dimensional)75 1462 y(grids,)19 b(or)f(tori.)28 b(These)19 b(structures)f(are)g(completely) h(de\014ned)h(b)o(y)e(the)h(n)o(um)o(b)q(er)f(of)g(dimensions)i(and)75 1519 y(the)15 b(n)o(um)o(b)q(ers)h(of)f(pro)q(cesses)g(in)h(eac)o(h)g (co)q(ordinate)f(direction.)22 b(Also,)15 b(the)g(mapping)h(of)f(grids) h(and)f(tori)75 1575 y(is)h(generally)h(an)f(easier)g(problem)g(then)g (that)f(of)g(general)h(graphs.)21 b(Th)o(us,)16 b(it)f(is)i(desirable)g (to)e(address)75 1631 y(these)g(cases)h(explicitly)l(.)166 1688 y(Pro)q(cess)i(co)q(ordinates)g(in)h(a)f(cartesian)g(structure)g (b)q(egin)h(their)f(n)o(um)o(b)q(ering)h(at)f(0.)28 b(Ro)o(w-ma)s(jor) 75 1744 y(n)o(um)o(b)q(ering)18 b(is)g(alw)o(a)o(ys)e(used)i(for)e(the) h(pro)q(cesses)h(in)f(a)g(cartesian)g(structure.)26 b(This)17 b(means)g(that,)g(for)75 1801 y(example,)i(the)f(relation)g(b)q(et)o(w) o(een)g(group)f(rank)h(and)g(co)q(ordinates)g(for)f(four)g(pro)q (cesses)h(in)h(a)e(\(2)11 b Ft(\002)i Fx(2\))75 1857 y(grid)j(is)f(as)g(follo)o(ws.)206 1949 y(co)q(ord)g(\(0,0\):)48 b(rank)15 b(0)206 2005 y(co)q(ord)g(\(0,1\):)48 b(rank)15 b(1)206 2062 y(co)q(ord)g(\(1,0\):)48 b(rank)15 b(2)206 2118 y(co)q(ord)g(\(1,1\):)48 b(rank)15 b(3)75 2249 y Fn(6.3)59 b(Emb)r(edding)18 b(in)i(MPI)75 2351 y Fx(The)13 b(supp)q(ort)f(for)g(virtual)h(top)q(ologies)g(as)f(de\014ned)h(in)h (this)e(c)o(hapter)h(is)f(consisten)o(t)h(with)g(other)f(parts)f(of)75 2407 y Fv(MPI)p Fx(,)k(and,)h(whenev)o(er)g(p)q(ossible,)h(mak)o(es)f (use)g(of)f(functions)i(that)e(are)h(de\014ned)h(elsewhere.)23 b(T)l(op)q(ology)75 2464 y(information)e(is)g(asso)q(ciated)h(with)f (comm)o(unicators.)36 b(It)21 b(is)h(added)f(to)g(comm)o(unicators)f (using)i(the)75 2520 y(cac)o(hing)16 b(mec)o(hanism)g(describ)q(ed)h (in)f(Chapter)f(5.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 176 184 176 183 bop 75 -100 a Fx(176)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fn(6.4)59 b(Overview)19 b(of)h(the)f(F)n(unctions)75 147 y Fx(The)h(functions)h Fv(MPI)p 459 147 14 2 v 15 w(GRAPH)p 624 147 V 18 w(CREA)l(TE)g Fx(and)f Fv(MPI)p 1007 147 V 16 w(CART)p 1142 147 V 17 w(CREA)l(TE)h Fx(are)e(used)i(to)e(create)g(general)75 203 y(\(graph\))e(virtual)h(top)q(ologies)g(and)g(cartesian)g(top)q (ologies,)g(resp)q(ectiv)o(ely)l(.)29 b(These)18 b(top)q(ology)g (creation)75 259 y(functions)13 b(are)g(collectiv)o(e.)21 b(As)12 b(with)h(other)g(collectiv)o(e)h(calls,)g(the)f(program)f(m)o (ust)g(b)q(e)h(written)g(to)f(w)o(ork)75 316 y(correctly)l(,)j(whether) h(the)f(call)h(sync)o(hronizes)g(or)f(not.)166 372 y(The)21 b(top)q(ology)f(creation)h(functions)g(tak)o(e)f(as)g(input)i(an)e (existing)i(comm)o(unicator)e Fv(comm)p 1795 372 V 10 w(old)p Fx(,)75 429 y(whic)o(h)14 b(de\014nes)h(the)e(set)g(of)g(pro)q (cesses)h(on)g(whic)o(h)g(the)f(top)q(ology)g(is)h(to)f(b)q(e)h(mapp)q (ed.)20 b(A)14 b(new)f(comm)o(uni-)75 485 y(cator)e Fv(comm)p 309 485 V 11 w(top)q(ol)h Fx(is)h(created)f(that)f(carries)h(the)g(top) q(ological)h(structure)f(as)f(cac)o(hed)i(information)f(\(see)75 542 y(Chapter)f(5\).)18 b(In)12 b(analogy)f(to)g(function)i Fv(MPI)p 849 542 V 15 w(COMM)p 1006 542 V 17 w(CREA)l(TE)p Fx(,)f(no)f(cac)o(hed)h(information)g(propagates)75 598 y(from)i Fv(comm)p 303 598 V 11 w(old)h Fx(to)g Fv(comm)p 563 598 V 10 w(top)q(ol)p Fx(.)166 655 y Fv(MPI)p 251 655 V 16 w(CART)p 386 655 V 17 w(CREA)l(TE)j Fx(can)g(b)q(e)g(used)g (to)f(describ)q(e)i(cartesian)f(structures)f(of)g(arbitrary)g(dimen-)75 711 y(sion.)i(F)l(or)12 b(eac)o(h)h(co)q(ordinate)g(direction)g(one)g (sp)q(eci\014es)h(whether)f(the)f(pro)q(cess)h(structure)f(is)h(p)q (erio)q(dic)h(or)75 768 y(not.)19 b(Note)c(that)f(an)h Fp(n)p Fx(-dimensional)j(h)o(yp)q(ercub)q(e)e(is)g(an)e Fp(n)p Fx(-dimensional)k(torus)c(with)i(2)e(pro)q(cesses)h(p)q(er)75 824 y(co)q(ordinate)h(direction.)22 b(Th)o(us,)15 b(sp)q(ecial)i(supp)q (ort)e(for)g(h)o(yp)q(ercub)q(e)i(structures)e(is)h(not)f(necessary)l (.)21 b(The)75 880 y(lo)q(cal)13 b(auxiliary)h(function)f Fv(MPI)p 626 880 V 15 w(DIMS)p 752 880 V 16 w(CREA)l(TE)h Fx(can)e(b)q(e)h(used)f(to)g(compute)g(a)g(balanced)h(distribution)75 937 y(of)i(pro)q(cesses)g(among)g(a)g(giv)o(en)g(n)o(um)o(b)q(er)h(of)f (dimensions.)189 1043 y Fl(R)n(ationale.)26 b Fx(Similar)18 b(functions)g(are)f(con)o(tained)h(in)g(EXPRESS)g([22)o(])f(and)h(P)l (ARMA)o(CS.)f(\()p Fl(End)189 1100 y(of)f(r)n(ationale.)p Fx(\))166 1206 y(The)h(function)g Fv(MPI)p 525 1206 V 16 w(TOPO)p 667 1206 V 17 w(TEST)f Fx(can)h(b)q(e)g(used)g(to)f (inquire)i(ab)q(out)e(the)h(top)q(ology)f(asso)q(ciated)75 1262 y(with)g(a)f(comm)o(unicator.)20 b(The)c(top)q(ological)g (information)f(can)h(b)q(e)g(extracted)f(from)g(the)g(comm)o(unica-)75 1319 y(tor)h(using)h(the)g(functions)h Fv(MPI)p 633 1319 V 15 w(GRAPHDIMS)p 909 1319 V 17 w(GET)f Fx(and)g Fv(MPI)p 1203 1319 V 16 w(GRAPH)p 1369 1319 V 18 w(GET)p Fx(,)f(for)g(general)i (graphs,)75 1375 y(and)e Fv(MPI)p 249 1375 V 15 w(CARTDIM)p 469 1375 V 17 w(GET)f Fx(and)h Fv(MPI)p 760 1375 V 16 w(CART)p 895 1375 V 17 w(GET)p Fx(,)f(for)g(cartesian)g(top)q(ologies.) 21 b(Sev)o(eral)16 b(additional)75 1432 y(functions)f(are)f(pro)o (vided)h(to)f(manipulate)i(cartesian)e(top)q(ologies:)20 b(the)15 b(functions)g Fv(MPI)p 1603 1432 V 16 w(CART)p 1738 1432 V 17 w(RANK)75 1488 y Fx(and)f Fv(MPI)p 247 1488 V 16 w(CART)p 382 1488 V 17 w(COORDS)h Fx(translate)e(cartesian)h (co)q(ordinates)f(in)o(to)h(a)f(group)h(rank,)f(and)h(vice-v)o(ersa;)75 1545 y(the)20 b(function)h Fv(MPI)p 426 1545 V 16 w(CART)p 561 1545 V 17 w(SUB)g Fx(can)f(b)q(e)h(used)g(to)f(extract)f(a)h (cartesian)g(subspace)h(\(analogous)f(to)75 1601 y Fv(MPI)p 160 1601 V 16 w(COMM)p 318 1601 V 16 w(SPLIT)p Fx(\).)e(The)i(function) g Fv(MPI)p 868 1601 V 16 w(CART)p 1003 1601 V 17 w(SHIFT)f Fx(pro)o(vides)g(the)h(information)f(needed)i(to)75 1658 y(comm)o(unicate)46 b(with)h(neigh)o(b)q(ors)f(in)h(a)f(cartesian)g (dimension.)114 b(The)47 b(t)o(w)o(o)d(functions)75 1714 y Fv(MPI)p 160 1714 V 16 w(GRAPH)p 326 1714 V 17 w(NEIGHBORS)p 594 1714 V 19 w(COUNT)19 b Fx(and)h Fv(MPI)p 963 1714 V 15 w(GRAPH)p 1128 1714 V 18 w(NEIGHBORS)h Fx(can)e(b)q(e)h(used)f(to) g(extract)75 1770 y(the)h(neigh)o(b)q(ors)g(of)f(a)g(no)q(de)h(in)h(a)e (graph.)33 b(The)19 b(function)i Fv(MPI)p 1204 1770 V 16 w(CART)p 1339 1770 V 17 w(SUB)f Fx(is)g(collectiv)o(e)h(o)o(v)o(er)e (the)75 1827 y(input)d(comm)o(unicator's)f(group;)f(all)i(other)f (functions)h(are)f(lo)q(cal.)166 1883 y(Tw)o(o)c(additional)i (functions,)g Fv(MPI)p 767 1883 V 16 w(GRAPH)p 933 1883 V 18 w(MAP)f Fx(and)g Fv(MPI)p 1229 1883 V 16 w(CART)p 1364 1883 V 17 w(MAP)g Fx(are)g(presen)o(ted)g(in)h(the)75 1940 y(last)f(section.)19 b(In)12 b(general)g(these)g(functions)h(are)e (not)g(called)i(b)o(y)f(the)g(user)f(directly)l(.)21 b(Ho)o(w)o(ev)o(er,)11 b(together)75 1996 y(with)h(the)f(comm)o (unicator)g(manipulation)i(functions)f(presen)o(ted)g(in)g(Chapter)f (5,)g(they)h(are)f(su\016cien)o(t)h(to)75 2053 y(implemen)o(t)k(all)h (other)d(top)q(ology)h(functions.)21 b(The)15 b(last)g(section)h (outlines)h(suc)o(h)e(an)g(implemen)o(tation.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 177 185 177 184 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fx(177)75 45 y Fn(6.5)59 b(T)-5 b(op)r(ology)19 b(Constructo)n(rs)75 148 y Fj(6.5.1)49 b(Ca)o(rtesian)16 b(Constructo)o(r)75 329 y Fv(MPI)p 160 329 14 2 v 16 w(CART)p 295 329 V 17 w(CREA)l(TE\(comm)p 617 329 V 11 w(old,)f(ndims,)e(dims,)g(p)q(erio)q (ds,)j(reo)o(rder,)e(comm)p 1412 329 V 10 w(ca)o(rt\))117 406 y Fk(IN)171 b Fv(comm)p 454 406 V 10 w(old)383 b Fk(input)14 b(comm)o(unicator)d(\(handle\))117 481 y(IN)171 b Fv(ndims)458 b Fk(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i (cartesian)g(grid)g(\(in)o(teger\))117 556 y(IN)171 b Fv(dims)481 b Fk(in)o(teger)19 b(arra)o(y)f(of)f(size)i Fi(ndims)d Fk(sp)q(ecifying)i(the)g(n)o(um)o(b)q(er)g(of)905 612 y(pro)q(cesses)f(in)c(eac)o(h)i(dimension)117 688 y(IN)171 b Fv(p)q(erio)q(ds)436 b Fk(logical)12 b(arra)o(y)h(of)g(size) h Fi(ndims)d Fk(sp)q(ecifying)i(whether)h(the)g(grid)905 744 y(is)g(p)q(erio)q(dic)g(\()p Fi(true)p Fk(\))h(or)f(not)g(\()p Fi(false)p Fk(\))g(in)g(eac)o(h)g(dimension)117 819 y(IN)171 b Fv(reo)o(rder)438 b Fk(ranking)10 b(ma)o(y)g(b)q(e)h(reordered)i(\()p Fi(true)p Fk(\))f(or)f(not)g(\()p Fi(false)p Fk(\))g(\(logical\))117 894 y(OUT)124 b Fv(comm)p 454 894 V 10 w(ca)o(rt)367 b Fk(comm)o(unicator)11 b(with)j(new)g(cartesian)h(top)q(ology)d (\(handle\))75 1019 y Fu(int)23 b(MPI)p 245 1019 15 2 v 17 w(Cart)p 358 1019 V 17 w(create\(MPI)p 615 1019 V 16 w(Comm)g(comm)p 846 1019 V 17 w(old,)g(int)g(ndims,)g(int)h (*dims,)f(int)g(*periods,)393 1075 y(int)h(reorder,)e(MPI)p 778 1075 V 17 w(Comm)h(*comm)p 1034 1075 V 17 w(cart\))75 1162 y(MPI)p 150 1162 V 17 w(CART)p 263 1162 V 16 w(CREATE\(COMM)p 543 1162 V 16 w(OLD,)g(NDIMS,)g(DIMS,)h(PERIODS,)e(REORDER,)h(COMM)p 1514 1162 V 17 w(CART,)g(IERROR\))170 1218 y(INTEGER)g(COMM)p 460 1218 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(COMM)p 1074 1218 V 17 w(CART,)g(IERROR)170 1275 y(LOGICAL)g(PERIODS\(*\),)g (REORDER)166 1361 y Fv(MPI)p 251 1361 14 2 v 16 w(CART)p 386 1361 V 17 w(CREA)l(TE)18 b Fx(returns)f(a)g(handle)i(to)e(a)g(new)g (comm)o(unicator)g(to)g(whic)o(h)h(the)g(cartesian)75 1418 y(top)q(ology)13 b(information)g(is)g(attac)o(hed.)19 b(If)13 b Fv(reo)o(rder)f(=)h(false)h Fx(then)f(the)g(rank)g(of)f(eac)o (h)h(pro)q(cess)h(in)g(the)f(new)75 1474 y(group)g(is)g(iden)o(tical)i (to)d(its)h(rank)g(in)h(the)f(old)h(group.)k(Otherwise,)c(the)f (function)h(ma)o(y)e(reorder)h(the)g(pro-)75 1530 y(cesses)h(\(p)q (ossibly)i(so)d(as)h(to)g(c)o(ho)q(ose)g(a)f(go)q(o)q(d)h(em)o(b)q (edding)i(of)e(the)g(virtual)h(top)q(ology)e(on)o(to)g(the)i(ph)o (ysical)75 1587 y(mac)o(hine\).)k(If)12 b(the)g(total)f(size)i(of)e (the)h(cartesian)g(grid)g(is)g(smaller)h(than)f(the)f(size)i(of)f(the)f (group)h(of)f Fv(comm)m Fx(,)75 1643 y(then)k(some)g(pro)q(cesses)h (are)f(returned)g Fi(MPI)p 826 1643 13 2 v 15 w(COMM)p 971 1643 V 14 w(NULL)p Fx(,)h(in)g(analogy)f(to)f Fv(MPI)p 1475 1643 14 2 v 16 w(COMM)p 1633 1643 V 17 w(SPLIT)p Fx(.)g(The)75 1700 y(call)i(is)g(erroneous)f(if)h(it)f(sp)q(eci\014es)i (a)e(grid)h(that)e(is)i(larger)f(than)g(the)g(group)g(size.)75 1822 y Fj(6.5.2)49 b(Ca)o(rtesian)16 b(Convenience)g(F)o(unction:)21 b(MPI)p 996 1822 15 2 v 18 w(DIMS)p 1131 1822 V 18 w(CREA)l(TE)75 1907 y Fx(F)l(or)10 b(cartesian)i(top)q(ologies,)f(the)h(function)f Fv(MPI)p 896 1907 14 2 v 16 w(DIMS)p 1023 1907 V 16 w(CREA)l(TE)h Fx(helps)h(the)e(user)g(to)f(select)i(a)f(balanced)75 1964 y(the)17 b(n)o(um)o(b)q(ers)h(of)f(pro)q(cesses)h(p)q(er)f(co)q (ordinate)h(direction,)h(dep)q(ending)g(on)f(the)f(n)o(um)o(b)q(er)h (of)f(pro)q(cesses)75 2020 y(in)k(the)f(group)g(to)f(b)q(e)i(balanced)h (and)e(optional)h(constrain)o(ts)e(that)h(can)g(b)q(e)h(sp)q(eci\014ed) h(b)o(y)e(the)g(user.)75 2077 y(One)f(use)g(is)h(to)e(partition)h(all)g (the)g(pro)q(cesses)g(\(the)f(size)i(of)e Fi(MPI)p 1227 2077 13 2 v 15 w(COMM)p 1372 2077 V 14 w(W)o(ORLD)p Fx('s)g(group\))g (in)o(to)h(an)75 2133 y Fp(n)p Fx(-dimensional)f(top)q(ology)l(.)75 2284 y Fv(MPI)p 160 2284 14 2 v 16 w(DIMS)p 287 2284 V 16 w(CREA)l(TE\(nno)q(des,)f(ndims,)c(dims\))117 2361 y Fk(IN)171 b Fv(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(a)f(grid)h(\(in)o(teger\))117 2436 y(IN)171 b Fv(ndims)458 b Fk(n)o(um)o(b)q(er)13 b(of)h(cartesian)g(dimensions)f(\(in)o(teger\))117 2512 y(INOUT)78 b Fv(dims)481 b Fk(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fb(ndims)e Fk(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 2568 y(no)q(des)e(in)e(eac)o(h)i(dimension)75 2692 y Fu(int)23 b(MPI)p 245 2692 15 2 v 17 w(Dims)p 358 2692 V 17 w(create\(int)f(nnodes,)h(int)h(ndims,)f(int)g(*dims\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 178 186 178 185 bop 75 -100 a Fx(178)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(DIMS)p 263 45 V 16 w(CREATE\(NNODES,)22 b(NDIMS,)h(DIMS,)h (IERROR\))170 102 y(INTEGER)f(NNODES,)g(NDIMS,)g(DIMS\(*\),)g(IERROR) 166 188 y Fx(The)13 b(en)o(tries)h(in)f(the)g(arra)o(y)f Fv(dims)f Fx(are)i(set)f(to)h(describ)q(e)i(a)d(cartesian)h(grid)h (with)f Fv(ndims)e Fx(dimensions)75 244 y(and)i(a)g(total)g(of)f Fv(nno)q(des)k Fx(no)q(des.)j(The)14 b(dimensions)h(are)d(set)h(to)g(b) q(e)h(as)e(close)i(to)f(eac)o(h)g(other)g(as)g(p)q(ossible,)75 301 y(using)i(an)g(appropriate)g(divisibili)q(t)o(y)i(algorithm.)i(The) c(caller)h(ma)o(y)e(further)g(constrain)h(the)g(op)q(eration)75 357 y(of)e(this)h(routine)g(b)o(y)g(sp)q(ecifying)h(elemen)o(ts)f(of)f (arra)o(y)g Fu(dims)p Fx(.)19 b(If)13 b Fu(dims[i])g Fx(is)h(set)f(to)g(a)g(p)q(ositiv)o(e)i(n)o(um)o(b)q(er,)75 414 y(the)i(routine)g(will)h(not)e(mo)q(dify)i(the)e(n)o(um)o(b)q(er)h (of)f(no)q(des)i(in)f(dimension)h Fu(i)p Fx(;)f(only)g(those)g(en)o (tries)g(where)75 470 y Fu(dims[i])23 b(=)h(0)15 b Fx(are)g(mo)q (di\014ed)h(b)o(y)f(the)h(call.)166 527 y(Negativ)o(e)e(input)i(v)m (alues)f(of)f Fu(dims[i])f Fx(are)h(erroneous.)20 b(An)15 b(error)e(will)j(o)q(ccur)f(if)g Fu(nnodes)e Fx(is)i(not)f(a)75 583 y(m)o(ultiple)j(of)366 543 y Fe(Y)305 637 y Ff(i;dims)p Fg([)p Ff(i)p Fg(])p Fh(6)p Fg(=0)489 583 y Fp(dims)p Fx([)p Fp(i)p Fx(].)166 685 y(F)l(or)g Fu(dims[i])g Fx(set)g(b)o(y)g (the)h(call,)h Fu(dims[i])e Fx(will)i(b)q(e)f(ordered)g(in)h (non-increasing)g(order.)27 b(Arra)o(y)75 741 y Fu(dims)20 b Fx(is)i(suitable)g(for)f(use)g(as)g(input)h(to)e(routine)i Fv(MPI)p 1076 741 14 2 v 16 w(CART)p 1211 741 V 17 w(CREA)l(TE)p Fx(.)g Fv(MPI)p 1514 741 V 15 w(DIMS)p 1640 741 V 16 w(CREA)l(TE)g Fx(is)75 798 y(lo)q(cal.)75 1055 y Fq(Example)c(6.1)p 380 873 1247 2 v 379 930 2 57 v 405 913 a Fu(dims)p 634 930 V 159 w Fx(function)e(call)p 1303 930 V 424 w Fu(dims)p 1626 930 V 379 986 V 405 969 a Fx(b)q(efore)g(call)p 634 986 V 1303 986 V 720 w(on)f(return)p 1626 986 V 380 988 1247 2 v 379 1044 2 57 v 405 1027 a(\(0,0\))p 634 1044 V 160 w Fv(MPI)p 745 1027 14 2 v 16 w(DIMS)p 872 1027 V 15 w(CREA)l(TE\(6,)h(2,)e(dims\))p 1303 1044 2 57 v 48 w Fx(\(3,2\))p 1626 1044 V 379 1101 V 405 1084 a(\(0,0\))p 634 1101 V 160 w Fv(MPI)p 745 1084 14 2 v 16 w(DIMS)p 872 1084 V 15 w(CREA)l(TE\(7,)i(2,)e(dims\))p 1303 1101 2 57 v 48 w Fx(\(7,1\))p 1626 1101 V 379 1157 V 405 1140 a(\(0,3,0\))p 634 1157 V 124 w Fv(MPI)p 745 1140 14 2 v 16 w(DIMS)p 872 1140 V 15 w(CREA)l(TE\(6,)i(3,)e(dims\))p 1303 1157 2 57 v 48 w Fx(\(2,3,1\))p 1626 1157 V 379 1214 V 405 1197 a(\(0,3,0\))p 634 1214 V 124 w Fv(MPI)p 745 1197 14 2 v 16 w(DIMS)p 872 1197 V 15 w(CREA)l(TE\(7,)i(3,)e (dims\))p 1303 1214 2 57 v 48 w Fx(erroneous)h(call)p 1626 1214 V 380 1215 1247 2 v 75 1399 a Fj(6.5.3)49 b(General)16 b(\(Graph\))f(Constructo)o(r)75 1579 y Fv(MPI)p 160 1579 14 2 v 16 w(GRAPH)p 326 1579 V 17 w(CREA)l(TE\(comm)p 649 1579 V 12 w(old,)g(nno)q(des,)h(index,)g(edges,)g(reo)o(rder,)e (comm)p 1444 1579 V 10 w(graph\))117 1656 y Fk(IN)171 b Fv(comm)p 454 1656 V 10 w(old)383 b Fk(input)14 b(comm)o(unicator)d (without)j(top)q(ology)e(\(handle\))117 1731 y(IN)171 b Fv(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g (graph)g(\(in)o(teger\))117 1806 y(IN)171 b Fv(index)473 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i(describing)f(no)q(de)h(degrees)g (\(see)g(b)q(elo)o(w\))117 1882 y(IN)171 b Fv(edges)468 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i(describing)f(graph)g(edges)h (\(see)g(b)q(elo)o(w\))117 1957 y(IN)171 b Fv(reo)o(rder)438 b Fk(ranking)10 b(ma)o(y)g(b)q(e)h(reordered)i(\()p Fi(true)p Fk(\))f(or)f(not)g(\()p Fi(false)p Fk(\))g(\(logical\))117 2032 y(OUT)124 b Fv(comm)p 454 2032 V 10 w(graph)333 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q(ology)e(added)i (\(handle\))75 2156 y Fu(int)23 b(MPI)p 245 2156 15 2 v 17 w(Graph)p 382 2156 V 17 w(create\(MPI)p 639 2156 V 16 w(Comm)g(comm)p 870 2156 V 16 w(old,)h(int)f(nnodes,)g(int)h (*index,)f(int)g(*edges,)393 2213 y(int)h(reorder,)e(MPI)p 778 2213 V 17 w(Comm)h(*comm)p 1034 2213 V 17 w(graph\))75 2299 y(MPI)p 150 2299 V 17 w(GRAPH)p 287 2299 V 16 w(CREATE\(COMM)p 567 2299 V 16 w(OLD,)g(NNODES,)g(INDEX,)g(EDGES,)g(REORDER,)g(COMM)p 1538 2299 V 17 w(GRAPH,)393 2356 y(IERROR\))170 2412 y(INTEGER)g(COMM)p 460 2412 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g (EDGES\(*\),)f(COMM)p 1360 2412 V 17 w(GRAPH,)h(IERROR)170 2469 y(LOGICAL)g(REORDER)166 2555 y Fv(MPI)p 251 2555 14 2 v 16 w(GRAPH)p 417 2555 V 17 w(CREA)l(TE)f Fx(returns)e(a)g (handle)h(to)f(a)g(new)g(comm)o(unicator)g(to)g(whic)o(h)h(the)f(graph) 75 2611 y(top)q(ology)e(information)h(is)g(attac)o(hed.)30 b(If)19 b Fv(reo)o(rder)f(=)h(false)g Fx(then)g(the)g(rank)f(of)h(eac)o (h)f(pro)q(cess)h(in)h(the)75 2668 y(new)13 b(group)g(is)h(iden)o (tical)h(to)d(its)i(rank)f(in)h(the)f(old)g(group.)19 b(Otherwise,)14 b(the)g(function)g(ma)o(y)e(reorder)h(the)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 179 187 179 186 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fx(179)75 45 y(pro)q(cesses.)25 b(If)17 b(the)g(size,)h Fv(nno)q(des)p Fx(,)h(of)e(the)g(graph)f(is)i(smaller)f(than)g(the)g (size)h(of)e(the)h(group)g(of)f Fv(comm)m Fx(,)75 102 y(then)f(some)f(pro)q(cesses)h(are)f(returned)g Fi(MPI)p 822 102 13 2 v 15 w(COMM)p 967 102 V 14 w(NULL)p Fx(,)h(in)h(analogy)e (to)f Fv(MPI)p 1468 102 14 2 v 16 w(CART)p 1603 102 V 17 w(CREA)l(TE)j Fx(and)75 158 y Fv(MPI)p 160 158 V 16 w(COMM)p 318 158 V 16 w(SPLIT)p Fx(.)c(The)h(call)h(is)g(erroneous)e (if)i(it)f(sp)q(eci\014es)h(a)f(graph)f(that)h(is)g(larger)g(than)f (the)h(group)75 214 y(size)j(of)f(the)g(input)h(comm)o(unicator.)166 271 y(The)d(three)h(parameters)e Fv(nno)q(des,)j(index)f Fx(and)g Fv(edges)g Fx(de\014ne)g(the)f(graph)g(structure.)19 b Fv(nno)q(des)c Fx(is)f(the)75 327 y(n)o(um)o(b)q(er)f(of)g(no)q(des)g (of)f(the)h(graph.)19 b(The)13 b(no)q(des)h(are)e(n)o(um)o(b)q(ered)i (from)e Fu(0)h Fx(to)f Fu(nnodes-1)p Fx(.)18 b(The)13 b Fu(i)p Fx(th)f(en)o(try)75 384 y(of)k(arra)o(y)e Fv(index)j Fx(stores)f(the)g(total)f(n)o(um)o(b)q(er)h(of)g(neigh)o(b)q(ors)h(of)e (the)h(\014rst)g Fu(i)g Fx(graph)g(no)q(des.)22 b(The)17 b(lists)f(of)75 440 y(neigh)o(b)q(ors)d(of)g(no)q(des)g Fu(0,)24 b(1,)f Fp(:)8 b(:)g(:)e Fu(,)24 b(nnodes-1)11 b Fx(are)h(stored)h(in)g(consecutiv)o(e)h(lo)q(cations)f(in)h(arra)o(y) d Fv(edges)p Fx(.)75 497 y(The)16 b(arra)o(y)e Fv(edges)i Fx(is)g(a)f(\015attened)h(represen)o(tation)f(of)g(the)h(edge)f(lists.) 21 b(The)16 b(total)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)75 553 y(in)h Fv(index)g Fx(is)g Fv(nno)q(des)h Fx(and)e(the)g(total)g(n)o (um)o(b)q(er)g(of)g(en)o(tries)h(in)g Fv(edges)g Fx(is)f(equal)h(to)f (the)g(n)o(um)o(b)q(er)h(of)e(graph)75 610 y(edges.)166 666 y(The)19 b(de\014nitions)h(of)e(the)h(argumen)o(ts)f Fu(nnodes)p Fx(,)g Fu(index)p Fx(,)h(and)g Fu(edges)f Fx(are)g(illustrated)i(with)f(the)75 723 y(follo)o(wing)d(simple)h (example.)75 827 y Fq(Example)h(6.2)k Fx(Assume)g(there)g(are)g(four)f (pro)q(cesses)i(0,)g(1,)f(2,)h(3)f(with)g(the)g(follo)o(wing)h (adjacency)75 884 y(matrix:)p 181 929 433 2 v 180 985 2 57 v 206 968 a(pro)q(cess)p 373 985 V 50 w(neigh)o(b)q(ors)p 613 985 V 181 987 433 2 v 180 1043 2 57 v 266 1026 a(0)p 373 1043 V 110 w(1,)15 b(3)p 613 1043 V 180 1100 V 266 1083 a(1)p 373 1100 V 110 w(0)p 613 1100 V 180 1156 V 266 1139 a(2)p 373 1156 V 110 w(3)p 613 1156 V 180 1213 V 266 1196 a(3)p 373 1213 V 110 w(0,)g(2)p 613 1213 V 181 1214 433 2 v 166 1289 a(Then,)g(the)h(input)g(argumen)o(ts)e(are:) 206 1381 y(nno)q(des)i(=)50 b(4)206 1437 y(index)16 b(=)81 b(2,)15 b(3,)f(4,)h(6)206 1494 y(edges)g(=)82 b(1,)15 b(3,)f(0,)h(3,)f(0,)h(2)166 1586 y(Th)o(us,)h(in)h(C,)e Fu(index[0])g Fx(is)i(the)f(degree)g(of)g(no)q(de)h(zero,)f(and)g Fu(index[i])23 b(-)h(index[i-1])14 b Fx(is)j(the)75 1642 y(degree)22 b(of)g(no)q(de)g Fu(i,)i(i=1,)f Fp(:)8 b(:)g(:)e Fu(,)23 b(nnodes-1)p Fx(;)h(the)e(list)h(of)e(neigh)o(b)q(ors)i(of)e (no)q(de)i(zero)e(is)i(stored)e(in)75 1699 y Fu(edges[j])p Fx(,)14 b(for)h Fu(0)e Ft(\024)h Fu(j)f Ft(\024)h Fu(index)p Fx([)p Fu(0)p Fx(])9 b Ft(\000)h Fu(1)16 b Fx(and)g(the)f(list)i(of)e (neigh)o(b)q(ors)h(of)f(no)q(de)i Fu(i)p Fx(,)e Fu(i)e Fp(>)h Fu(0)p Fx(,)h(is)h(stored)f(in)75 1755 y Fu(edges[j])p Fx(,)f Fu(index)p Fx([)p Fu(i)9 b Ft(\000)h Fu(1)p Fx(])i Ft(\024)h Fu(j)g Ft(\024)g Fu(index)p Fx([)p Fu(i)p Fx(])8 b Ft(\000)j Fu(1)p Fx(.)166 1811 y(In)17 b(F)l(ortran,)f Fu(index\(1\))f Fx(is)j(the)e(degree)i(of)e(no)q(de)h(zero,)g(and)g Fu(index\(i+1\))22 b(-)i(index\(i\))16 b Fx(is)h(the)75 1868 y(degree)22 b(of)g(no)q(de)g Fu(i,)i(i=1,)f Fp(:)8 b(:)g(:)e Fu(,)23 b(nnodes-1)p Fx(;)h(the)e(list)h(of)e(neigh)o(b)q (ors)i(of)e(no)q(de)i(zero)e(is)i(stored)e(in)75 1924 y Fu(edges\(j\))p Fx(,)d(for)g Fu(1)f Ft(\024)i Fu(j)f Ft(\024)g Fu(index)p Fx(\()p Fu(1)p Fx(\))f(and)i(the)f(list)h(of)f (neigh)o(b)q(ors)h(of)f(no)q(de)i Fu(i)p Fx(,)e Fu(i)g Fp(>)h Fu(0)p Fx(,)f(is)h(stored)f(in)75 1981 y Fu(edges\(j\))p Fx(,)c Fu(index)p Fx(\()p Fu(i)p Fx(\))8 b(+)j Fu(1)h Ft(\024)h Fu(j)g Ft(\024)g Fu(index)p Fx(\()p Fu(i)c Fx(+)h Fu(1)p Fx(\).)189 2086 y Fl(A)n(dvic)n(e)17 b(to)i (implementors.)56 b Fx(The)19 b(follo)o(wing)f(top)q(ology)g (information)g(is)g(lik)o(ely)i(to)d(b)q(e)h(stored)189 2142 y(with)d(a)g(comm)o(unicator:)243 2236 y Ft(\017)23 b Fx(T)o(yp)q(e)15 b(of)g(top)q(ology)g(\(cartesian/graph\),)243 2308 y Ft(\017)23 b Fx(F)l(or)14 b(a)h(cartesian)g(top)q(ology:)316 2381 y(1.)22 b Fu(ndims)14 b Fx(\(n)o(um)o(b)q(er)h(of)g(dimensions\),) 316 2445 y(2.)22 b Fu(dims)14 b Fx(\(n)o(um)o(b)q(ers)h(of)g(pro)q (cesses)h(p)q(er)f(co)q(ordinate)h(direction\),)316 2510 y(3.)22 b Fu(periods)14 b Fx(\(p)q(erio)q(dicit)o(y)j(information\),) 316 2574 y(4.)22 b Fu(own_position)12 b Fx(\(o)o(wn)h(p)q(osition)i(in) f(grid,)h(could)f(also)g(b)q(e)h(computed)f(from)f(rank)h(and)374 2631 y(dims\))243 2704 y Ft(\017)23 b Fx(F)l(or)14 b(a)h(graph)g(top)q (ology:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 180 188 180 187 bop 75 -100 a Fx(180)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)316 45 y Fx(1.)22 b Fu(index)p Fx(,)316 110 y(2.)g Fu(edges)p Fx(,)289 183 y(whic)o(h)16 b(are)f(the)g(v)o(ectors)f(de\014ning)j(the)e(graph)g(structure.)189 277 y(F)l(or)k(a)g(graph)h(structure)f(the)h(n)o(um)o(b)q(er)g(of)f(no) q(des)i(is)f(equal)h(to)e(the)h(n)o(um)o(b)q(er)g(of)f(pro)q(cesses)h (in)189 333 y(the)h(group.)36 b(Therefore,)22 b(the)f(n)o(um)o(b)q(er)g (of)g(no)q(des)g(do)q(es)g(not)g(ha)o(v)o(e)f(to)g(b)q(e)i(stored)e (explicitly)l(.)189 390 y(An)c(additional)i(zero)f(en)o(try)f(at)f(the) i(start)e(of)h(arra)o(y)f Fv(index)j Fx(simpli\014es)g(access)f(to)f (the)g(top)q(ology)189 446 y(information.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 568 y Fj(6.5.4)49 b(T)l(op)q(ology)19 b(inquiry)e(functions)75 654 y Fx(If)11 b(a)f(top)q(ology)g(has)h(b)q(een)g(de\014ned)h(with)f (one)g(of)f(the)g(ab)q(o)o(v)o(e)g(functions,)i(then)f(the)g(top)q (ology)f(information)75 710 y(can)15 b(b)q(e)h(lo)q(ok)o(ed)g(up)g (using)f(inquiry)i(functions.)k(They)15 b(all)h(are)f(lo)q(cal)i (calls.)75 861 y Fv(MPI)p 160 861 14 2 v 16 w(TOPO)p 302 861 V 17 w(TEST\(comm)m(,)12 b(status\))117 938 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))117 1013 y(OUT)124 b Fv(status)460 b Fk(top)q(ology)13 b(t)o(yp)q(e)h(of)g (comm)o(uni)o(cator)e Fb(comm)h Fk(\(c)o(hoice\))75 1138 y Fu(int)23 b(MPI)p 245 1138 15 2 v 17 w(Topo)p 358 1138 V 17 w(test\(MPI)p 567 1138 V 16 w(Comm)g(comm,)g(int)h(*status\))75 1224 y(MPI)p 150 1224 V 17 w(TOPO)p 263 1224 V 16 w(TEST\(COMM,)f (STATUS,)g(IERROR\))170 1281 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)166 1367 y Fx(The)h(function)g Fv(MPI)p 537 1367 14 2 v 16 w(TOPO)p 679 1367 V 17 w(TEST)g Fx(returns)f(the)h(t)o(yp)q(e)g(of)f (top)q(ology)g(that)g(is)h(assigned)g(to)f(a)75 1424 y(comm)o(unicator.)166 1480 y(The)15 b(output)g(v)m(alue)i Fu(status)d Fx(is)i(one)f(of)g(the)g(follo)o(wing:)117 1584 y Fi(MPI)p 194 1584 13 2 v 14 w(GRAPH)602 b Fx(graph)15 b(top)q(ology)117 1638 y Fi(MPI)p 194 1638 V 14 w(CART)631 b Fx(cartesian)15 b(top)q(ology)117 1693 y Fi(MPI)p 194 1693 V 14 w(UNDEFINED)508 b Fx(no)15 b(top)q(ology)75 1870 y Fv(MPI)p 160 1870 14 2 v 16 w(GRAPHDIMS)p 437 1870 V 17 w(GET\(comm)m(,)d(nno)q(des,)17 b(nedges\))117 1947 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)7 b(for)i(group)h(with)f(graph)h(structure)i(\(handle\))117 2022 y(OUT)124 b Fv(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(graph)g(\(in)o(teger\))117 2097 y(OUT)124 b Fv(nedges)445 b Fk(n)o(um)o(b)q(er)13 b(of)h(edges)h(in)e (graph)h(\(in)o(teger\))75 2222 y Fu(int)23 b(MPI)p 245 2222 15 2 v 17 w(Graphdims)p 478 2222 V 16 w(get\(MPI)p 662 2222 V 16 w(Comm)h(comm,)f(int)g(*nnodes,)g(int)h(*nedges\))75 2308 y(MPI)p 150 2308 V 17 w(GRAPHDIMS)p 383 2308 V 16 w(GET\(COMM,)e(NNODES,)h(NEDGES,)g(IERROR\))170 2365 y(INTEGER)g(COMM,)h(NNODES,)e(NEDGES,)h(IERROR)166 2451 y Fx(F)l(unctions)c Fv(MPI)p 462 2451 14 2 v 16 w(GRAPHDIMS)p 739 2451 V 17 w(GET)g Fx(and)g Fv(MPI)p 1037 2451 V 16 w(GRAPH)p 1203 2451 V 18 w(GET)g Fx(retriev)o(e)g(the)f(graph-top)q (ology)75 2508 y(information)d(that)g(w)o(as)f(asso)q(ciated)i(with)f (a)g(comm)o(unicator)g(b)o(y)g Fv(MPI)p 1310 2508 V 16 w(GRAPH)p 1476 2508 V 17 w(CREA)l(TE)p Fx(.)166 2564 y(The)21 b(information)g(pro)o(vided)h(b)o(y)f Fv(MPI)p 864 2564 V 16 w(GRAPHDIMS)p 1141 2564 V 17 w(GET)g Fx(can)g(b)q(e)g (used)h(to)e(dimension)j(the)75 2621 y(v)o(ectors)14 b Fu(index)h Fx(and)g Fu(edges)g Fx(correctly)g(for)g(the)g(follo)o (wing)h(call)g(to)f Fv(MPI)p 1339 2621 V 16 w(GRAPH)p 1505 2621 V 17 w(GET)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 181 189 181 188 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fx(181)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(GET\(comm)n(,)11 b(maxindex,)i(maxedges,)g(index,)j (edges\))117 122 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(graph)g(structure)i(\(handle\))117 197 y(IN)171 b Fv(maxi)o(ndex)392 b Fk(length)14 b(of)f(v)o(ector)i Fi(index)g Fk(in)f(the)g(calling)f(program)905 254 y(\(in)o(teger\))117 329 y(IN)171 b Fv(maxedges)386 b Fk(length)14 b(of)f(v)o(ector)i Fi(edges)g Fk(in)e(the)i(calling)d(program)905 385 y(\(in)o(teger\))117 461 y(OUT)124 b Fv(index)473 b Fk(arra)o(y)17 b(of)g(in)o(tegers)i(con) o(taining)d(the)i(graph)g(structure)h(\(for)905 517 y(details)14 b(see)h(the)g(de\014nition)e(of)g Fi(MPI)p 1484 517 13 2 v 15 w(GRAPH)p 1638 517 V 14 w(CREA)m(TE)p Fk(\))117 592 y(OUT)124 b Fv(edges)468 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i(con) o(taining)e(the)h(graph)g(structure)75 717 y Fu(int)23 b(MPI)p 245 717 15 2 v 17 w(Graph)p 382 717 V 17 w(get\(MPI)p 567 717 V 16 w(Comm)g(comm,)g(int)h(maxindex,)f(int)g(maxedges,)g(int)g (*index,)393 773 y(int)h(*edges\))75 860 y(MPI)p 150 860 V 17 w(GRAPH)p 287 860 V 16 w(GET\(COMM,)f(MAXINDEX,)g(MAXEDGES,)f (INDEX,)h(EDGES,)g(IERROR\))170 916 y(INTEGER)g(COMM,)h(MAXINDEX,)e (MAXEDGES,)h(INDEX\(*\),)f(EDGES\(*\),)h(IERROR)75 1097 y Fv(MPI)p 160 1097 14 2 v 16 w(CARTDIM)p 381 1097 V 16 w(GET\(comm)m(,)12 b(ndims\))117 1174 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j (\(handle\))117 1249 y(OUT)124 b Fv(ndims)458 b Fk(n)o(um)o(b)q(er)12 b(of)f(dimensions)g(of)h(the)h(cartesian)g(structure)h(\(in)o(te-)905 1306 y(ger\))75 1430 y Fu(int)23 b(MPI)p 245 1430 15 2 v 17 w(Cartdim)p 430 1430 V 16 w(get\(MPI)p 614 1430 V 17 w(Comm)g(comm,)g(int)h(*ndims\))75 1517 y(MPI)p 150 1517 V 17 w(CARTDIM)p 335 1517 V 16 w(GET\(COMM,)f(NDIMS,)g (IERROR\))170 1573 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)166 1660 y Fx(The)14 b(functions)h Fv(MPI)p 538 1660 14 2 v 16 w(CARTDIM)p 759 1660 V 16 w(GET)f Fx(and)g Fv(MPI)p 1046 1660 V 16 w(CART)p 1181 1660 V 17 w(GET)g Fx(return)g(the)g (cartesian)g(top)q(ology)75 1716 y(information)h(that)g(w)o(as)f(asso)q (ciated)i(with)f(a)g(comm)o(unicator)g(b)o(y)g Fv(MPI)p 1310 1716 V 16 w(CART)p 1445 1716 V 17 w(CREA)l(TE)p Fx(.)75 1867 y Fv(MPI)p 160 1867 V 16 w(CART)p 295 1867 V 17 w(GET\(comm)m(,)d(maxdim)m(s,)g(dims,)h(p)q(erio)q(ds,)j(co)q(o)o (rds\))117 1944 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 2019 y(IN)171 b Fv(maxdim)m(s)402 b Fk(length)16 b(of)f(v)o(ectors)i Fb(dims,)k(periods)p Fk(,)15 b(and)g Fb(coords)g Fk(in)g(the)905 2076 y(calling)e(program)f(\(in)o(teger\))117 2151 y(OUT)124 b Fv(dims)481 b Fk(n)o(um)o(b)q(er)13 b(of)g(pro)q(cesses)i(for)e(eac)o (h)h(cartesian)g(dimension)e(\(ar-)905 2207 y(ra)o(y)i(of)f(in)o (teger\))117 2283 y(OUT)124 b Fv(p)q(erio)q(ds)436 b Fk(p)q(erio)q(dicit)o(y)21 b(\()p Fi(true)p Fk(/)p Fi(false)p Fk(\))g(for)g(eac)o(h)g(cartesian)h(dimension)905 2339 y(\(arra)o(y)14 b(of)f(logical\))117 2414 y(OUT)124 b Fv(co)q(o)o(rds)449 b Fk(co)q(ordinates)21 b(of)f(calling)e(pro)q(cess) k(in)e(cartesian)h(structure)905 2471 y(\(arra)o(y)14 b(of)f(in)o(teger\))75 2595 y Fu(int)23 b(MPI)p 245 2595 15 2 v 17 w(Cart)p 358 2595 V 17 w(get\(MPI)p 543 2595 V 16 w(Comm)g(comm,)h(int)f(maxdims,)g(int)g(*dims,)g(int)h(*periods,) 393 2651 y(int)g(*coords\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 182 190 182 189 bop 75 -100 a Fx(182)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(CART)p 263 45 V 16 w(GET\(COMM,)23 b(MAXDIMS,)g(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(COORDS\(*\),)g(IERROR)170 158 y(LOGICAL)g(PERIODS\(*\))75 339 y Fv(MPI)p 160 339 14 2 v 16 w(CART)p 295 339 V 17 w(RANK\(comm)m(,)12 b(co)q(o)o(rds,)j(rank\))117 416 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g (structure)j(\(handle\))117 491 y(IN)171 b Fv(co)q(o)o(rds)449 b Fk(in)o(teger)17 b(arra)o(y)f(\(of)g(size)h Fb(ndims)p Fk(\))e(sp)q(ecifying)h(the)h(cartesian)905 548 y(co)q(ordinates)e(of)e (a)h(pro)q(cess)117 623 y(OUT)124 b Fv(rank)488 b Fk(rank)14 b(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 747 y Fu(int)23 b(MPI)p 245 747 15 2 v 17 w(Cart)p 358 747 V 17 w(rank\(MPI)p 567 747 V 16 w(Comm)g(comm,)g(int)h(*coords,)f (int)g(*rank\))75 834 y(MPI)p 150 834 V 17 w(CART)p 263 834 V 16 w(RANK\(COMM,)g(COORDS,)g(RANK,)g(IERROR\))170 890 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h(IERROR)166 977 y Fx(F)l(or)12 b(a)h(pro)q(cess)g(group)g(with)g(cartesian)g (structure,)g(the)g(function)g Fv(MPI)p 1397 977 14 2 v 16 w(CART)p 1532 977 V 17 w(RANK)g Fx(translates)75 1033 y(the)22 b(logical)h(pro)q(cess)f(co)q(ordinates)g(to)g(pro)q (cess)g(ranks)f(as)h(they)g(are)f(used)i(b)o(y)e(the)h(p)q(oin)o (t-to-p)q(oin)o(t)75 1090 y(routines.)166 1146 y(F)l(or)f(dimension)i Fu(i)e Fx(with)h Fu(periods\(i\))h(=)h(true)p Fx(,)e(if)g(the)f(co)q (ordinate,)j Fu(coords\(i\))p Fx(,)d(is)h(out)f(of)75 1203 y(range,)14 b(that)g(is,)h Fu(coords\(i\))22 b Fp(<)j Fu(0)14 b Fx(or)g Fu(coords\(i\))23 b Ft(\025)h Fu(dims\(i\))p Fx(,)13 b(it)i(is)g(shifted)h(bac)o(k)e(to)g(the)h(in)o(terv)m(al)75 1259 y Fu(0)24 b Ft(\024)g Fu(coords\(i\))f Ft(\024)h Fu(dims\(i\))e Fx(automatically)l(.)46 b(Out-of-range)23 b(co)q(ordinates)h(are)f(erroneous)h(for)75 1315 y(non-p)q(erio)q(dic) 18 b(dimensions.)75 1466 y Fv(MPI)p 160 1466 V 16 w(CART)p 295 1466 V 17 w(COORDS\(comm)n(,)12 b(rank,)j(maxdim)m(s,)d(co)q(o)o (rds\))117 1544 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 1619 y(IN)171 b Fv(rank)488 b Fk(rank)14 b(of)f(a)h(pro)q(cess)i(within)d (group)g(of)h Fi(comm)8 b Fk(\(in)o(teger\))117 1694 y(IN)171 b Fv(maxdim)m(s)402 b Fk(length)11 b(of)f(v)o(ector)i Fb(coord)d Fk(in)h(the)i(calling)d(program)g(\(in)o(teger\))117 1769 y(OUT)124 b Fv(co)q(o)o(rds)449 b Fk(in)o(teger)16 b(arra)o(y)e(\(of)h(size)h Fb(ndims)p Fk(\))e(con)o(taining)g(the)h (cartesian)905 1825 y(co)q(ordinates)g(of)e(sp)q(eci\014ed)j(pro)q (cess)f(\(in)o(teger\))75 1950 y Fu(int)23 b(MPI)p 245 1950 15 2 v 17 w(Cart)p 358 1950 V 17 w(coords\(MPI)p 615 1950 V 16 w(Comm)g(comm,)g(int)h(rank,)f(int)g(maxdims,)g(int)g (*coords\))75 2036 y(MPI)p 150 2036 V 17 w(CART)p 263 2036 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g(COORDS,)f(IERROR\))170 2093 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f(COORDS\(*\),)h(IERROR)166 2179 y Fx(The)10 b(in)o(v)o(erse)h(mapping,)g(rank-to-co)q(ordinates)f (translation)g(is)h(pro)o(vided)g(b)o(y)f Fv(MPI)p 1583 2179 14 2 v 16 w(CART)p 1718 2179 V 17 w(COORDS)p Fx(.)75 2330 y Fv(MPI)p 160 2330 V 16 w(GRAPH)p 326 2330 V 17 w(NEIGHBORS)p 594 2330 V 19 w(COUNT\(comm)m(,)i(rank,)j(nneighb)q(o)o (rs\))117 2407 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q(ology)e(\(handle\))117 2483 y(IN)171 b Fv(rank)488 b Fk(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(of)f Fi(comm)8 b Fk(\(in)o(teger\))117 2558 y(OUT)124 b Fv(nneighb)q(o)o(rs) 369 b Fk(n)o(um)o(b)q(er)13 b(of)h(neigh)o(b)q(ors)g(of)f(sp)q (eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 2682 y Fu(int)23 b(MPI)p 245 2682 15 2 v 17 w(Graph)p 382 2682 V 17 w(neighbors)p 615 2682 V 16 w(count\(MPI)p 847 2682 V 16 w(Comm)g(comm,)g(int)h (rank,)f(int)g(*nneighbors\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 183 191 183 190 bop 75 -100 a Fm(6.5.)29 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)963 b Fx(183)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(GRAPH)p 287 45 V 16 w(NEIGHBORS)p 519 45 V 16 w(COUNT\(COMM,)23 b(RANK,)g(NNEIGHBORS,)f(IERROR\))170 102 y(INTEGER)h(COMM,)h(RANK,)f (NNEIGHBORS,)f(IERROR)166 188 y Fv(MPI)p 251 188 14 2 v 16 w(GRAPH)p 417 188 V 17 w(NEIGHBORS)p 685 188 V 18 w(COUNT)h Fx(and)e Fv(MPI)p 1058 188 V 16 w(GRAPH)p 1224 188 V 18 w(NEIGHBORS)i Fx(pro)o(vide)f(adjacency)75 244 y(information)15 b(for)g(a)g(general,)g(graph)g(top)q(ology)l(.)75 395 y Fv(MPI)p 160 395 V 16 w(GRAPH)p 326 395 V 17 w(NEIGHBORS\(comm)o (,)d(rank,)j(maxneighb)q(o)o(rs,)d(neighb)q(o)o(rs\))117 473 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q(ology)e(\(handle\))117 547 y(IN)171 b Fv(rank)488 b Fk(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(of)f Fi(comm)8 b Fk(\(in)o(teger\))117 621 y(OUT)124 b Fv(maxneighb)q(o)o (rs)310 b Fk(size)15 b(of)e(arra)o(y)h Fi(neighb)q(o)o(rs)h Fk(\(in)o(teger\))117 695 y(OUT)124 b Fv(neighb)q(o)o(rs)392 b Fk(ranks)13 b(of)g(pro)q(cesses)i(that)e(are)h(neigh)o(b)q(ors)f(to)g (sp)q(eci\014ed)h(pro-)905 751 y(cess)i(\(arra)o(y)d(of)h(in)o(teger\)) 75 876 y Fu(int)23 b(MPI)p 245 876 15 2 v 17 w(Graph)p 382 876 V 17 w(neighbors\(MPI)p 711 876 V 15 w(Comm)g(comm,)h(int)f (rank,)g(int)h(*maxneighbors,)393 932 y(int)g(*neighbors\))75 1018 y(MPI)p 150 1018 V 17 w(GRAPH)p 287 1018 V 16 w(NEIGHBORS\(COMM,)e (RANK,)h(MAXNEIGHBORS,)f(NEIGHBORS,)h(IERROR\))170 1075 y(INTEGER)g(COMM,)h(RANK,)f(MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR)75 1203 y Fq(Example)c(6.3)k Fx(Supp)q(ose)14 b(that)d Fu(comm)h Fx(is)g(a)g(comm)o(unicator)g(with)g(a)g(sh)o(u\017e-exc)o(hange)h(top) q(ology)l(.)19 b(The)75 1260 y(group)h(has)f(2)317 1243 y Ff(n)360 1260 y Fx(mem)o(b)q(ers.)34 b(Eac)o(h)19 b(pro)q(cess)h(is)h (lab)q(eled)h(b)o(y)d Fp(a)1166 1267 y Fg(1)1186 1260 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)1312 1267 y Ff(n)1354 1260 y Fx(with)21 b Fp(a)1487 1267 y Ff(i)1521 1260 y Ft(2)f(f)p Fx(0)p Fp(;)8 b Fx(1)p Ft(g)p Fx(,)19 b(and)h(has)75 1316 y(three)d(neigh)o(b)q(ors:)23 b(exc)o(hange\()p Fp(a)639 1323 y Fg(1)659 1316 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)785 1323 y Ff(n)808 1316 y Fx(\))14 b(=)i Fp(a)915 1323 y Fg(1)935 1316 y Fp(;)8 b(:)g(:)g(:)t(;)g(a)1060 1323 y Ff(n)p Fh(\000)p Fg(1)1128 1316 y Fp(;)g Fx(\026)-23 b Fp(a)1173 1323 y Ff(n)1213 1316 y Fx(\()q(\026)f Fp(a)14 b Fx(=)i(1)11 b Ft(\000)g Fp(a)p Fx(\),)17 b(sh)o(u\017e\()p Fp(a)1639 1323 y Fg(1)1658 1316 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)1784 1323 y Ff(n)1807 1316 y Fx(\))15 b(=)75 1373 y Fp(a)99 1380 y Fg(2)119 1373 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)245 1380 y Ff(n)267 1373 y Fp(;)g(a)312 1380 y Fg(1)331 1373 y Fx(,)24 b(and)e(unsh)o(u\017e\()p Fp(a)680 1380 y Fg(1)700 1373 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)826 1380 y Ff(n)849 1373 y Fx(\))24 b(=)g Fp(a)974 1380 y Ff(n)998 1373 y Fp(;)8 b(a)1043 1380 y Fg(1)1062 1373 y Fp(;)g(:)g(:)g(:)t(;)g(a)1187 1380 y Ff(n)p Fh(\000)p Fg(1)1255 1373 y Fx(.)41 b(The)22 b(graph)g(adjacency)h(list)f(is)75 1429 y(illustrated)17 b(b)q(elo)o(w)e(for)g Fp(n)e Fx(=)g(3.)p 75 1494 1119 2 v 74 1550 2 57 v 133 1533 a Fq(no)q(de)p 300 1550 V 104 w(exc)o(hange)122 b(sh)o(u\017e)h(unsh)o(u\017e)p 1193 1550 V 74 1606 V 300 1606 V 326 1590 a Fx(neigh)o(b)q(ors\(1\))49 b(neigh)o(b)q(ors\(2\))h(neigh)o(b)q(ors\(3\))p 1193 1606 V 75 1608 1119 2 v 74 1665 2 57 v 100 1648 a(0)f(\(000\))p 300 1665 V 161 w(1)275 b(0)g(0)p 1193 1665 V 74 1721 V 100 1704 a(1)49 b(\(001\))p 300 1721 V 161 w(0)275 b(2)g(4)p 1193 1721 V 74 1778 V 100 1761 a(2)49 b(\(010\))p 300 1778 V 161 w(3)275 b(4)g(1)p 1193 1778 V 74 1834 V 100 1817 a(3)49 b(\(011\))p 300 1834 V 161 w(2)275 b(6)g(5)p 1193 1834 V 74 1890 V 100 1873 a(4)49 b(\(100\))p 300 1890 V 161 w(5)275 b(1)g(2)p 1193 1890 V 74 1947 V 100 1930 a(5)49 b(\(101\))p 300 1947 V 161 w(4)275 b(3)g(6)p 1193 1947 V 74 2003 V 100 1986 a(6)49 b(\(110\))p 300 2003 V 161 w(7)275 b(5)g(3)p 1193 2003 V 74 2060 V 100 2043 a(7)49 b(\(111\))p 300 2060 V 161 w(6)275 b(7)g(7)p 1193 2060 V 75 2061 1119 2 v 166 2097 a(Supp)q(ose)15 b(that)e(the)g(comm)o(unicator)h Fu(comm)f Fx(has)g(this)h(top)q(ology) f(asso)q(ciated)h(with)g(it.)20 b(The)14 b(follo)o(w-)75 2154 y(ing)g(co)q(de)h(fragmen)o(t)e(cycles)h(through)g(the)g(three)g (t)o(yp)q(es)g(of)f(neigh)o(b)q(ors)i(and)f(p)q(erforms)f(an)h (appropriate)75 2210 y(p)q(erm)o(utation)h(for)g(eac)o(h.)75 2308 y Fu(C)48 b(assume:)22 b(each)i(process)f(has)g(stored)g(a)h(real) f(number)g(A.)75 2365 y(C)48 b(extract)22 b(neighborhood)h(information) 218 2421 y(CALL)g(MPI_COMM_RANK\(comm,)f(myrank,)h(ierr\))218 2478 y(CALL)g(MPI_GRAPH_NEIGHBORS\(comm,)e(myrank,)i(3,)g(neighbors,)g (ierr\))75 2534 y(C)48 b(perform)22 b(exchange)h(permutation)218 2591 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e (neighbors\(1\),)h(0,)194 2647 y(+)119 b(neighbors\(1\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 2704 y(C)48 b(perform)22 b(shuffle)h(permutation)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 184 192 184 191 bop 75 -100 a Fx(184)887 b Fm(CHAPTER)15 b(6.)30 b(PR)o(OCESS)15 b(TOPOLOGIES)218 45 y Fu(CALL)23 b (MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(neighbors\(2\),)h(0,)194 102 y(+)119 b(neighbors\(3\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 158 y(C)48 b(perform)22 b(unshuffle)h(permutation)218 214 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e (neighbors\(3\),)h(0,)194 271 y(+)119 b(neighbors\(2\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 393 y Fj(6.5.5)49 b(Ca)o(rtesian)16 b(Shift)g(Co)q(o)o(rdinates)75 478 y Fx(If)f(the)g(pro)q(cess)f(top)q(ology)h(is)g(a)f(cartesian)h (structure,)f(a)g Fv(MPI)p 1145 478 14 2 v 16 w(SENDRECV)i Fx(op)q(eration)f(is)g(lik)o(ely)h(to)e(b)q(e)75 535 y(used)19 b(along)g(a)f(co)q(ordinate)h(direction)h(to)e(p)q(erform)g (a)g(shift)h(of)f(data.)29 b(As)19 b(input,)h Fv(MPI)p 1629 535 V 16 w(SENDRECV)75 591 y Fx(tak)o(es)14 b(the)h(rank)g(of)f(a) h(source)f(pro)q(cess)i(for)e(the)h(receiv)o(e,)g(and)g(the)g(rank)g (of)f(a)h(destination)g(pro)q(cess)h(for)75 648 y(the)11 b(send.)18 b(If)11 b(the)g(function)g Fv(MPI)p 641 648 V 16 w(CART)p 776 648 V 17 w(SHIFT)f Fx(is)h(called)h(for)e(a)h (cartesian)f(pro)q(cess)h(group,)g(it)f(pro)o(vides)75 704 y(the)j(calling)h(pro)q(cess)e(with)h(the)g(ab)q(o)o(v)o(e)f(iden)o (ti\014ers,)i(whic)o(h)f(then)g(can)g(b)q(e)g(passed)f(to)g Fv(MPI)p 1616 704 V 16 w(SENDRECV)p Fx(.)75 761 y(The)17 b(user)h(sp)q(eci\014es)h(the)e(co)q(ordinate)h(direction)g(and)f(the)h (size)g(of)f(the)g(step)g(\(p)q(ositiv)o(e)h(or)f(negativ)o(e\).)75 817 y(The)e(function)h(is)g(lo)q(cal.)75 968 y Fv(MPI)p 160 968 V 16 w(CART)p 295 968 V 17 w(SHIFT\(com)o(m)n(,)11 b(direction,)16 b(disp,)g(rank)p 977 968 V 16 w(source,)g(rank)p 1224 968 V 16 w(dest\))117 1045 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 1120 y(IN)171 b Fv(direction)408 b Fk(co)q(ordinate)14 b(dimension)f(of)g(shift)h(\(in)o(teger\))117 1196 y(IN)171 b Fv(disp)497 b Fk(displacemen)o(t)18 b(\()p Fa(>)h Fk(0:)27 b(up)o(w)o(ards)18 b(shift,)h Fa(<)g Fk(0:)27 b(do)o(wn)o(w)o(ards)905 1252 y(shift\))14 b(\(in)o(teger\))117 1327 y(OUT)124 b Fv(rank)p 420 1327 V 16 w(source)353 b Fk(rank)14 b(of)f(source)j (pro)q(cess)f(\(in)o(teger\))117 1402 y(OUT)124 b Fv(rank)p 420 1402 V 16 w(dest)396 b Fk(rank)14 b(of)f(destination)h(pro)q(cess)i (\(in)o(teger\))75 1527 y Fu(int)23 b(MPI)p 245 1527 15 2 v 17 w(Cart)p 358 1527 V 17 w(shift\(MPI)p 591 1527 V 16 w(Comm)g(comm,)g(int)h(direction,)e(int)i(disp,)f(int)g(*rank)p 1681 1527 V 17 w(source,)393 1583 y(int)h(*rank)p 612 1583 V 16 w(dest\))75 1670 y(MPI)p 150 1670 V 17 w(CART)p 263 1670 V 16 w(SHIFT\(COMM,)f(DIRECTION,)f(DISP,)h(RANK)p 1067 1670 V 17 w(SOURCE,)g(RANK)p 1371 1670 V 17 w(DEST,)g(IERROR\))170 1726 y(INTEGER)g(COMM,)h(DIRECTION,)e(DISP,)h(RANK)p 1009 1726 V 17 w(SOURCE,)g(RANK)p 1313 1726 V 16 w(DEST,)h(IERROR)166 1813 y Fx(Dep)q(ending)18 b(on)e(the)g(p)q(erio)q(dicit)o(y)i(of)e(the) g(cartesian)h(group)f(in)h(the)f(sp)q(eci\014ed)i(co)q(ordinate)f (direc-)75 1869 y(tion,)c Fv(MPI)p 264 1869 14 2 v 16 w(CART)p 399 1869 V 17 w(SHIFT)g Fx(pro)o(vides)h(the)f(iden)o (ti\014ers)i(for)e(a)g(circular)h(or)f(an)g(end-o\013)g(shift.)20 b(In)14 b(the)f(case)75 1926 y(of)j(an)g(end-o\013)g(shift,)h(the)f(v)m (alue)i Fi(MPI)p 735 1926 13 2 v 14 w(PROC)p 861 1926 V 13 w(NULL)f Fx(ma)o(y)f(b)q(e)h(returned)f(in)h Fv(rank)p 1479 1926 14 2 v 17 w(source)g Fx(or)e Fv(rank)p 1771 1926 V 17 w(dest)p Fx(,)75 1982 y(indicating)i(that)e(the)g(source)g (or)g(the)g(destination)h(for)f(the)g(shift)h(is)f(out)g(of)g(range.)75 2088 y Fq(Example)j(6.4)k Fx(The)15 b(comm)o(unicator,)g Fv(comm)m Fx(,)c(has)k(a)f(t)o(w)o(o-dimensional,)i(p)q(erio)q(dic)h (cartesian)e(top)q(ol-)75 2145 y(ogy)e(asso)q(ciated)h(with)g(it.)19 b(A)14 b(t)o(w)o(o-dimensional)g(arra)o(y)f(of)g Fu(REAL)p Fx(s)g(is)h(stored)f(one)h(elemen)o(t)g(p)q(er)g(pro)q(cess,)75 2201 y(in)i(v)m(ariable)h Fu(A)p Fx(.)d(One)i(wishes)g(to)f(sk)o(ew)g (this)g(arra)o(y)l(,)f(b)o(y)h(shifting)h(column)g Fu(i)f Fx(b)o(y)g Fu(i)g Fx(steps.)75 2295 y Fu(....)75 2351 y(C)24 b(find)f(process)g(rank)218 2408 y(CALL)g(MPI_COMM_RANK\(comm,)f (rank,)h(ierr\)\))75 2464 y(C)h(find)f(cartesian)g(coordinates)218 2521 y(CALL)g(MPI_CART_COORDS\(comm,)e(rank,)j(coords,)f(ierr\))75 2577 y(C)h(compute)f(shift)g(source)g(and)g(destination)218 2634 y(CALL)g(MPI_CART_SHIFT\(comm,)f(2,)h(coords\(2\),)g(source,)g (dest,)g(ierr\))75 2690 y(C)h(skew)f(array)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 185 193 185 192 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fx(185)218 45 y Fu(CALL)23 b(MPI_SENDRECV_REPLACE\(A,)e(1,)j (MPI_REAL,)e(dest,)i(0,)f(source,)g(0,)h(comm,)194 102 y(+)621 b(status,)23 b(ierr\))75 223 y Fj(6.5.6)49 b(P)o(a)o (rtitioning)17 b(of)g(Ca)o(rtesian)e(structures)75 404 y Fv(MPI)p 160 404 14 2 v 16 w(CART)p 295 404 V 17 w(SUB\(comm)m(,)d (remain)p 690 404 V 14 w(dims,)g(new)o(comm)n(\))117 481 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 556 y(IN)171 b Fv(rem)o(ain)p 467 556 V 14 w(dims)337 b Fk(the)24 b Fb(i)p Fk(th)f(en)o(try)h(of)e Fi(remain)p 1365 556 13 2 v 13 w(dims)e Fk(sp)q(eci\014es)25 b(whether)f(the)905 612 y Fb(i)p Fk(th)9 b(dimension)f(is)h(k)o(ept)h(in)f(the)h(subgrid)f (\()p Fb(true)p Fk(\))g(or)g(is)g(dropp)q(ed)905 669 y(\()p Fb(false)p Fk(\))k(\(logical)g(v)o(ector\))117 744 y(OUT)124 b Fv(new)o(comm)377 b Fk(comm)o(unicator)20 b(con)o(taining)h(the)i(subgrid)f(that)g(includes)905 800 y(the)15 b(calling)d(pro)q(cess)k(\(handle\))75 925 y Fu(int)23 b(MPI)p 245 925 15 2 v 17 w(Cart)p 358 925 V 17 w(sub\(MPI)p 543 925 V 16 w(Comm)g(comm,)h(int)f(*remain)p 1085 925 V 16 w(dims,)g(MPI)p 1316 925 V 17 w(Comm)h(*newcomm\))75 1011 y(MPI)p 150 1011 V 17 w(CART)p 263 1011 V 16 w(SUB\(COMM,)f (REMAIN)p 662 1011 V 16 w(DIMS,)h(NEWCOMM,)e(IERROR\))170 1068 y(INTEGER)h(COMM,)h(NEWCOMM,)e(IERROR)170 1124 y(LOGICAL)h(REMAIN) p 508 1124 V 17 w(DIMS\(*\))166 1211 y Fx(If)28 b(a)f(cartesian)g(top)q (ology)g(has)h(b)q(een)g(created)g(with)g Fv(MPI)p 1263 1211 14 2 v 15 w(CART)p 1397 1211 V 17 w(CREA)l(TE)p Fx(,)h(the)e(function)75 1267 y Fv(MPI)p 160 1267 V 16 w(CART)p 295 1267 V 17 w(SUB)c Fx(can)f(b)q(e)h(used)f(to)g(partition)g (the)g(comm)o(unicator)g(group)g(in)o(to)g(subgroups)g(that)75 1324 y(form)17 b(lo)o(w)o(er-dimensional)j(cartesian)f(subgrids,)g(and) f(to)g(build)i(for)d(eac)o(h)h(subgroup)h(a)e(comm)o(unica-)75 1380 y(tor)k(with)h(the)g(asso)q(ciated)f(subgrid)i(cartesian)f(top)q (ology)l(.)39 b(\(This)22 b(function)g(is)h(closely)f(related)h(to)75 1437 y Fv(MPI)p 160 1437 V 16 w(COMM)p 318 1437 V 16 w(SPLIT)p Fx(.\))75 1543 y Fq(Example)18 b(6.5)k Fx(Assume)12 b(that)g Fv(MPI)p 726 1543 V 15 w(CART)p 860 1543 V 17 w(CREA)l(TE)p Fu(\(...,)25 b(comm\))11 b Fx(has)h(de\014ned)h(a)e(\(2)s Ft(\002)s Fx(3)s Ft(\002)s Fx(4\))i(grid.)75 1599 y(Let)i Fu(remain)p 303 1599 15 2 v 17 w(dims)23 b(=)h(\(true,)f(false,)g (true\))p Fx(.)c(Then)d(a)f(call)h(to,)194 1693 y Fu (MPI_CART_SUB\(comm,)22 b(remain_dims,)g(comm_new\),)75 1787 y Fx(will)k(create)d(three)i(comm)o(unicators)e(eac)o(h)h(with)h (eigh)o(t)f(pro)q(cesses)g(in)h(a)f(2)15 b Ft(\002)i Fx(4)23 b(cartesian)i(top)q(ol-)75 1843 y(ogy)l(.)36 b(If)21 b Fu(remain)p 388 1843 V 16 w(dims)j(=)f(\(false,)g(false,)g (true\))d Fx(then)h(the)g(call)h(to)e Fv(MPI)p 1494 1843 14 2 v 16 w(CART)p 1629 1843 V 17 w(SUB\(comm)m(,)75 1900 y(remain)p 209 1900 V 13 w(dims,)i(comm)p 465 1900 V 10 w(new\))h Fx(will)h(create)e(six)g(non-o)o(v)o(erlapping)h(comm)o (unicators,)h(eac)o(h)e(with)g(four)75 1956 y(pro)q(cesses,)15 b(in)h(a)f(one-dimensional)j(cartesian)d(top)q(ology)l(.)75 2078 y Fj(6.5.7)49 b(Lo)o(w-level)18 b(top)q(ology)g(functions)75 2164 y Fx(The)d(t)o(w)o(o)e(additional)k(functions)e(in)o(tro)q(duced)h (in)g(this)f(section)h(can)e(b)q(e)i(used)f(to)g(implemen)o(t)h(all)f (other)75 2220 y(top)q(ology)g(functions.)21 b(In)15 b(general)h(they)f(will)i(not)e(b)q(e)h(called)h(b)o(y)e(the)g(user)h (directly)l(,)g(unless)g(he)g(or)f(she)75 2277 y(is)h(creating)f (additional)i(virtual)f(top)q(ology)e(capabilit)o(y)j(other)e(than)g (that)f(pro)o(vided)i(b)o(y)g Fv(MPI)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 186 194 186 193 bop 75 -100 a Fx(186)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(CART)p 295 45 V 17 w(MAP\(comm)m(,)d(ndims,)h(dims,)f(p)q (erio)q(ds,)k(newrank\))117 122 y Fk(IN)171 b Fv(comm)450 b Fk(input)14 b(comm)o(unicator)d(\(handle\))117 197 y(IN)171 b Fv(ndims)458 b Fk(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of) i(cartesian)g(structure)i(\(in)o(teger\))117 273 y(IN)171 b Fv(dims)481 b Fk(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fb(ndims)e Fk(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 329 y(pro)q(cesses)g(in)c(eac)o(h)i(co)q(ordinate)f(direction)117 404 y(IN)171 b Fv(p)q(erio)q(ds)436 b Fk(logical)16 b(arra)o(y)h(of)g (size)h Fb(ndims)f Fk(sp)q(ecifying)g(the)h(p)q(erio)q(dicit)o(y)905 461 y(sp)q(eci\014cation)d(in)e(eac)o(h)i(co)q(ordinate)f(direction)117 536 y(OUT)124 b Fv(newrank)414 b Fk(reordered)11 b(rank)f(of)e(the)i (calling)e(pro)q(cess;)k Fi(MPI)p 1643 536 13 2 v 15 w(UNDEFINED)905 592 y Fk(if)h(calling)g(pro)q(cess)i(do)q(es)g(not)f(b) q(elong)f(to)h(grid)g(\(in)o(teger\))75 717 y Fu(int)23 b(MPI)p 245 717 15 2 v 17 w(Cart)p 358 717 V 17 w(map\(MPI)p 543 717 V 16 w(Comm)g(comm,)h(int)f(ndims,)g(int)h(*dims,)f(int)g (*periods,)393 773 y(int)h(*newrank\))75 860 y(MPI)p 150 860 V 17 w(CART)p 263 860 V 16 w(MAP\(COMM,)f(NDIMS,)g(DIMS,)g (PERIODS,)g(NEWRANK,)g(IERROR\))170 916 y(INTEGER)g(COMM,)h(NDIMS,)f (DIMS\(*\),)f(NEWRANK,)h(IERROR)170 972 y(LOGICAL)g(PERIODS\(*\))166 1059 y Fv(MPI)p 251 1059 14 2 v 16 w(CART)p 386 1059 V 17 w(MAP)12 b Fx(computes)h(an)f(\\optimal")g(placemen)o(t)i(for)d (the)i(calling)h(pro)q(cess)f(on)f(the)h(ph)o(ys-)75 1115 y(ical)i(mac)o(hine.)20 b(A)13 b(p)q(ossible)i(implemen)o(tation)g (of)e(this)h(function)g(is)g(to)e(alw)o(a)o(ys)h(return)g(the)h(rank)f (of)g(the)75 1172 y(calling)k(pro)q(cess,)e(that)g(is,)g(not)g(to)f(p)q (erform)h(an)o(y)g(reordering.)189 1278 y Fl(A)n(dvic)n(e)22 b(to)h(implementors.)84 b Fx(The)22 b(function)i Fv(MPI)p 1138 1278 V 15 w(CART)p 1272 1278 V 17 w(CREA)l(TE\(comm)n(,)d(ndims,)h (dims,)189 1335 y(p)q(erio)q(ds,)h(reo)o(rder,)d(comm)p 647 1335 V 11 w(ca)o(rt\))p Fx(,)h(with)g Fu(reorder)i(=)h(true)c Fx(can)h(b)q(e)g(implemen)o(ted)i(b)o(y)d(calling)189 1391 y Fv(MPI)p 274 1391 V 15 w(CART)p 408 1391 V 17 w(MAP\(comm)m(,)71 b(ndims,)g(dims,)g(p)q(erio)q(ds,)j(newrank\))p Fx(,)g(then)62 b(calling)189 1447 y Fv(MPI)p 274 1447 V 15 w(COMM)p 431 1447 V 17 w(SPLIT\(com)o(m)m(,)26 b(colo)o(r,)i(k)o (ey)l(,)h(comm)p 1096 1447 V 11 w(ca)o(rt\))p Fx(,)f(with)f Fu(color)d(=)f(0)k Fx(if)g Fu(newrank)c Ft(6)p Fx(=)189 1504 y Fu(MPI)p 264 1504 15 2 v 16 w(UNDEFINED)p Fx(,)14 b Fu(color)23 b(=)h(MPI)p 786 1504 V 17 w(UNDEFINED)14 b Fx(otherwise,)h(and)g Fu(key)23 b(=)h(newrank)p Fx(.)189 1579 y(The)c(function)h Fv(MPI)p 555 1579 14 2 v 15 w(CART)p 689 1579 V 17 w(SUB\(comm)n(,)d(rema)o(in)p 1090 1579 V 14 w(dims,)h(comm)p 1344 1579 V 10 w(new\))i Fx(can)f(b)q(e)g (implemen)o(ted)189 1636 y(b)o(y)c(a)g(call)i(to)e Fv(MPI)p 519 1636 V 16 w(COMM)p 677 1636 V 16 w(SPLIT\(comm)m(,)d(colo)o(r,)j(k) o(ey)l(,)h(comm)p 1304 1636 V 10 w(new\))p Fx(,)g(using)g(a)g(single)h (n)o(um)o(b)q(er)189 1692 y(enco)q(ding)11 b(of)f(the)g(lost)g (dimensions)i(as)e Fu(color)g Fx(and)g(a)g(single)h(n)o(um)o(b)q(er)g (enco)q(ding)h(of)d(the)i(preserv)o(ed)189 1748 y(dimensions)16 b(as)f Fu(key)p Fx(.)189 1824 y(All)f(other)g(cartesian)f(top)q(ology)g (functions)i(can)e(b)q(e)i(implemen)o(ted)g(lo)q(cally)l(,)g(using)g (the)e(top)q(ology)189 1880 y(information)h(that)g(is)i(cac)o(hed)f (with)g(the)g(comm)o(unicator.)k(\()p Fl(End)c(of)h(advic)n(e)g(to)g (implementors.)p Fx(\))166 1986 y(The)f(corresp)q(onding)i(new)e (function)h(for)f(general)g(graph)g(structures)g(is)h(as)f(follo)o(ws.) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 187 195 187 194 bop 75 -100 a Fm(6.6.)34 b(AN)15 b(APPLICA)l(TION)i(EXAMPLE)974 b Fx(187)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(MAP\(comm)m(,)12 b(nno)q(des,)17 b(index,)f(edges,)g (newrank\))117 122 y Fk(IN)171 b Fv(comm)450 b Fk(input)14 b(comm)o(unicator)d(\(handle\))117 197 y(IN)171 b Fv(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(graph)f(no)q(des)i(\(in)o(teger\))117 273 y(IN)171 b Fv(index)473 b Fk(in)o(teger)30 b(arra)o(y)g(sp)q (ecifying)f(the)i(graph)e(structure,)36 b(see)905 329 y Fi(MPI)p 982 329 13 2 v 15 w(GRAPH)p 1136 329 V 14 w(CREA)m(TE)117 404 y Fk(IN)171 b Fv(edges)468 b Fk(in)o(teger)15 b(arra)o(y)e(sp)q(ecifying)h(the)g(graph)g(structure)117 479 y(OUT)124 b Fv(newrank)414 b Fk(reordered)11 b(rank)f(of)e(the)i (calling)e(pro)q(cess;)k Fi(MPI)p 1643 479 V 15 w(UNDEFINED)905 536 y Fk(if)j(the)i(calling)d(pro)q(cess)k(do)q(es)e(not)g(b)q(elong)g (to)f(graph)h(\(in)o(te-)905 592 y(ger\))75 717 y Fu(int)23 b(MPI)p 245 717 15 2 v 17 w(Graph)p 382 717 V 17 w(map\(MPI)p 567 717 V 16 w(Comm)g(comm,)g(int)h(nnodes,)f(int)g(*index,)g(int)h (*edges,)393 773 y(int)g(*newrank\))75 860 y(MPI)p 150 860 V 17 w(GRAPH)p 287 860 V 16 w(MAP\(COMM,)f(NNODES,)g(INDEX,)g (EDGES,)g(NEWRANK,)g(IERROR\))170 916 y(INTEGER)g(COMM,)h(NNODES,)e (INDEX\(*\),)h(EDGES\(*\),)g(NEWRANK,)f(IERROR)189 1052 y Fl(A)n(dvic)n(e)17 b(to)h(implementors.)53 b Fx(The)18 b(function)g Fv(MPI)p 1087 1052 14 2 v 16 w(GRAPH)p 1253 1052 V 17 w(CREA)l(TE\(comm)n(,)d(nno)q(des,)k(index,)189 1109 y(edges,)j(reo)o(rder,)e(comm)p 614 1109 V 11 w(graph\))p Fx(,)h(with)g Fu(reorder)i(=)h(true)c Fx(can)g(b)q(e)h(implemen)o(ted)i (b)o(y)d(calling)189 1165 y Fv(MPI)p 274 1165 V 15 w(GRAPH)p 439 1165 V 18 w(MAP\(comm)m(,)65 b(nno)q(des,)70 b(index,)e(edges,)h (newrank\))p Fx(,)f(then)58 b(calling)189 1222 y Fv(MPI)p 274 1222 V 15 w(COMM)p 431 1222 V 17 w(SPLIT\(com)o(m)m(,)21 b(colo)o(r,)h(k)o(ey)l(,)i(comm)p 1080 1222 V 11 w(graph\))p Fx(,)f(with)g Fu(color)g(=)h(0)e Fx(if)h Fu(newrank)g Ft(6)p Fx(=)189 1278 y Fu(MPI)p 264 1278 15 2 v 16 w(UNDEFINED)p Fx(,)14 b Fu(color)23 b(=)h(MPI)p 786 1278 V 17 w(UNDEFINED)14 b Fx(otherwise,)h(and)g Fu(key)23 b(=)h(newrank)p Fx(.)189 1353 y(All)c(other)f(graph)g(top)q(ology)g(functions)h(can)f(b)q(e)h (implemen)o(ted)h(lo)q(cally)l(,)h(using)e(the)f(top)q(ology)189 1410 y(information)14 b(that)g(is)i(cac)o(hed)f(with)g(the)g(comm)o (unicator.)k(\()p Fl(End)c(of)h(advic)n(e)g(to)g(implementors.)p Fx(\))75 1553 y Fn(6.6)59 b(An)20 b(Application)f(Example)75 1654 y Fq(Example)f(6.6)k Fx(The)11 b(example)h(in)f(\014gure)g(6.1)f (sho)o(ws)g(ho)o(w)g(the)h(grid)g(de\014nition)h(and)f(inquiry)h (functions)75 1711 y(can)20 b(b)q(e)g(used)h(in)f(an)g(application)h (program.)33 b(A)19 b(partial)h(di\013eren)o(tial)h(equation,)g(for)e (instance)i(the)75 1767 y(P)o(oisson)e(equation,)g(is)g(to)g(b)q(e)g (solv)o(ed)g(on)g(a)f(rectangular)h(domain.)31 b(First,)19 b(the)g(pro)q(cesses)g(organize)75 1824 y(themselv)o(es)e(in)h(a)e(t)o (w)o(o-dimensional)h(structure.)24 b(Eac)o(h)16 b(pro)q(cess)h(then)g (inquires)h(ab)q(out)f(the)f(ranks)g(of)75 1880 y(its)e(neigh)o(b)q (ors)h(in)g(the)f(four)g(directions)h(\(up,)f(do)o(wn,)g(righ)o(t,)g (left\).)19 b(The)c(n)o(umerical)g(problem)g(is)g(solv)o(ed)75 1937 y(b)o(y)g(an)g(iterativ)o(e)h(metho)q(d,)f(the)g(details)h(of)f (whic)o(h)h(are)f(hidden)i(in)f(the)f(subroutine)h Fu(relax)p Fx(.)166 1993 y(In)e(eac)o(h)f(relaxation)g(step)g(eac)o(h)g(pro)q (cess)h(computes)f(new)g(v)m(alues)h(for)f(the)g(solution)h(grid)f (function)75 2050 y(at)h(all)i(p)q(oin)o(ts)f(o)o(wned)g(b)o(y)g(the)g (pro)q(cess.)20 b(Then)15 b(the)g(v)m(alues)h(at)e(in)o(ter-pro)q(cess) i(b)q(oundaries)g(ha)o(v)o(e)e(to)g(b)q(e)75 2106 y(exc)o(hanged)e (with)h(neigh)o(b)q(oring)g(pro)q(cesses.)19 b(F)l(or)11 b(example,)i(the)f(exc)o(hange)g(subroutine)h(migh)o(t)f(con)o(tain)75 2162 y(a)i(call)i(lik)o(e)g Fv(MPI)p 363 2162 14 2 v 16 w(SEND\(...,neigh)p 666 2162 V 15 w(rank\(1\),...\))i Fx(to)c(send)h(up)q(dated)g(v)m(alues)h(to)e(the)h(left-hand)g(neigh)o (b)q(or)75 2219 y Fu(\(i-1,j\))p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 188 196 188 195 bop 75 -100 a Fx(188)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)194 163 y Fu(integer)23 b(ndims,)g(num)p 627 163 15 2 v 17 w(neigh)194 219 y(logical)g(reorder) 194 276 y(parameter)g(\(ndims=2,)g(num)p 747 276 V 16 w(neigh=4,)g(reorder=.true.\))194 332 y(integer)g(comm,)g(comm)p 627 332 V 17 w(cart,)g(dims\(ndims\),)f(neigh)p 1217 332 V 17 w(def\(ndims\))194 388 y(integer)h(neigh)p 508 388 V 17 w(rank\(num)p 717 388 V 16 w(neigh\),)g(own)p 996 388 V 16 w(position\(ndims\),)f(i,)i(j)194 445 y(logical)f (periods\(ndims\))194 501 y(real)p Ft(\003)p Fu(8)g(u\(0:101,0:101\),)f (f\(0:101,0:101\))194 558 y(data)i(dims)f(/)h(ndims)f Ft(\003)g Fu(0)h(/)194 614 y(comm)g(=)f(MPI)p 436 614 V 17 w(COMM)p 549 614 V 17 w(WORLD)75 671 y(C)107 b(Set)23 b(process)g(grid)g(size)h(and)f(periodicity)194 727 y(call)h(MPI)p 389 727 V 16 w(DIMS)p 501 727 V 17 w(CREATE\(comm,)e(ndims,)h(dims\)) 194 784 y(periods\(1\))g(=)h(.TRUE.)194 840 y(periods\(2\))f(=)h (.TRUE.)75 897 y(C)107 b(Create)23 b(a)g(grid)h(structure)e(in)i(WORLD) f(group)g(and)h(inquire)f(about)g(own)g(position)194 953 y(call)h(MPI)p 389 953 V 16 w(CART)p 501 953 V 17 w(CREATE)f(\(comm,)g(ndims,)g(dims,)g(periods,)g(reorder,)g(comm)p 1688 953 V 17 w(cart\))194 1009 y(call)h(MPI)p 389 1009 V 16 w(CART)p 501 1009 V 17 w(GET)f(\(comm)p 733 1009 V 17 w(cart,)g(ndims,)g(dims,)g(periods,)g(own)p 1490 1009 V 17 w(position\))75 1066 y(C)107 b(Look)23 b(up)h(the)f(ranks)g (for)h(the)f(neighbors.)46 b(Own)24 b(process)f(coordinates)f(are)i (\(i,j\).)75 1122 y(C)107 b(Neighbors)22 b(are)i(\(i-1,j\),)f (\(i+1,j\),)f(\(i,j-1\),)h(\(i,j+1\))194 1179 y(i)h(=)g(own)p 365 1179 V 17 w(position\(1\))194 1235 y(j)g(=)g(own)p 365 1235 V 17 w(position\(2\))194 1292 y(neigh)p 317 1292 V 17 w(def\(1\))f(=)h(i-1)194 1348 y(neigh)p 317 1348 V 17 w(def\(2\))f(=)h(j)194 1405 y(call)g(MPI)p 389 1405 V 16 w(CART)p 501 1405 V 17 w(RANK)f(\(comm)p 757 1405 V 17 w(cart,)g(neigh)p 1037 1405 V 17 w(def,)g(neigh)p 1293 1405 V 16 w(rank\(1\)\))194 1461 y(neigh)p 317 1461 V 17 w(def\(1\))g(=)h(i+1)194 1518 y(neigh)p 317 1518 V 17 w(def\(2\))f(=)h(j)194 1574 y(call)g(MPI)p 389 1574 V 16 w(CART)p 501 1574 V 17 w(RANK)f(\(comm)p 757 1574 V 17 w(cart,)g(neigh)p 1037 1574 V 17 w(def,)g(neigh)p 1293 1574 V 16 w(rank\(2\)\))194 1630 y(neigh)p 317 1630 V 17 w(def\(1\))g(=)h(i)194 1687 y(neigh)p 317 1687 V 17 w(def\(2\))f(=)h(j-1)194 1743 y(call)g(MPI)p 389 1743 V 16 w(CART)p 501 1743 V 17 w(RANK)f(\(comm)p 757 1743 V 17 w(cart,)g(neigh)p 1037 1743 V 17 w(def,)g(neigh)p 1293 1743 V 16 w(rank\(3\)\))194 1800 y(neigh)p 317 1800 V 17 w(def\(1\))g(=)h(i)194 1856 y(neigh)p 317 1856 V 17 w(def\(2\))f(=)h(j+1)194 1913 y(call)g(MPI)p 389 1913 V 16 w(CART)p 501 1913 V 17 w(RANK)f(\(comm)p 757 1913 V 17 w(cart,)g(neigh)p 1037 1913 V 17 w(def,)g(neigh)p 1293 1913 V 16 w(rank\(4\)\))75 1969 y(C)107 b(Initialize)22 b(the)i(grid)f(functions)g(and)g(start)g(the)h(iteration)194 2026 y(call)g(init)f(\(u,)g(f\))194 2082 y(do)h(10)f(it=1,100)242 2139 y(call)g(relax)h(\(u,)f(f\))75 2195 y(C)107 b(Exchange)22 b(data)i(with)f(neighbor)g(processes)242 2251 y(call)g(exchange)g(\(u,) h(comm)p 771 2251 V 16 w(cart,)f(neigh)p 1050 2251 V 17 w(rank,)g(num)p 1282 2251 V 17 w(neigh\))75 2308 y(10)71 b(continue)194 2364 y(call)24 b(output)f(\(u\))194 2421 y(end)154 2625 y Fx(Figure)16 b(6.1:)j(Set-up)d(of)e(pro)q(cess)i (structure)f(for)f(t)o(w)o(o-dimensional)i(parallel)h(P)o(oisson)e (solv)o(er.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 189 197 189 196 bop 75 356 a Fo(Chapter)34 b(7)75 564 y Fr(MPI)40 b(En)m(vironmen)m(tal)h(Managemen)m(t)75 786 y Fx(This)16 b(c)o(hapter)g(discusses)h(routines)g(for)e(getting)h(and,)f(where)h (appropriate,)g(setting)g(v)m(arious)g(param-)75 843 y(eters)g(that)f(relate)h(to)f(the)h Fv(MPI)f Fx(implemen)o(tation)i (and)f(the)g(execution)g(en)o(vironmen)o(t)h(\(suc)o(h)e(as)h(error)75 899 y(handling\).)21 b(The)13 b(pro)q(cedures)h(for)e(en)o(tering)i (and)f(lea)o(ving)h(the)f Fv(MPI)g Fx(execution)h(en)o(vironmen)o(t)f (are)g(also)75 956 y(describ)q(ed)k(here.)75 1099 y Fn(7.1)59 b(Implementation)17 b(info)n(rmation)75 1202 y Fj(7.1.1)49 b(Environmental)15 b(Inquiries)75 1288 y Fx(A)k(set)g(of)f(attributes)h (that)f(describ)q(e)i(the)f(execution)h(en)o(vironmen)o(t)f(are)g (attac)o(hed)f(to)g(the)h(comm)o(u-)75 1344 y(nicator)f Fi(MPI)p 309 1344 13 2 v 14 w(COMM)p 453 1344 V 15 w(W)o(ORLD)g Fx(when)h Fv(MPI)e Fx(is)i(initialized.)32 b(The)19 b(v)m(alue)g(of)f (these)g(attributes)g(can)g(b)q(e)75 1401 y(inquired)h(b)o(y)d(using)h (the)g(function)h Fv(MPI)p 784 1401 14 2 v 15 w(A)l(TTR)p 916 1401 V 17 w(GET)f Fx(describ)q(ed)i(in)e(Chapter)g(5.)23 b(It)17 b(is)g(erroneous)g(to)75 1457 y(delete)f(these)g(attributes)f (or)f(free)i(their)f(k)o(eys.)166 1514 y(The)g(list)h(of)f (prede\014ned)i(attribute)e(k)o(eys)g(include)75 1608 y Fi(MPI)p 152 1608 13 2 v 14 w(T)m(A)o(G)p 246 1608 V 14 w(UB)22 b Fx(Upp)q(er)16 b(b)q(ound)g(for)f(tag)f(v)m(alue.)75 1701 y Fi(MPI)p 152 1701 V 14 w(HOST)23 b Fx(Host)15 b(pro)q(cess)g(rank,)g(if)h(suc)o(h)f(exists,)g Fi(MPI)p 1041 1701 V 15 w(UNDEFINED)p Fx(,)e(otherwise.)75 1795 y Fi(MPI)p 152 1795 V 14 w(IO)22 b Fx(rank)14 b(of)f(a)g(no)q(de)h (that)f(has)h(regular)g(I/O)g(facilities)i(\(p)q(ossibly)f(m)o (yrank\).)j(No)q(des)c(in)h(the)e(same)189 1852 y(comm)o(unicator)h(ma) o(y)h(return)g(di\013eren)o(t)h(v)m(alues)g(for)e(this)i(parameter.)166 1946 y(V)l(endors)c(ma)o(y)g(add)g(implemen)o(tation)h(sp)q(eci\014c)h (parameters)d(\(suc)o(h)i(as)e(no)q(de)i(n)o(um)o(b)q(er,)g(real)f (mem-)75 2002 y(ory)j(size,)g(virtual)h(memory)f(size,)h(etc.\))166 2058 y(The)f(required)i(parameter)d(v)m(alues)i(are)f(discussed)i(in)f (more)f(detail)h(b)q(elo)o(w:)75 2179 y Fv(T)l(ag)f(values)75 2264 y Fx(T)l(ag)h(v)m(alues)h(range)f(from)f Fu(0)h Fx(to)f(the)i(v)m(alue)g(returned)f(for)g Fi(MPI)p 1162 2264 V 14 w(T)m(A)o(G)p 1256 2264 V 14 w(UB)f Fx(inclusiv)o(e.)26 b(These)16 b(v)m(alues)h(are)75 2321 y(guaran)o(teed)c(to)g(b)q(e)i (unc)o(hanging)f(during)h(the)f(execution)h(of)e(an)h Fv(MPI)f Fx(program.)18 b(In)c(addition,)h(the)f(tag)75 2377 y(upp)q(er)20 b(b)q(ound)g(v)m(alue)g(m)o(ust)f(b)q(e)g Fl(at)h(le)n(ast)j Fx(32767.)30 b(An)19 b Fv(MPI)f Fx(implemen)o (tation)i(is)g(free)f(to)f(mak)o(e)h(the)75 2434 y(v)m(alue)c(of)f Fi(MPI)p 319 2434 V 14 w(T)m(A)o(G)p 413 2434 V 14 w(UB)f Fx(larger)h(than)g(this;)h(for)e(example,)i(the)f(v)m(alue)h(2)1303 2417 y Fg(30)1348 2434 y Ft(\000)8 b Fx(1)14 b(is)h(also)f(a)f(legal)i (v)m(alue)h(for)75 2490 y Fi(MPI)p 152 2490 V 14 w(T)m(A)o(G)p 246 2490 V 14 w(UB)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 190 198 190 197 bop 75 -100 a Fx(190)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fv(Host)g(rank)75 131 y Fx(The)e(v)m(alue)h(returned)g(for)e Fi(MPI)p 612 131 13 2 v 14 w(HOST)i Fx(gets)e(the)h(rank)g(of)f(the)i Fu(HOST)e Fx(pro)q(cess)h(in)h(the)f(group)g(asso)q(ciated)75 187 y(with)22 b(comm)o(unicator)f Fi(MPI)p 565 187 V 14 w(COMM)p 709 187 V 15 w(W)o(ORLD)p Fx(,)g(if)h(there)g(is)g(suc)o (h.)39 b Fi(MPI)p 1343 187 V 15 w(PROC)p 1470 187 V 13 w(NULL)22 b Fx(is)h(returned)e(if)75 244 y(there)16 b(is)h(no)f(host.) 21 b Fv(MPI)16 b Fx(do)q(es)g(not)g(sp)q(ecify)h(what)f(it)g(means)g (for)f(a)h(pro)q(cess)g(to)f(b)q(e)i(a)f Fu(HOST)p Fx(,)f(nor)h(do)q (es)75 300 y(it)f(requires)h(that)f(a)g Fu(HOST)f Fx(exists.)75 420 y Fv(IO)h(rank)75 506 y Fx(The)e(v)m(alue)h(returned)f(for)f Fi(MPI)p 607 506 V 14 w(IO)f Fx(is)j(the)e(rank)h(of)f(a)g(pro)q (cessor)g(that)g(can)h(pro)o(vide)g(language-standard)75 563 y(I/O)i(facilities.)21 b(F)l(or)14 b(F)l(ortran,)e(this)j(means)f (that)f(all)i(of)f(the)g(F)l(ortran)f(I/O)h(op)q(erations)h(are)e(supp) q(orted)75 619 y(\(e.g.,)i Fu(OPEN)p Fx(,)h Fu(REWIND)p Fx(,)f Fu(WRITE)p Fx(\).)g(F)l(or)h(C,)g(this)h(means)g(that)f(all)h (of)f(the)h(ANSI-C)h(I/O)f(op)q(erations)f(are)75 676 y(supp)q(orted)g(\(e.g.,)d Fu(fopen)p Fx(,)h Fu(fprintf)p Fx(,)g Fu(lseek)p Fx(\).)166 732 y(If)i(ev)o(ery)g(pro)q(cess)h(can)f (pro)o(vide)h(language-standard)f(I/O,)g(then)h(the)f(v)m(alue)i Fi(MPI)p 1603 732 V 14 w(ANY)p 1702 732 V 15 w(SOURCE)75 788 y Fx(m)o(ust)10 b(b)q(e)g(returned.)19 b(If)10 b(no)g(pro)q(cess)h (can)f(pro)o(vide)h(language-standard)f(I/O,)g(then)h(the)f(v)m(alue)h Fi(MPI)p 1759 788 V 15 w(PROC)p 1886 788 V 13 w(NULL)75 845 y Fx(m)o(ust)17 b(b)q(e)h(returned.)27 b(If)18 b(sev)o(eral)g(pro)q (cesses)f(can)h(pro)o(vide)g(I/O,)g(then)f(an)o(y)h(them)f(ma)o(y)g(b)q (e)h(returned.)75 901 y(The)d(same)g(v)m(alue)i(\(rank\))d(need)i(not)f (b)q(e)h(returned)f(b)o(y)g(all)i(pro)q(cesses.)75 1052 y Fv(MPI)p 160 1052 14 2 v 16 w(GET)p 264 1052 V 17 w(PROCESSOR)p 540 1052 V 18 w(NAME\()f(len,)f(name)e(\))117 1129 y Fk(IN)155 b Fv(len)533 b Fk(Length)14 b(\(in)g(c)o(haracters\))i(of)d (the)i(space)g(in)e Fb(name)117 1205 y Fk(OUT)108 b Fv(name)483 b Fk(A)17 b(unique)g(sp)q(eci\014er)i(for)e(the)g(actual)g(\(as)g(opp)q (osed)h(to)e(vir-)905 1261 y(tual\))e(no)q(de.)75 1386 y Fu(int)23 b(MPI)p 245 1386 15 2 v 17 w(Get)p 334 1386 V 17 w(processor)p 567 1386 V 16 w(name\(int)g(len,)g(char)g(*name\))75 1472 y(MPI)p 150 1472 V 17 w(GET)p 239 1472 V 17 w(PROCESSOR)p 472 1472 V 15 w(NAME\()h(LEN,)f(NAME,)g(IERROR\))170 1528 y(CHARACTER*\(*\))g(NAME)170 1585 y(INTEGER)g(LEN,IERROR)166 1671 y Fx(The)c(name)g(is)g(a)f(c)o(haracter)g(string)h(for)f(maxim)o (um)h(\015exibilit)o(y)l(.)33 b(F)l(rom)18 b(this)h(v)m(alue)h(it)f(m)o (ust)f(b)q(e)75 1728 y(p)q(ossible)23 b(to)d(iden)o(tify)i(a)e(sp)q (eci\014c)j(piece)f(of)e(hardw)o(are;)j(p)q(ossible)f(v)m(alues)g (include)h(\\pro)q(cessor)e(9)f(in)75 1784 y(rac)o(k)f(4)h(of)f (mpp.cs.org")g(and)h(\\231")e(\(where)i(231)f(is)h(the)g(actual)g(pro)q (cessor)g(n)o(um)o(b)q(er)g(in)h(the)f(run-)75 1841 y(ning)f (homogeneous)f(system\).)29 b(A)18 b(error)g(co)q(de)h(of)e Fi(MPI)p 1062 1841 13 2 v 15 w(UNDEFINED)g Fx(is)i(allo)o(w)o(ed)g(but) f(strongly)g(dis-)75 1897 y(couraged.)31 b(The)19 b(v)m(alue)h(of)e Fi(MPI)p 646 1897 V 14 w(MAX)p 752 1897 V 15 w(PROCESSOR)p 1008 1897 V 13 w(NAME)h Fx(is)g(the)g(maxim)o(um)g(length)g(required)h (b)o(y)75 1954 y Fv(MPI)p 160 1954 14 2 v 16 w(GET)p 264 1954 V 17 w(PROCESSOR)p 540 1954 V 18 w(NAME)d Fx(\(including,)i (in)e(b)q(oth)g(the)g(C)f(and)g(F)l(ortran)g(bindings,)i(the)f(termi-) 75 2010 y(nating)e(n)o(ull\).)21 b(F)l(ortran)14 b(users)g(ma)o(y)h (determine)g(the)g(n)o(um)o(b)q(er)g(of)g(c)o(haracters)f(in)h(the)g (pro)q(cessor)g(name)75 2067 y(b)o(y)g(lo)q(oking)h(for)f(the)g(c)o (haracter)g Fu(char\(0\))p Fx(.)166 2123 y(This)i(routine)f(returns)g (the)g(name)h(of)e(the)i(pro)q(cessor)e(on)i(whic)o(h)g(it)f(w)o(as)f (called)j(at)d(the)i(momen)o(t)75 2179 y(of)g(the)h(call.)28 b(This)18 b(is)g(to)f(allo)o(w)h Fv(MPI)f Fx(implemen)o(tation)i(that)e (do)g(pro)q(cess)h(migration)f(to)g(return)h(the)75 2236 y(curren)o(t)i(pro)q(cessor.)36 b(Note)20 b(that)g(nothing)h(in)h Fv(MPI)e Fl(r)n(e)n(quir)n(es)f Fx(or)h(de\014nes)i(pro)q(cess)f (migration;)i(this)75 2292 y(de\014nition)17 b(of)e Fv(MPI)p 414 2292 V 16 w(GET)p 518 2292 V 16 w(PROCESSOR)p 793 2292 V 19 w(NAME)h Fx(simply)g(allo)o(ws)f(suc)o(h)h(an)f(implemen)o (tation.)166 2349 y(The)i(F)l(ortran)f(binding)j(for)e(this)g(routine)h (do)q(es)f(not)g(ha)o(v)o(e)g(a)g Fv(LEN)g Fx(argumen)o(t)f(b)q(ecause) i(F)l(ortran)75 2405 y(supplies)d(a)e(built-in)j(function)e(\()p Fu(LEN)p Fx(\))f(that)f(pro)o(vides)i(a)f(mec)o(hanism)h(for)f (obtaining)h(the)g(length)g(of)f(the)75 2462 y(c)o(haracter)h(string.) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 191 199 191 198 bop 75 -100 a Fm(7.2.)34 b(ERR)o(OR)17 b(HANDLING)1194 b Fx(191)75 45 y Fn(7.2)59 b(Erro)n(r)21 b(handling)75 147 y Fx(An)14 b Fv(MPI)f Fx(implemen)o(tation)i(cannot)f(or)f(ma)o(y)h (c)o(ho)q(ose)f(not)h(to)f(handle)i(some)f(errors)f(that)g(o)q(ccur)h (during)75 203 y Fv(MPI)e Fx(calls.)21 b(These)13 b(can)g(include)i (errors)e(that)f(generate)h(exceptions)h(or)e(traps,)h(suc)o(h)g(as)g (\015oating)f(p)q(oin)o(t)75 259 y(errors)17 b(or)g(access)g (violations.)27 b(The)18 b(set)f(of)g(errors)g(that)f(are)i(handled)g (b)o(y)g Fv(MPI)f Fx(is)h(implemen)o(tation-)75 316 y(dep)q(enden)o(t.) j(Eac)o(h)15 b(suc)o(h)h(error)e(generates)h(an)g Fq(MPI)i(exception)p Fx(.)166 372 y(A)f(user)h(can)f(asso)q(ciate)h(an)f(error)g(handler)h (with)g(a)f(comm)o(unicator.)23 b(The)16 b(sp)q(eci\014ed)j(error)d (han-)75 429 y(dling)21 b(routine)f(will)h(b)q(e)f(used)h(for)e(an)o(y) g Fv(MPI)g Fx(exception)h(that)f(o)q(ccurs)h(during)h(a)e(call)h(to)f Fv(MPI)g Fx(for)g(a)75 485 y(comm)o(unication)c(with)g(this)g(comm)o (unicator.)k Fv(MPI)14 b Fx(calls)i(that)d(are)h(not)h(related)f(to)g (an)o(y)g(comm)o(unica-)75 542 y(tor)d(are)h(considered)i(to)d(b)q(e)i (attac)o(hed)f(to)f(the)h(comm)o(unicator)g Fi(MPI)p 1230 542 13 2 v 15 w(COMM)p 1375 542 V 14 w(W)o(ORLD)p Fx(.)g(The)g(attac)o(hmen)o(t)75 598 y(of)i(error)g(handlers)i(to)e (comm)o(unicators)g(is)i(purely)f(lo)q(cal:)21 b(di\013eren)o(t)15 b(pro)q(cesses)g(ma)o(y)f(attac)o(h)g(di\013eren)o(t)75 655 y(error)g(handlers)j(to)d(the)i(same)e(comm)o(unicator.)166 711 y(A)j(newly)i(created)e(comm)o(unicator)h(inherits)g(the)g(error)f (handler)h(that)f(is)h(asso)q(ciated)g(with)g(the)75 768 y(\\paren)o(t")h(comm)o(unicator.)32 b(In)20 b(particular,)h(the)f (user)g(can)f(sp)q(ecify)i(a)e(\\global")h(error)f(handler)i(for)75 824 y(all)d(comm)o(unicators)e(b)o(y)h(asso)q(ciating)g(this)g(handler) h(with)f(the)g(comm)o(unicator)f Fi(MPI)p 1570 824 V 14 w(COMM)p 1714 824 V 15 w(W)o(ORLD)75 880 y Fx(immediately)h(after)d (initialization.)166 937 y(Sev)o(eral)i(prede\014ned)g(error)f (handlers)h(are)f(a)o(v)m(ailable)i(in)f Fv(MPI)p Fx(:)75 1026 y Fi(MPI)p 152 1026 V 14 w(ERRORS)p 326 1026 V 14 w(ARE)p 420 1026 V 14 w(F)m(A)m(T)m(AL)21 b Fx(The)15 b(handler,)f(when)h(called,)g(causes)f(the)g(program)f(to)g(ab)q(ort)h (on)g(all)g(exe-)189 1082 y(cuting)f(pro)q(cesses.)20 b(This)13 b(has)g(the)g(same)g(e\013ect)g(as)f(if)i Fv(MPI)p 1203 1082 14 2 v 16 w(ABORT)g Fx(w)o(as)e(called)j(b)o(y)e(the)g(pro)q (cess)189 1139 y(that)h(in)o(v)o(ok)o(ed)i(the)f(handler.)75 1231 y Fi(MPI)p 152 1231 13 2 v 14 w(ERRORS)p 326 1231 V 14 w(RETURN)22 b Fx(The)16 b(handler)g(has)f(no)g(e\013ect.)166 1319 y(Implemen)o(tations)k(ma)o(y)f(pro)o(vide)h(additional)h (prede\014ned)g(error)d(handlers)j(and)e(programmers)75 1376 y(can)d(co)q(de)h(their)g(o)o(wn)f(error)f(handlers.)166 1432 y(The)c(error)g(handler)h Fi(MPI)p 597 1432 V 14 w(ERRORS)p 771 1432 V 14 w(ARE)p 865 1432 V 14 w(F)m(A)m(T)m(AL)e Fx(is)h(asso)q(ciated)h(b)o(y)f(default)h(with)f Fi(MPI)p 1640 1432 V 14 w(COMM)p 1784 1432 V 15 w(W)o(ORLD)75 1489 y Fx(after)k(initialization.)22 b(Th)o(us,)14 b(if)h(the)f(user)h (c)o(ho)q(oses)f(not)g(to)g(con)o(trol)g(error)f(handling,)j(ev)o(ery)e (error)g(that)75 1545 y Fv(MPI)j Fx(handles)i(is)f(treated)g(as)f (fatal.)27 b(Since)19 b(\(almost\))e(all)h Fv(MPI)g Fx(calls)g(return)g (an)g(error)f(co)q(de,)h(a)g(user)75 1602 y(ma)o(y)e(c)o(ho)q(ose)g(to) g(handle)i(errors)d(in)i(its)g(main)g(co)q(de,)g(b)o(y)f(testing)h(the) f(return)g(co)q(de)i(of)d Fv(MPI)h Fx(calls)i(and)75 1658 y(executing)e(a)e(suitable)h(reco)o(v)o(ery)f(co)q(de)h(when)g (the)g(call)g(w)o(as)f(not)g(successful.)21 b(In)15 b(suc)o(h)g(case,)f (the)h(error)75 1715 y(handler)f Fi(MPI)p 314 1715 V 14 w(ERRORS)p 488 1715 V 14 w(RETURN)f Fx(will)h(b)q(e)g(used.)20 b(Usually)14 b(it)f(is)g(more)g(con)o(v)o(enien)o(t)h(and)f(more)f (e\016cien)o(t)75 1771 y(not)i(to)g(test)g(for)g(errors)g(after)g(eac)o (h)g Fv(MPI)g Fx(call,)i(and)e(ha)o(v)o(e)h(suc)o(h)g(error)e(handled)j (b)o(y)f(a)f(non)h(trivial)h Fv(MPI)75 1827 y Fx(error)e(handler.)166 1884 y(After)h(an)g(error)f(is)i(detected,)g(the)f(state)f(of)h Fv(MPI)g Fx(is)h(unde\014ned.)21 b(That)15 b(is,)g(using)h(a)f (user-de\014ned)75 1940 y(error)h(handler,)i(or)e Fi(MPI)p 501 1940 V 14 w(ERRORS)p 675 1940 V 14 w(RETURN)p Fx(,)g(do)q(es)h Fl(not)k Fx(necessarily)d(allo)o(w)f(the)f(user)h(to)f(con)o(tin)o(ue)i (to)75 1997 y(use)c Fv(MPI)g Fx(after)f(an)h(error)f(is)h(detected.)20 b(The)15 b(purp)q(ose)f(of)g(these)g(error)f(handlers)i(is)f(to)g(allo) o(w)g(a)g(user)g(to)75 2053 y(issue)f(user-de\014ned)i(error)d (messages)g(and)g(to)g(tak)o(e)g(actions)h(unrelated)g(to)f Fv(MPI)g Fx(\(suc)o(h)h(as)f(\015ushing)i(I/O)75 2110 y(bu\013ers\))j(b)q(efore)g(a)g(program)f(exits.)27 b(An)17 b Fv(MPI)g Fx(implemen)o(tation)h(is)g(free)f(to)g(allo)o(w)g Fv(MPI)g Fx(to)f(con)o(tin)o(ue)75 2166 y(after)e(an)i(error)e(but)h (is)h(not)f(required)h(to)f(do)g(so.)189 2266 y Fl(A)n(dvic)n(e)i(to)i (implementors.)56 b Fx(A)17 b(go)q(o)q(d)h(qualit)o(y)g(implemen)o (tation)h(will,)h(to)d(the)h(greatest)f(p)q(os-)189 2322 y(sible,)25 b(exten)o(t,)e(circumscrib)q(e)i(the)d(impact)g(of)g(an)g (error,)h(so)f(that)g(normal)g(pro)q(cessing)h(can)189 2378 y(con)o(tin)o(ue)14 b(after)f(an)h(error)g(handler)h(w)o(as)e(in)o (v)o(ok)o(ed.)19 b(The)c(implemen)o(tation)g(do)q(cumen)o(tation)f (will)189 2435 y(pro)o(vide)k(information)g(on)g(the)g(p)q(ossible)i (e\013ect)e(of)f(eac)o(h)h(class)h(of)e(errors.)28 b(\()p Fl(End)18 b(of)h(advic)n(e)g(to)189 2491 y(implementors.)p Fx(\))166 2591 y(An)f Fv(MPI)f Fx(error)g(handler)i(is)f(an)g(opaque)g (ob)s(ject,)f(whic)o(h)i(is)f(accessed)h(b)o(y)e(a)h(handle.)29 b Fv(MPI)17 b Fx(calls)75 2647 y(are)d(pro)o(vided)h(to)f(create)h(new) f(error)g(handlers,)h(to)f(asso)q(ciate)h(error)e(handlers)j(with)f (comm)o(unicators,)75 2704 y(and)g(to)g(test)g(whic)o(h)h(error)e (handler)i(is)g(asso)q(ciated)g(with)f(a)g(comm)o(unicator.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 192 200 192 199 bop 75 -100 a Fx(192)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ERRHANDLER)p 469 45 V 18 w(CREA)l(TE\()g(function,)g(errhandler)f(\))117 122 y Fk(IN)155 b Fv(function)434 b Fk(user)15 b(de\014ned)g(error)g (handling)e(pro)q(cedure)117 197 y(OUT)108 b Fv(errhandler)397 b Fk(MPI)14 b(error)h(handler)f(\(handle\))75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Errhandler)p 502 322 V 16 w(create\(MPI)p 758 322 V 16 w(Handler)p 942 322 V 16 w(function)g(*function,)393 378 y(MPI)p 468 378 V 17 w(Errhandler)f(*errhandler\))75 465 y(MPI)p 150 465 V 17 w(ERRHANDLER)p 407 465 V 16 w(CREATE\(FUNCTION,)f(HANDLER,)i (IERROR\))170 521 y(EXTERNAL)g(FUNCTION)170 578 y(INTEGER)g (ERRHANDLER,)g(IERROR)166 664 y Fx(Register)13 b(the)f(user)h(routine)g Fv(handler)p 805 664 14 2 v 17 w(function)h Fx(for)e(use)h(as)f(an)g Fv(MPI)g Fx(exception)h(handler.)20 b(Returns)75 721 y(in)c Fv(errhandler)f Fx(a)g(handle)i(to)d(the)i(registered)f (exception)h(handler.)189 827 y Fl(A)n(dvic)n(e)g(to)h(implementors.)46 b Fx(The)16 b(handle)h(returned)g(ma)o(y)e(con)o(tain)h(the)h(address)f (of)f(the)i(error)189 883 y(handling)f(routine.)k(Suc)o(h)15 b(call)g(is)g(sup)q(er\015uous)g(in)g(C,)f(whic)o(h)h(has)f(a)g (referencing)i(op)q(erator,)d(but)189 940 y(is)i(necessary)h(in)g(F)l (ortran.)i(\()p Fl(End)e(of)g(advic)n(e)h(to)f(implementors.)p Fx(\))166 1046 y(The)g(user)h(routine)f(should)i(b)q(e)f(a)e(C)h (function)h(of)f(t)o(yp)q(e)h Fi(MPI)p 1224 1046 13 2 v 14 w(Handler)p 1371 1046 V 17 w(function)p Fx(,)g(whic)o(h)g(is)g (de\014ned)75 1103 y(as)75 1209 y Fu(typedef)23 b(void)g (\(MPI_Handler_function\)\(MPI_C)o(omm)e(*,)i(int)h(*,)f(...\);)75 1315 y Fx(The)14 b(\014rst)f(argumen)o(t)f(is)i(the)g(comm)o(unicator)f (in)h(use.)19 b(The)14 b(second)g(is)g(the)f(error)g(co)q(de)h(to)f(b)q (e)h(returned)75 1372 y(b)o(y)19 b(the)g Fv(MPI)g Fx(routine.)32 b(The)19 b(remaining)i(argumen)o(ts)d(are)h(\\)p Fu(stdargs)p Fx(")e(argumen)o(ts)i(whose)g(n)o(um)o(b)q(er)75 1428 y(and)g(meaning)h(is)g(implemen)o(tation-dep)q(enden)o(t.)34 b(An)20 b(implemen)o(tation)g(should)g(clearly)h(do)q(cumen)o(t)75 1484 y(these)15 b(argumen)o(ts.)k(Addresses)d(are)f(used)h(so)f(that)f (the)h(handler)i(ma)o(y)d(b)q(e)i(written)f(in)h(F)l(ortran.)189 1591 y Fl(R)n(ationale.)72 b Fx(The)21 b(v)m(ariable)h(argumen)o(t)e (list)h(is)g(pro)o(vided)h(b)q(ecause)f(it)g(pro)o(vides)g(an)g(ANSI-) 189 1647 y(standard)13 b(ho)q(ok)h(for)g(pro)o(viding)h(additional)g (information)f(to)g(the)g(error)f(handler;)i(without)g(this)189 1704 y(ho)q(ok,)f(ANSI)i(C)f(prohibits)i(additional)f(argumen)o(ts.)j (\()p Fl(End)d(of)g(r)n(ationale.)p Fx(\))75 1904 y Fv(MPI)p 160 1904 14 2 v 16 w(ERRHANDLER)p 469 1904 V 18 w(SET\()f(comm)m(,)d (errhandler)j(\))117 1982 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(to)j(set)h(the)f(error)h(handler)f(for)g (\(handle\))117 2057 y(IN)155 b Fv(errhandler)397 b Fk(new)15 b(MPI)f(error)g(handler)h(for)e(comm)o(unicator)e(\(handle\))75 2181 y Fu(int)23 b(MPI)p 245 2181 15 2 v 17 w(Errhandler)p 502 2181 V 16 w(set\(MPI)p 686 2181 V 16 w(Comm)h(comm,)f(MPI)p 1037 2181 V 17 w(Errhandler)f(errhandler\))75 2268 y(MPI)p 150 2268 V 17 w(ERRHANDLER)p 407 2268 V 16 w(SET\(COMM,)g(ERRHANDLER,)h (IERROR\))170 2324 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)166 2411 y Fx(Asso)q(ciates)16 b(the)g(new)g(error)f(handler)h Fv(erro)o(rhandler)f Fx(with)h(comm)o(unicator)g Fv(comm)9 b Fx(at)15 b(the)h(calling)75 2467 y(pro)q(cess.)k(Note)15 b(that)f(an)i(error)e(handler)i(is)g(alw)o(a)o(ys)f(asso)q(ciated)g (with)g(the)h(comm)o(unicator.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 193 201 193 200 bop 75 -100 a Fm(7.2.)34 b(ERR)o(OR)17 b(HANDLING)1194 b Fx(193)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ERRHANDLER)p 469 45 V 18 w(GET\()15 b(comm)m(,)d(errhandler)j(\))117 122 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(to)j(get)g(the)h(error)g(handler)f(from)e(\(handle\))117 197 y(OUT)108 b Fv(errhandler)397 b Fk(MPI)16 b(error)g(handler)g (curren)o(tly)g(asso)q(ciated)g(with)f(comm)o(u-)905 254 y(nicator)f(\(handle\))75 378 y Fu(int)23 b(MPI)p 245 378 15 2 v 17 w(Errhandler)p 502 378 V 16 w(get\(MPI)p 686 378 V 16 w(Comm)h(comm,)f(MPI)p 1037 378 V 17 w(Errhandler)f (*errhandler\))75 465 y(MPI)p 150 465 V 17 w(ERRHANDLER)p 407 465 V 16 w(GET\(COMM,)g(ERRHANDLER,)h(IERROR\))170 521 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)166 608 y Fx(Returns)16 b(in)g Fv(errhandler)g Fx(\(a)f(handle)i(to\))e(the)g (error)g(handler)i(that)e(is)h(curren)o(tly)g(asso)q(ciated)g(with)75 664 y(comm)o(unicator)f Fv(comm)m Fx(.)166 720 y(Example:)22 b(A)16 b(library)h(function)g(ma)o(y)f(register)g(at)f(its)i(en)o(try)f (p)q(oin)o(t)g(the)g(curren)o(t)g(error)g(handler)75 777 y(for)h(a)h(comm)o(unicator,)f(set)h(its)g(o)o(wn)f(priv)m(ate)i (error)e(handler)i(for)e(this)h(comm)o(unicator,)g(and)g(restore)75 833 y(b)q(efore)d(exiting)i(the)e(previous)h(error)e(handler.)75 984 y Fv(MPI)p 160 984 14 2 v 16 w(ERRHANDLER)p 469 984 V 18 w(FREE\()h(errhandler)g(\))117 1061 y Fk(IN)155 b Fv(errhandler)397 b Fk(MPI)14 b(error)h(handler)f(\(handle\))75 1186 y Fu(int)23 b(MPI)p 245 1186 15 2 v 17 w(Errhandler)p 502 1186 V 16 w(free\(MPI)p 710 1186 V 16 w(Errhandler)g(*errhandler\)) 75 1272 y(MPI)p 150 1272 V 17 w(ERRHANDLER)p 407 1272 V 16 w(FREE\(ERRHANDLER,)e(IERROR\))170 1329 y(INTEGER)i(ERRHANDLER,)g (IERROR)166 1415 y Fx(Marks)14 b(the)h(error)g(handler)h(asso)q(ciated) g(with)f Fv(errhandler)g Fx(for)g(deallo)q(cation)i(and)e(sets)g Fv(errhandler)75 1472 y Fx(to)h Fi(MPI)p 209 1472 13 2 v 14 w(ERRHANDLER)p 492 1472 V 15 w(NULL)p Fx(.)h(The)g(error)f (handler)i(will)g(b)q(e)g(deallo)q(cated)g(after)e(all)h(comm)o (unicators)75 1528 y(asso)q(ciated)e(with)h(it)f(ha)o(v)o(e)g(b)q(een)i (deallo)q(cated.)75 1679 y Fv(MPI)p 160 1679 14 2 v 16 w(ERROR)p 323 1679 V 18 w(STRING\()e(erro)o(rco)q(de,)f(len,)i(string)f (\))117 1756 y Fk(IN)155 b Fv(erro)o(rco)q(de)410 b Fk(Error)15 b(co)q(de)g(returned)g(b)o(y)f(an)f(MPI)i(routine)117 1831 y(IN)155 b Fv(len)533 b Fk(Length)14 b(of)g Fi(string)117 1906 y Fk(OUT)108 b Fv(string)481 b Fk(T)m(ext)14 b(that)g(corresp)q (onds)i(to)e(the)g Fi(erro)o(rco)q(de)75 2031 y Fu(int)23 b(MPI)p 245 2031 15 2 v 17 w(Error)p 382 2031 V 17 w(string\(int)f (errorcode,)h(int)g(len,)g(char)h(*string\))75 2117 y(MPI)p 150 2117 V 17 w(ERROR)p 287 2117 V 16 w(STRING\(ERRORCODE,)e(LEN,)h (STRING,)g(IERROR\))170 2174 y(INTEGER)g(ERRORCODE,)g(LEN,)g(IERROR)170 2230 y(CHARACTER\(*\))g(STRING)166 2317 y Fx(Returns)11 b(the)f(error)f(string)h(asso)q(ciated)h(with)f(an)g(error)g(co)q(de.) 19 b(The)10 b(v)m(alue)h(of)f Fi(MPI)p 1557 2317 13 2 v 14 w(MAX)p 1663 2317 V 15 w(ERROR)p 1815 2317 V 14 w(STRING)75 2373 y Fx(is)j(the)f(maxim)o(um)g(length)h(required)g(b)o (y)f Fv(MPI)p 858 2373 14 2 v 16 w(ERROR)p 1021 2373 V 18 w(STRING)h Fx(\(including,)i(in)e(b)q(oth)f(the)g(C)g(and)g(F)l (or-)75 2429 y(tran)g(bindings,)i(the)f(terminating)g(n)o(ull\).)20 b(F)l(ortran)11 b(users)h(ma)o(y)g(determine)i(the)e(n)o(um)o(b)q(er)h (of)f(c)o(haracters)75 2486 y(in)k(the)f(error)g(string)g(b)o(y)g(lo)q (oking)h(for)f(the)g(c)o(haracter)g Fu(char\(0\))p Fx(.)189 2591 y Fl(R)n(ationale.)38 b Fx(The)13 b(form)f(of)g(this)h(prop)q (osal)g(w)o(as)f(c)o(hosen)h(to)f(mak)o(e)g(the)h(F)l(ortran)f(and)h(C) f(bindings)189 2647 y(similar.)30 b(A)18 b(v)o(ersion)g(that)g(returns) g(a)g(p)q(oin)o(ter)g(to)g(a)g(string)g(has)g(t)o(w)o(o)f (di\016culties.)31 b(First,)18 b(the)189 2704 y(return)10 b(string)h(m)o(ust)g(b)q(e)g(statically)h(allo)q(cated)g(and)f (di\013eren)o(t)g(for)f(eac)o(h)h(error)f(message)g(\(allo)o(wing)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 194 202 194 201 bop 75 -100 a Fx(194)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)189 45 y Fx(the)d(p)q(oin)o(ters)g(returned)g(b)o(y)g(successiv)o(e)h(calls)g (to)e Fi(MPI)p 1117 45 13 2 v 14 w(ERROR)p 1268 45 V 14 w(STRING)h Fx(to)f(p)q(oin)o(t)h(to)f(the)h(correct)189 102 y(message\).)25 b(Second,)19 b(in)f(F)l(ortran,)e(a)i(function)g (declared)g(as)f(returning)h Fi(CHARA)o(CTER*\(*\))e Fx(can)189 158 y(not)e(b)q(e)i(referenced)g(in,)g(for)f(example,)g(a)g Fi(PRINT)g Fx(statemen)o(t.)k(\()p Fl(End)c(of)i(r)n(ationale.)p Fx(\))75 301 y Fn(7.3)59 b(Erro)n(r)21 b(co)r(des)e(and)g(classes)75 403 y Fx(The)13 b(error)f(co)q(des)h(returned)f(b)o(y)h Fv(MPI)f Fx(are)g(left)h(en)o(tirely)g(to)f(the)h(implemen)o(tation)h (\(with)e(the)h(exception)75 459 y(of)h Fi(MPI)p 203 459 V 15 w(SUCCESS)p Fx(\).)e(This)j(is)h(done)f(to)f(allo)o(w)h(an)f (implemen)o(tation)i(to)f(pro)o(vide)g(as)f(m)o(uc)o(h)h(information)75 516 y(as)g(p)q(ossible)i(in)f(the)f(error)g(co)q(de)g(\(for)g(use)g (with)h Fv(MPI)p 1004 516 14 2 v 15 w(ERROR)p 1166 516 V 19 w(STRING)p Fx(\).)166 572 y(T)l(o)10 b(mak)o(e)f(it)i(p)q(ossible) h(for)d(an)h(application)i(to)e(in)o(terpret)g(an)g(error)f(co)q(de,)j (the)e(routine)h Fv(MPI)p 1722 572 V 16 w(ERROR)p 1885 572 V 18 w(CLASS)75 629 y Fx(con)o(v)o(erts)j(an)g(error)g(co)q(de)h (in)o(to)g(one)g(of)f(a)g(small)h(set)g(of)f(sp)q(eci\014ed)j(v)m (alues,)e(called)h Fl(err)n(or)g(classes)p Fx(.)i(V)l(alid)75 685 y(error)c(classes)i(include)344 778 y(MPI)p 436 778 V 16 w(SUCCESS)226 b(No)15 b(error)344 835 y(MPI)p 436 835 V 16 w(ERR)p 549 835 V 17 w(BUFFER)128 b(In)o(v)m(alid)17 b(bu\013er)f(p)q(oin)o(ter)344 891 y(MPI)p 436 891 V 16 w(ERR)p 549 891 V 17 w(COUNT)149 b(In)o(v)m(alid)17 b(coun)o(t)e(argumen)o(t)344 948 y(MPI)p 436 948 V 16 w(ERR)p 549 948 V 17 w(TYPE)189 b(In)o(v)m(alid)17 b(datat)o(yp)q(e)e (argumen)o(t)344 1004 y(MPI)p 436 1004 V 16 w(ERR)p 549 1004 V 17 w(T)l(A)o(G)220 b(In)o(v)m(alid)17 b(tag)e(argumen)o(t)344 1061 y(MPI)p 436 1061 V 16 w(ERR)p 549 1061 V 17 w(COMM)166 b(In)o(v)m(alid)17 b(comm)o(unicator)344 1117 y(MPI)p 436 1117 V 16 w(ERR)p 549 1117 V 17 w(RANK)182 b(In)o(v)m(alid)17 b(rank)344 1174 y(MPI)p 436 1174 V 16 w(ERR)p 549 1174 V 17 w(REQUEST)96 b(In)o(v)m(alid)17 b(request)f(\(handle\))344 1230 y(MPI)p 436 1230 V 16 w(ERR)p 549 1230 V 17 w(R)o(OOT)183 b(In)o(v)m(alid)17 b(ro)q(ot)344 1287 y(MPI)p 436 1287 V 16 w(ERR)p 549 1287 V 17 w(GR)o(OUP)150 b(In)o(v)m(alid)17 b(group)344 1343 y(MPI)p 436 1343 V 16 w(ERR)p 549 1343 V 17 w(OP)252 b(In)o(v)m(alid)17 b(op)q(eration)344 1399 y(MPI)p 436 1399 V 16 w(ERR)p 549 1399 V 17 w(TOPOLOGY)51 b(In)o(v)m(alid)17 b(top)q(ology)344 1456 y(MPI)p 436 1456 V 16 w(ERR)p 549 1456 V 17 w(DIMS)200 b(In)o(v)m(alid)17 b(dimension)g(argumen)o(t)344 1512 y(MPI)p 436 1512 V 16 w(ERR)p 549 1512 V 17 w(AR)o(G)216 b(In)o(v)m(alid)17 b(argumen)o(t)e(of)g(some)f(other)h(kind)344 1569 y(MPI)p 436 1569 V 16 w(ERR)p 549 1569 V 17 w(UNKNO)o(WN)66 b(Unkno)o(wn)15 b(error)344 1625 y(MPI)p 436 1625 V 16 w(ERR)p 549 1625 V 17 w(TR)o(UNCA)l(TE)58 b(Message)15 b(truncated)g(on)g(receiv)o(e)344 1682 y(MPI)p 436 1682 V 16 w(ERR)p 549 1682 V 17 w(OTHER)152 b(Kno)o(wn)15 b(error)g(not)g(in)h(this)f(list)344 1738 y(MPI)p 436 1738 V 16 w(ERR)p 549 1738 V 17 w(INTERN)137 b(In)o(ternal)16 b(MPI)f(error)344 1795 y(MPI)p 436 1795 V 16 w(ERR)p 549 1795 V 17 w(LASTCODE)64 b(Last)15 b(standard)g(error)f (co)q(de)166 1897 y(The)c(di\013erence)i(b)q(et)o(w)o(een)e Fi(MPI)p 700 1897 13 2 v 14 w(ERR)p 793 1897 V 15 w(UNKNO)o(WN)g Fx(and)g Fi(MPI)p 1190 1897 V 15 w(ERR)p 1284 1897 V 14 w(OTHER)g Fx(is)h(that)e Fi(MPI)p 1657 1897 V 14 w(ERROR)p 1808 1897 V 14 w(STRING)75 1954 y Fx(can)15 b(return)g(useful)i (information)e(ab)q(out)g Fi(MPI)p 881 1954 V 14 w(ERR)p 974 1954 V 15 w(OTHER)p Fx(.)166 2010 y(An)j(implemen)o(tation)i(is)f (free)f(to)f(de\014ne)j(more)d(error)h(classes;)i(ho)o(w)o(ev)o(er,)d (these)i(m)o(ust)f(b)q(e)g(used)75 2066 y(where)d(appropriate.)75 2217 y Fv(MPI)p 160 2217 14 2 v 16 w(ERROR)p 323 2217 V 18 w(CLASS\()g(erro)o(rco)q(de,)f(erro)o(rclass)g(\))117 2295 y Fk(IN)155 b Fv(erro)o(rco)q(de)410 b Fk(Error)15 b(co)q(de)g(returned)g(b)o(y)f(an)f(MPI)i(routine)117 2370 y(OUT)108 b Fv(erro)o(rclass)410 b Fk(Error)15 b(class)f(asso)q (ciated)h(with)e Fi(erro)o(rco)q(de)75 2494 y Fu(int)23 b(MPI)p 245 2494 15 2 v 17 w(Error)p 382 2494 V 17 w(class\(int)f (errorcode,)h(int)g(*errorclass\))75 2581 y(MPI)p 150 2581 V 17 w(ERROR)p 287 2581 V 16 w(CLASS\(ERRORCODE,)f(ERRORCLASS,)g (IERROR\))170 2637 y(INTEGER)h(ERRORCODE,)g(ERRORCLASS,)f(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 195 203 195 202 bop 75 -100 a Fm(7.4.)34 b(TIMERS)1446 b Fx(195)75 45 y Fn(7.4)59 b(Timers)75 147 y Fv(MPI)19 b Fx(de\014nes)i(t)o(w)o(o)d (timers.)34 b(These)20 b(timers)f(are)h(sp)q(eci\014ed)i(ev)o(en)e (though)f(they)h(are)f(not)h(\\message-)75 203 y(passing")f(b)q(ecause) g(timing)h(parallel)g(programs)d(is)i(imp)q(ortan)o(t)g(in)g(\\p)q (erformance)f(debugging")i(and)75 259 y(b)q(ecause)d(existing)g(timers) f(\(b)q(oth)g(in)h(POSIX)g(1003.1-1988)d(and)i(1003.4D)e(14.1)h(and)h (in)h(F)l(ortran)e(90\))75 316 y(are)g(either)h(incon)o(v)o(enien)o(t)g (or)f(do)g(not)g(pro)o(vide)h(adequate)f(access)g(to)g(high-resolution) i(timers.)75 467 y Fv(MPI)p 160 467 14 2 v 16 w(WTIME\(\))75 571 y Fu(double)23 b(MPI)p 317 571 15 2 v 17 w(Wtime\(\))75 657 y(DOUBLE)g(PRECISION)g(MPI)p 556 657 V 16 w(WTIME\(\))166 743 y Fx(returns)16 b(a)g(\015oating-p)q(oin)o(t)h(n)o(um)o(b)q(er)g (of)e(seconds,)i(represen)o(ting)g(elapsed)g(w)o(all-clo)q(c)o(k)h (time)f(since)75 800 y(some)e(time)g(in)h(the)g(past.)166 856 y(The)21 b(\\time)g(in)h(the)f(past")g(is)g(guaran)o(teed)g(not)f (to)h(c)o(hange)g(during)h(the)f(life)h(of)f(the)g(pro)q(cess.)75 913 y(The)d(user)g(is)g(resp)q(onsible)h(for)e(con)o(v)o(erting)h (large)f(n)o(um)o(b)q(ers)h(of)f(seconds)h(to)f(other)g(units)i(if)f (they)f(are)75 969 y(preferred.)166 1026 y(These)g(functions)g(are)f(p) q(ortable)h(\(return)f(seconds,)h(not)f("tic)o(ks"\),)g(allo)o(w)h (high-resolution,)h(and)75 1082 y(carry)d(no)g(unnecessary)h(baggage.)j (One)d(w)o(ould)f(use)h(them)f(lik)o(e)h(this:)75 1188 y Fu({)147 1245 y(double)23 b(starttime,)f(endtime;)147 1301 y(starttime)g(=)i(double)f(MPI_Wtime\(\);)170 1358 y(....)48 b(stuff)23 b(to)g(be)h(timed)47 b(...)147 1414 y(endtime)70 b(=)24 b(double)f(MPI_Wtime\(\);)147 1471 y(printf\("That)f(took)h(\045f)h(seconds\\n",endtime-start)o(time\);)75 1527 y(})166 1633 y Fx(The)16 b(times)f(returned)h(b)o(y)g(these)g (routines)f(are)h(lo)q(cal)g(to)f(the)h(no)q(de)g(that)f(called)i (them.)k(There)15 b(is)75 1690 y(no)g(requiremen)o(t)h(that)e (di\013eren)o(t)i(no)q(des)g(return)f(\\the)g(same)g(time.")75 1841 y Fv(MPI)p 160 1841 14 2 v 16 w(WTICK\(\))75 1945 y Fu(double)23 b(MPI)p 317 1945 15 2 v 17 w(Wtick\(\))75 2031 y(DOUBLE)g(PRECISION)g(MPI)p 556 2031 V 16 w(WTICK\(\))166 2117 y Fx(returns)e(the)h(resolution)g(of)f Fv(MPI)p 773 2117 14 2 v 16 w(WTIME)g Fx(in)h(seconds.)40 b(That)21 b(is,)i(it)f(returns,)g(as)f(a)h(double)75 2174 y(precision)c(v)m (alue,)f(the)g(n)o(um)o(b)q(er)g(of)f(seconds)g(b)q(et)o(w)o(een)h (successiv)o(e)h(clo)q(c)o(k)f(tic)o(ks.)23 b(F)l(or)16 b(example,)h(if)g(the)75 2230 y(clo)q(c)o(k)g(is)h(implemen)o(ted)g(b)o (y)f(the)g(hardw)o(are)f(as)g(a)h(coun)o(ter)f(that)g(is)i(incremen)o (ted)g(ev)o(ery)e(millisecond,)75 2287 y(the)f(v)m(alue)i(returned)e(b) o(y)g Fv(MPI)p 603 2287 V 16 w(WTICK)f Fx(should)j(b)q(e)f(10)1031 2270 y Fh(\000)p Fg(3)1077 2287 y Fx(.)75 2430 y Fn(7.5)59 b(Sta)n(rtup)75 2532 y Fx(One)11 b(goal)f(of)g Fv(MPI)f Fx(is)i(to)f(ac)o(hiev)o(e)g Fl(sour)n(c)n(e)i(c)n(o)n(de)f(p)n (ortability)p Fx(.)18 b(By)11 b(this)f(w)o(e)g(mean)g(that)g(a)g (program)f(written)75 2588 y(using)16 b Fv(MPI)e Fx(and)i(complying)g (with)g(the)f(relev)m(an)o(t)h(language)f(standards)g(is)h(p)q(ortable) f(as)g(written,)g(and)75 2644 y(m)o(ust)h(not)g(require)i(an)o(y)e (source)h(co)q(de)g(c)o(hanges)g(when)g(mo)o(v)o(ed)f(from)g(one)h (system)f(to)g(another.)24 b(This)75 2701 y(explicitly)d(do)q(es)e Fl(not)j Fx(sa)o(y)c(an)o(ything)g(ab)q(out)h(ho)o(w)e(an)i Fv(MPI)e Fx(program)h(is)h(started)e(or)h(launc)o(hed)i(from)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 196 204 196 203 bop 75 -100 a Fx(196)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fx(the)g(command)g(line,)h(nor)f(what)f(the)h(user)g(m)o(ust)f(do)h (to)f(set)h(up)g(the)g(en)o(vironmen)o(t)g(in)h(whic)o(h)g(an)f Fv(MPI)75 102 y Fx(program)h(will)j(run.)28 b(Ho)o(w)o(ev)o(er,)18 b(an)g(implemen)o(tation)h(ma)o(y)f(require)g(some)g(setup)h(to)e(b)q (e)i(p)q(erformed)75 158 y(b)q(efore)14 b(other)g Fv(MPI)f Fx(routines)h(ma)o(y)g(b)q(e)g(called.)21 b(T)l(o)14 b(pro)o(vide)g(for)f(this,)i Fv(MPI)e Fx(includes)j(an)e (initialization)75 214 y(routine)i Fv(MPI)p 317 214 14 2 v 15 w(INIT)p Fx(.)75 365 y Fv(MPI)p 160 365 V 16 w(INIT\(\))75 469 y Fu(int)23 b(MPI)p 245 469 15 2 v 17 w(Init\(int)g(*argc,)g(char)g (***argv\))75 556 y(MPI)p 150 556 V 17 w(INIT\(IERROR\))170 612 y(INTEGER)g(IERROR)166 698 y Fx(This)17 b(routine)h(m)o(ust)e(b)q (e)i(called)g(b)q(efore)f(an)o(y)g(other)f Fv(MPI)h Fx(routine.)25 b(It)17 b(m)o(ust)f(b)q(e)i(called)h(at)d(most)75 755 y(once;)f(subsequen)o(t)h(calls)g(are)f(erroneous)g(\(see)g Fv(MPI)p 982 755 14 2 v 16 w(INITIALIZED)p Fx(\).)166 811 y(All)k Fv(MPI)d Fx(programs)g(m)o(ust)h(con)o(tain)h(a)f(call)h (to)f Fv(MPI)p 1089 811 V 16 w(init)p Fx(;)h(this)g(routine)g(m)o(ust)f (b)q(e)h(called)h(b)q(efore)75 868 y(an)o(y)f(other)g Fv(MPI)f Fx(routine)i(\(apart)e(from)g Fu(MPI)p 872 868 15 2 v 17 w(INITIALIZED)p Fx(\))f(is)j(called.)30 b(The)18 b(v)o(ersion)h(for)e(ANSI)i(C)75 924 y(accepts)c(the)h Fi(a)o(rgc)f Fx(and)g Fi(a)o(rgv)g Fx(that)g(are)g(pro)o(vided)h(b)o(y) f(the)g(argumen)o(ts)f(to)h Fu(main)p Fx(:)75 1018 y Fu(MPI_init\()23 b(argc,)g(argv)g(\);)75 1112 y Fx(The)15 b(F)l(ortran)f(v)o(ersion)i(tak)o(es)e(only)i Fi(IERROR)p Fx(.)75 1263 y Fv(MPI)p 160 1263 14 2 v 16 w(FINALIZE\(\))75 1367 y Fu(int)23 b(MPI)p 245 1367 15 2 v 17 w(Finalize\(\))75 1453 y(MPI)p 150 1453 V 17 w(FINALIZE\(IERROR\))170 1509 y(INTEGER)g(IERROR)166 1596 y Fx(This)18 b(routines)g(cleans)g(all)h Fv(MPI)d Fx(state.)26 b(Once)19 b(this)f(routine)g(is)g(called,)h(no)e Fv(MPI)g Fx(routine)h(\(ev)o(en)75 1652 y Fv(MPI)p 160 1652 14 2 v 16 w(INIT)p Fx(\))12 b(ma)o(y)h(b)q(e)h(called.)21 b(The)14 b(user)g(m)o(ust)f(ensure)h(that)f(all)h(p)q(ending)i(comm)o (unications)e(in)o(v)o(olving)75 1709 y(a)h(pro)q(cess)g(complete)h(b)q (efore)g(the)f(pro)q(cess)g(calls)i Fv(MPI)p 1023 1709 V 16 w(FINALIZE)p Fx(.)75 1860 y Fv(MPI)p 160 1860 V 16 w(INITIALIZED\()c(\015ag)i(\))117 1937 y Fk(OUT)108 b Fv(\015ag)518 b Fk(F)p Fx(lag)15 b(is)f(true)h(if)f Fv(MPI)p 1270 1937 V 16 w(INIT)g Fx(has)g(b)q(een)h(called)h(and)f (false)905 1993 y(otherwise.)75 2118 y Fu(int)23 b(MPI)p 245 2118 15 2 v 17 w(Initialized\(int)f(*flag\))75 2204 y(MPI)p 150 2204 V 17 w(INITIALIZED\(FLAG,)f(IERROR\))170 2261 y(LOGICAL)i(FLAG)170 2317 y(INTEGER)g(IERROR)166 2404 y Fx(This)c(routine)f(ma)o(y)g(b)q(e)h(used)g(to)e(determine)i (whether)g Fv(MPI)p 1242 2404 14 2 v 16 w(INIT)e Fx(has)h(b)q(een)h (called.)31 b(It)18 b(is)h(the)75 2460 y Fl(only)c Fx(routine)g(that)g (ma)o(y)f(b)q(e)i(called)h(b)q(efore)e Fv(MPI)p 938 2460 V 16 w(INIT)g Fx(is)g(called.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 197 205 197 204 bop 75 -100 a Fm(7.5.)34 b(ST)l(AR)l(TUP)1411 b Fx(197)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ABORT\()16 b(comm)m(,)c(erro)o(rco)q(de)i(\))117 122 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(of)j(tasks)g(to)g(ab)q(ort)117 197 y(IN)155 b Fv(erro)o(rco)q(de)410 b Fk(error)15 b(co)q(de)g(to)e (return)j(to)d(in)o(v)o(oking)f(en)o(vironmen)o(t)75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Abort\(MPI)p 478 322 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 408 y(MPI)p 150 408 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 465 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)166 551 y Fx(This)d(routine)f(mak)o(es)g(a)g(\\b)q(est)g(attempt")f(to)h(ab)q (ort)g(all)h(tasks)e(in)i(the)g(group)e(of)h Fv(comm)m Fx(.)26 b(This)75 608 y(function)21 b(do)q(es)g(not)f(require)h(that)f (the)g(in)o(v)o(oking)h(en)o(vironmen)o(t)f(tak)o(e)g(an)o(y)g(action)h (with)f(the)h(error)75 664 y(co)q(de.)g(Ho)o(w)o(ev)o(er,)14 b(a)h(Unix)i(or)e(POSIX)h(en)o(vironmen)o(t)g(should)g(handle)h(this)f (as)f(a)g Fu(return)23 b(errorcode)75 721 y Fx(from)14 b(the)i(main)f(program)f(or)h(an)g Fu(abort\(errorcode\))p Fx(.)166 777 y Fv(MPI)f Fx(implemen)o(tations)i(are)e(required)i(to)d (de\014ne)j(the)f(b)q(eha)o(vior)g(of)f Fv(MPI)p 1441 777 14 2 v 16 w(ABORT)i Fx(at)e(least)g(for)g(a)75 834 y Fu(comm)i Fx(of)g Fi(MPI)p 317 834 13 2 v 14 w(COMM)p 461 834 V 15 w(W)o(ORLD)p Fx(.)f Fv(MPI)h Fx(implemen)o(tations)i(ma)o (y)d(ignore)i(the)g Fu(comm)e Fx(argumen)o(t)h(and)g(act)75 890 y(as)f(if)g(the)h Fu(comm)e Fx(w)o(as)h Fi(MPI)p 526 890 V 14 w(COMM)p 670 890 V 15 w(W)o(ORLD)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 198 206 198 205 bop 75 356 a Fo(Chapter)34 b(8)75 564 y Fr(Pro\014ling)42 b(In)m(terface)75 805 y Fn(8.1)59 b(Requirements)75 906 y Fx(T)l(o)15 b(meet)g(the)g Fv(MPI)g Fx(pro\014ling)h(in)o(terface,)f (an)h(implemen)o(tation)g(of)f(the)g Fv(MPI)g Fx(functions)h Fl(must)131 997 y Fx(1.)22 b(pro)o(vide)14 b(a)g(mec)o(hanism)g (through)g(whic)o(h)h(all)g(of)e(the)h Fv(MPI)f Fx(de\014ned)j (functions)f(ma)o(y)e(b)q(e)h(accessed)189 1054 y(with)e(a)h(name)f (shift.)19 b(Th)o(us)13 b(all)g(of)f(the)h Fv(MPI)f Fx(functions)h (\(whic)o(h)g(normally)g(start)e(with)i(the)g(pre\014x)189 1110 y(\\)p Fu(MPI)p 287 1110 15 2 v 16 w Fx("\))i(should)h(also)f(b)q (e)h(accessible)h(with)e(the)h(pre\014x)f(\\)p Fu(PMPI)p 1289 1110 V 17 w Fx(".)131 1203 y(2.)22 b(ensure)d(that)e(those)i Fv(MPI)e Fx(functions)j(whic)o(h)f(are)f(not)g(replaced)i(ma)o(y)e (still)h(b)q(e)h(link)o(ed)g(in)o(to)e(an)189 1260 y(executable)e (image)f(without)h(causing)f(name)h(clashes.)131 1352 y(3.)22 b(do)q(cumen)o(t)c(the)h(implemen)o(tation)g(of)f(di\013eren)o (t)h(language)f(bindings)i(of)e(the)g Fv(MPI)g Fx(in)o(terface)g(if)189 1409 y(they)f(are)h(la)o(y)o(ered)g(on)f(top)g(of)h(eac)o(h)f(other,)h (so)f(that)g(the)h(pro\014ler)g(dev)o(elop)q(er)h(kno)o(ws)e(whether) 189 1465 y(she)d(m)o(ust)f(implemen)o(t)i(the)e(pro\014le)i(in)o (terface)f(for)f(eac)o(h)h(binding,)h(or)e(can)h(economise)h(b)o(y)e (imple-)189 1522 y(men)o(ting)i(it)h(only)f(for)g(the)g(lo)o(w)o(est)g (lev)o(el)h(routines.)131 1615 y(4.)22 b(where)15 b(the)g(implemen)o (tation)i(of)d(di\013eren)o(t)i(language)f(bindings)i(is)f(is)g(done)f (through)g(a)g(la)o(y)o(ered)189 1671 y(approac)o(h)h(\(e.g.)24 b(the)17 b(F)l(ortran)f(binding)j(is)e(a)f(set)h(of)g(\\wrapp)q(er")f (functions)i(whic)o(h)f(call)h(the)f(C)189 1727 y(implemen)o(tation\),) g(ensure)g(that)f(these)g(wrapp)q(er)h(functions)g(are)f(separable)h (from)f(the)g(rest)g(of)189 1784 y(the)f(library)l(.)189 1859 y(This)k(is)h(necessary)f(to)f(allo)o(w)h(a)g(separate)f (pro\014ling)j(library)e(to)g(b)q(e)g(correctly)h(implemen)o(ted,)189 1915 y(since)g(\(at)f(least)g(with)h(Unix)g(link)o(er)h(seman)o(tics\)) e(the)h(pro\014ling)g(library)g(m)o(ust)f(con)o(tain)h(these)189 1971 y(wrapp)q(er)14 b(functions)h(if)g(it)f(is)h(to)f(p)q(erform)g(as) f(exp)q(ected.)21 b(This)15 b(requiremen)o(t)g(allo)o(ws)f(the)h(p)q (erson)189 2028 y(who)c(builds)i(the)e(pro\014ling)h(library)g(to)f (extract)f(these)i(functions)g(from)e(the)h(original)i Fv(MPI)d Fx(library)189 2084 y(and)k(add)g(them)g(in)o(to)g(the)f (pro\014ling)j(library)e(without)g(bringing)h(along)f(an)o(y)g(other)f (unnecessary)189 2141 y(co)q(de.)131 2234 y(5.)22 b(pro)o(vide)15 b(a)g(no-op)h(routine)f Fv(MPI)p 756 2234 14 2 v 16 w(PCONTROL)h Fx(in)g(the)g Fv(MPI)e Fx(library)l(.)75 2376 y Fn(8.2)59 b(Discussion)75 2478 y Fx(The)14 b(ob)s(jectiv)o(e)f(of)g(the)h Fv(MPI)f Fx(pro\014ling)h(in)o(terface)g(is)g(to)f(ensure)h(that)f(it)h (is)g(relativ)o(ely)g(easy)g(for)f(authors)75 2534 y(of)19 b(pro\014ling)h(\(and)f(other)g(similar\))h(to)q(ols)g(to)e(in)o (terface)i(their)f(co)q(des)h(to)f Fv(MPI)g Fx(implemen)o(tations)h(on) 75 2591 y(di\013eren)o(t)15 b(mac)o(hines.)166 2647 y(Since)20 b Fv(MPI)f Fx(is)g(a)g(mac)o(hine)h(indep)q(enden)o(t)h(standard)e (with)g(man)o(y)g(di\013eren)o(t)g(implemen)o(tations,)75 2704 y(it)e(is)g(unreasonable)g(to)f(exp)q(ect)i(that)e(the)g(authors)g (of)g(pro\014ling)i(to)q(ols)f(for)e Fv(MPI)h Fx(will)j(ha)o(v)o(e)d (access)h(to)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 199 207 199 206 bop 75 -100 a Fm(8.3.)34 b(LOGIC)16 b(OF)f(THE)g(DESIGN)1092 b Fx(199)75 45 y(the)14 b(source)g(co)q(de)g(whic)o(h)h(implemen)o(ts)g Fv(MPI)e Fx(on)h(an)o(y)f(particular)i(mac)o(hine.)20 b(It)14 b(is)g(therefore)g(necessary)75 102 y(to)21 b(pro)o(vide)i(a)f (mec)o(hanism)g(b)o(y)g(whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o (h)g(to)q(ols)g(can)g(collect)i(whatev)o(er)75 158 y(p)q(erformance)15 b(information)h(they)f(wish)h Fl(without)g Fx(access)f(to)g(the)g (underlying)i(implemen)o(tation.)166 214 y(W)l(e)e(b)q(eliev)o(e)h (that)e(ha)o(ving)h(suc)o(h)g(an)f(in)o(terface)h(is)g(imp)q(ortan)o(t) f(if)i Fv(MPI)d Fx(is)j(to)e(b)q(e)h(attractiv)o(e)e(to)h(end)75 271 y(users,)h(since)i(the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g (di\013eren)o(t)h(to)q(ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h (factor)f(in)h(attracting)75 327 y(users)f(to)g(the)g Fv(MPI)g Fx(standard.)166 384 y(The)e(pro\014ling)h(in)o(terface)e(is)h (just)g(that,)f(an)g(in)o(terface.)20 b(It)12 b(sa)o(ys)g Fl(nothing)g Fx(ab)q(out)g(the)h(w)o(a)o(y)f(in)h(whic)o(h)75 440 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f (to)g(la)o(y)h(do)o(wn)g(what)g(information)g(is)h(collected)g(through) 75 497 y(the)g(in)o(terface,)g(or)g(ho)o(w)g(the)g(collected)i (information)e(is)h(sa)o(v)o(ed,)e(\014ltered,)i(or)f(displa)o(y)o(ed.) 166 553 y(While)g(the)g(initial)h(imp)q(etus)f(for)f(the)g(dev)o (elopmen)o(t)h(of)e(this)i(in)o(terface)f(arose)g(from)f(the)h(desire)i (to)75 610 y(p)q(ermit)g(the)f(implemen)o(tation)i(of)d(pro\014ling)j (to)q(ols,)d(it)i(is)f(clear)h(that)f(an)g(in)o(terface)g(lik)o(e)h (that)f(sp)q(eci\014ed)75 666 y(ma)o(y)j(also)g(pro)o(v)o(e)g(useful)h (for)f(other)g(purp)q(oses,)i(suc)o(h)e(as)h(\\in)o(ternet)o(w)o (orking")e(m)o(ultiple)j Fv(MPI)e Fx(imple-)75 723 y(men)o(tations.)j (Since)c(all)f(that)f(is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there) h(is)g(no)f(ob)s(jection)h(to)f(its)h(b)q(eing)g(used)75 779 y(wherev)o(er)f(it)h(is)f(useful.)166 835 y(As)j(the)g(issues)h(b)q (eing)g(addressed)f(here)h(are)f(in)o(timately)h(tied)f(up)h(with)f (the)g(w)o(a)o(y)f(in)i(whic)o(h)g(ex-)75 892 y(ecutable)j(images)e (are)g(built,)j(whic)o(h)e(ma)o(y)f(di\013er)h(greatly)f(on)g (di\013eren)o(t)h(mac)o(hines,)h(the)e(examples)75 948 y(giv)o(en)d(b)q(elo)o(w)f(should)i(b)q(e)e(treated)g(solely)h(as)f (one)g(w)o(a)o(y)f(of)h(implemen)o(ting)i(the)e(ob)s(jectiv)o(e)h(of)e (the)i Fv(MPI)75 1005 y Fx(pro\014ling)g(in)o(terface.)k(The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o(tation)h(are)e (those)g(detailed)75 1061 y(in)i(the)f(Requiremen)o(ts)i(section)e(ab)q (o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this)i(c)o(hapter)f (is)g(only)h(presen)o(t)f(as)75 1118 y(justi\014cation)g(and)f (discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.)166 1174 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f(in)h (whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed)75 1231 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 1287 y(equally)d(v)m(alid\).)75 1428 y Fn(8.3)59 b(Logic)20 b(of)g(the)f(design)75 1529 y Fx(Pro)o(vided)d(that)g(an)g Fv(MPI)f Fx(implemen)o(tation)i(meets)f(the)g(requiremen)o(ts)h(ab)q(o) o(v)o(e,)e(it)h(is)h(p)q(ossible)h(for)d(the)75 1586 y(implemen)o(tor)20 b(of)f(the)h(pro\014ling)g(system)f(to)g(in)o (tercept)h(all)g(of)f(the)h Fv(MPI)f Fx(calls)h(whic)o(h)g(are)g(made)f (b)o(y)75 1642 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h (collect)g(whatev)o(er)f(information)g(she)h(requires)g(b)q(efore)f (calling)75 1698 y(the)d(underlying)h Fv(MPI)e Fx(implemen)o(tation)h (\(through)f(its)h(name)f(shifted)h(en)o(try)f(p)q(oin)o(ts\))h(to)f (ac)o(hiev)o(e)h(the)75 1755 y(desired)g(e\013ects.)75 1874 y Fj(8.3.1)49 b(Miscellaneous)17 b(control)g(of)f(p)o(ro\014ling) 75 1960 y Fx(There)e(is)g(a)g(clear)g(requiremen)o(t)h(for)e(the)h (user)g(co)q(de)g(to)f(b)q(e)i(able)f(to)f(con)o(trol)h(the)g (pro\014ler)g(dynamically)75 2016 y(at)h(run)g(time.)20 b(This)c(is)g(normally)g(used)f(for)g(\(at)f(least\))h(the)g(purp)q (oses)h(of)143 2096 y Ft(\017)23 b Fx(Enabling)16 b(and)g(disabling)h (pro\014ling)f(dep)q(ending)i(on)d(the)g(state)f(of)h(the)g (calculation.)143 2184 y Ft(\017)23 b Fx(Flushing)16 b(trace)f(bu\013ers)g(at)g(non-critical)i(p)q(oin)o(ts)e(in)h(the)g (calculation)143 2272 y Ft(\017)23 b Fx(Adding)16 b(user)f(ev)o(en)o (ts)g(to)g(a)g(trace)f(\014le.)166 2351 y(These)h(requiremen)o(ts)h (are)f(met)g(b)o(y)g(use)h(of)e(the)i Fv(MPI)p 1092 2351 14 2 v 15 w(PCONTROL)p Fx(.)75 2502 y Fv(MPI)p 160 2502 V 16 w(PCONTROL\(level,)f Fp(:)8 b(:)g(:)e Fv(\))117 2579 y Fk(IN)155 b Fv(level)504 b Fk(Pro\014ling)13 b(lev)o(el)75 2704 y Fu(int)23 b(MPI)p 245 2704 15 2 v 17 w(Pcontrol\(const)f(int)i (level,)f Fp(:)8 b(:)g(:)d Fu(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 200 208 200 207 bop 75 -100 a Fx(200)873 b Fm(CHAPTER)15 b(8.)35 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(PCONTROL\(level\))170 102 y(INTEGER)23 b(LEVEL,)g Fp(:)8 b(:)g(:)166 188 y Fv(MPI)16 b Fx(libraries)j (themselv)o(es)f(mak)o(e)e(no)h(use)h(of)e(this)i(routine,)g(and)f (simply)h(return)f(immediately)75 244 y(to)d(the)g(user)h(co)q(de.)20 b(Ho)o(w)o(ev)o(er)13 b(the)i(presence)g(of)f(calls)i(to)d(this)i (routine)g(allo)o(ws)g(a)f(pro\014ling)h(pac)o(k)m(age)g(to)75 301 y(b)q(e)h(explicitly)i(called)f(b)o(y)e(the)g(user.)166 357 y(Since)k Fv(MPI)f Fx(has)g(no)g(con)o(trol)g(of)f(the)i(implemen)o (tation)g(of)f(the)g(pro\014ling)h(co)q(de,)g(w)o(e)f(are)g(unable)75 414 y(to)13 b(sp)q(ecify)i(precisely)h(the)e(seman)o(tics)g(whic)o(h)h (will)g(b)q(e)g(pro)o(vided)f(b)o(y)g(calls)h(to)e Fv(MPI)p 1505 414 14 2 v 16 w(PCONTROL)p Fx(.)h(This)75 470 y(v)m(agueness)i (extends)f(to)g(the)g(n)o(um)o(b)q(er)h(of)f(argumen)o(ts)f(to)h(the)g (function,)g(and)h(their)g(datat)o(yp)q(es.)166 527 y(Ho)o(w)o(ev)o(er) 11 b(to)g(pro)o(vide)h(some)g(lev)o(el)h(of)e(p)q(ortabilit)o(y)i(of)e (user)h(co)q(des)h(to)e(di\013eren)o(t)h(pro\014ling)h(libraries,)75 583 y(w)o(e)i(request)g(the)g(follo)o(wing)h(meanings)g(for)f(certain)g (v)m(alues)i(of)d(lev)o(el.)143 670 y Ft(\017)23 b Fu(level==0)14 b Fx(Pro\014ling)i(is)g(disabled.)143 761 y Ft(\017)23 b Fu(level==1)14 b Fx(Pro\014ling)i(is)g(enabled)g(at)f(a)g(normal)g (default)h(lev)o(el)g(of)f(detail.)143 852 y Ft(\017)23 b Fu(level==2)14 b Fx(Pro\014le)h(bu\013ers)g(are)g(\015ushed.)21 b(\(This)16 b(ma)o(y)e(b)q(e)i(a)f(no-op)g(in)h(some)f(pro\014lers\).) 143 943 y Ft(\017)23 b Fx(All)13 b(other)f(v)m(alues)h(of)f Fu(level)f Fx(ha)o(v)o(e)h(pro\014le)h(library)g(de\014ned)g(e\013ects) f(and)h(additional)g(argumen)o(ts.)166 1030 y(W)l(e)18 b(also)f(request)h(that)f(the)h(default)g(state)f(after)g Fv(MPI)p 1147 1030 V 16 w(INIT)g Fx(has)g(b)q(een)i(called)g(is)g(for)e (pro\014ling)75 1087 y(to)e(b)q(e)h(enabled)h(at)e(the)h(normal)f (default)h(lev)o(el.)22 b(\(i.e.)f(as)15 b(if)h Fv(MPI)p 1204 1087 V 16 w(PCONTROL)h Fx(had)e(just)h(b)q(een)g(called)75 1143 y(with)h(the)f(argumen)o(t)g(1\).)23 b(This)17 b(allo)o(ws)f (users)h(to)e(link)j(with)f(a)f(pro\014ling)i(library)f(and)g(obtain)f (pro\014le)75 1200 y(output)f(without)g(ha)o(ving)h(to)e(mo)q(dify)i (their)g(source)f(co)q(de)h(at)e(all.)166 1256 y(The)h(pro)o(vision)g (of)f Fv(MPI)p 589 1256 V 16 w(PCONTROL)i Fx(as)e(a)g(no-op)h(in)g(the) g(standard)f Fv(MPI)g Fx(library)h(allo)o(ws)g(them)75 1313 y(to)g(mo)q(dify)h(their)g(source)f(co)q(de)h(to)f(obtain)h(more)f (detailed)i(pro\014ling)f(information,)g(but)f(still)i(b)q(e)f(able)75 1369 y(to)f(link)h(exactly)g(the)f(same)g(co)q(de)h(against)f(the)g (standard)g Fv(MPI)f Fx(library)l(.)75 1511 y Fn(8.4)59 b(Examples)75 1614 y Fj(8.4.1)49 b(Pro\014ler)16 b(implem)o(ent)o (ation)75 1700 y Fx(Supp)q(ose)25 b(that)e(the)h(pro\014ler)h(wishes)g (to)e(accum)o(ulate)i(the)f(total)f(amoun)o(t)g(of)h(data)f(sen)o(t)h (b)o(y)g(the)75 1756 y(MPI)p 167 1756 V 16 w(SEND)18 b(function,)g(along)f(with)h(the)f(total)g(elapsed)h(time)f(sp)q(en)o (t)h(in)g(the)f(function.)27 b(This)18 b(could)75 1813 y(trivially)f(b)q(e)f(ac)o(hiev)o(ed)g(th)o(us)75 1909 y Fu(static)23 b(int)g(totalBytes;)75 1966 y(static)g(double)g (totalTime;)75 2079 y(int)g(MPI_SEND\(void)g(*)g(buffer,)g(const)g(int) h(count,)f(MPI_Datatype)f(datatype,)385 2135 y(int)i(dest,)f(int)g (tag,)h(MPI_comm)e(comm\))75 2192 y({)147 2248 y(double)h(tstart)g(=)g (MPI_Wtime\(\);)166 b(/*)23 b(Pass)h(on)f(all)h(the)f(arguments)g(*/) 147 2305 y(int)g(extent;)147 2361 y(int)g(result)95 b(=)23 b(PMPI_Send\(buffer,count,dataty)o(pe,dest)o(,tag,com)o(m\);)147 2474 y(totalBytes)f(+=)i(count)f(*)h(MPI_Type_size\(datatype,)o (&extent)o(\);)d(/*)j(Accumulate)e(byte)h(count)h(*/)147 2530 y(totalTime)46 b(+=)24 b(MPI_Wtime\(\))e(-)i(tstart;)214 b(/*)23 b(and)h(time)333 b(*/)147 2643 y(return)23 b(result;)75 2700 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 201 209 201 208 bop 75 -100 a Fm(8.4.)29 b(EXAMPLES)1375 b Fx(201)75 45 y Fj(8.4.2)49 b(MPI)17 b(lib)o(ra)o(ry)f(implem)o(ent)o(ation)75 131 y Fx(On)i(a)g(Unix)g(system,)g(in)h(whic)o(h)f(the)g Fv(MPI)f Fx(library)i(is)f(implemen)o(ted)h(in)g(C,)e(then)h(there)g (are)f(v)m(arious)75 187 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i (t)o(w)o(o)e(of)g(the)h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.) 35 b(Whic)o(h)20 b(is)h(b)q(etter)75 244 y(dep)q(ends)c(on)e(whether)g (the)h(link)o(er)g(and)f(compiler)i(supp)q(ort)e(w)o(eak)g(sym)o(b)q (ols.)75 364 y Fv(Systems)f(with)i(w)o(eak)f(symb)q(ols)75 450 y Fx(If)k(the)f(compiler)i(and)f(link)o(er)h(supp)q(ort)f(w)o(eak)f (external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g (system)75 506 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h (library)g(is)f(required)h(through)f(the)g(use)g(of)g Fu(#pragma)23 b(weak)13 b Fx(th)o(us)75 612 y Fu(#pragma)23 b(weak)g(MPI_Example)g(=)g(PMPI_Example)75 725 y(int)g (PMPI_Example\(/*)f(appropriate)h(args)g(*/\))75 782 y({)170 838 y(/*)h(Useful)f(content)g(*/)75 895 y(})166 1001 y Fx(The)d(e\013ect)g(of)g(this)g Fu(#pragma)f Fx(is)i(to)e (de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Fu(MPI)p 1472 1001 15 2 v 17 w(Example)f Fx(as)h(a)g(w)o(eak)75 1057 y(de\014nition.)h(This)15 b(means)f(that)g(the)g(link)o(er)h(will) h(not)e(complain)i(if)e(there)h(is)f(another)g(de\014nition)i(of)e(the) 75 1114 y(sym)o(b)q(ol)h(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1170 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1290 y Fv(Systems)f(without)j(w)o(eak)e(symb)q(ols) 75 1376 y Fx(In)k(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h(then)g (one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h(the)f(C)h (macro)75 1432 y(pre-pro)q(cessor)c(th)o(us)75 1538 y Fu(#ifdef)23 b(PROFILELIB)75 1595 y(#)95 b(ifdef)23 b(__STDC__)75 1651 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1708 y(#)95 b(else)75 1764 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 1821 y(#)95 b(endif)75 1877 y(#else)75 1934 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 1990 y(#endif)166 2096 y Fx(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2202 y Fu(int)23 b(FUNCTION\(MPI_Example\)\(/*)e (appropriate)h(args)i(*/\))75 2259 y({)170 2315 y(/*)g(Useful)f (content)g(*/)75 2372 y(})166 2478 y Fx(The)d(same)g(source)g(\014le)h (can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q(duce)i(b)q(oth)f(v)o (ersions)g(of)g(the)g(library)l(,)75 2534 y(dep)q(ending)d(on)f(the)f (state)f(of)h(the)g Fu(PROFILELIB)f Fx(macro)h(sym)o(b)q(ol.)166 2591 y(It)e(is)h(required)g(that)f(the)g(standard)g Fv(MPI)f Fx(library)i(b)q(e)g(built)h(in)f(suc)o(h)f(a)g(w)o(a)o(y)f(that)g(the) i(inclusion)h(of)75 2647 y Fv(MPI)g Fx(functions)h(can)f(b)q(e)h(ac)o (hiev)o(ed)h(one)e(at)g(a)g(time.)20 b(This)c(is)g(a)f(somewhat)g (unpleasan)o(t)h(requiremen)o(t,)75 2704 y(since)22 b(it)f(ma)o(y)f (mean)h(that)f(eac)o(h)g(external)i(function)f(has)g(to)f(b)q(e)h (compiled)i(from)d(a)g(separate)g(\014le.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 202 210 202 209 bop 75 -100 a Fx(202)873 b Fm(CHAPTER)15 b(8.)35 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)75 45 y Fx(Ho)o(w)o(ev)o(er)14 b(this)i(is)g(necessary)g(so)f(that)g(the)g(author)g(of)g(the)h (pro\014ling)g(library)g(need)h(only)f(de\014ne)g(those)75 102 y Fv(MPI)f Fx(functions)g(whic)o(h)h(she)g(wishes)g(to)e(in)o (tercept,)i(references)g(to)e(an)o(y)h(others)g(b)q(eing)h(ful\014lled) i(b)o(y)d(the)75 158 y(normal)g Fv(MPI)g Fx(library)l(.)21 b(Therefore)15 b(the)g(link)i(step)e(can)g(lo)q(ok)h(something)f(lik)o (e)h(this)75 264 y Fu(\045)24 b(cc)f(...)h(-lmyprof)f(-lpmpi)g(-lmpi) 166 371 y Fx(Here)f Fu(libmyprof.a)d Fx(con)o(tains)j(the)f(pro\014ler) h(functions)h(whic)o(h)f(in)o(tercept)g(some)f(of)g(the)h Fv(MPI)75 427 y Fx(functions.)k Fu(libpmpi.a)16 b Fx(con)o(tains)h(the) g(\\name)g(shifted")g Fv(MPI)g Fx(functions,)h(and)f Fu(libmpi.a)f Fx(con)o(tains)75 483 y(the)f(normal)g(de\014nitions)i (of)e(the)g Fv(MPI)g Fx(functions.)75 605 y Fj(8.4.3)49 b(Complications)75 691 y Fv(Multiple)16 b(counting)75 777 y Fx(Since)g(parts)e(of)g(the)g Fv(MPI)g Fx(library)i(ma)o(y)d (themselv)o(es)i(b)q(e)h(implemen)o(ted)g(using)f(more)f(basic)h Fv(MPI)f Fx(func-)75 833 y(tions)h(\(e.g.)k(a)c(p)q(ortable)h(implemen) o(tation)g(of)f(the)g(collectiv)o(e)i(op)q(erations)f(implemen)o(ted)h (using)e(p)q(oin)o(t)75 890 y(to)e(p)q(oin)o(t)h(comm)o(unications\),)g (there)g(is)h(p)q(oten)o(tial)f(for)f(pro\014ling)i(functions)g(to)e(b) q(e)h(called)i(from)d(within)75 946 y(an)19 b Fv(MPI)g Fx(function)h(whic)o(h)g(w)o(as)e(called)j(from)e(a)g(pro\014ling)h (function.)33 b(This)20 b(could)g(lead)g(to)f(\\double)75 1003 y(coun)o(ting")d(of)g(the)g(time)h(sp)q(en)o(t)f(in)h(the)f(inner) i(routine.)23 b(Since)17 b(this)g(e\013ect)f(could)h(actually)g(b)q(e)g (useful)75 1059 y(under)e(some)e(circumstances)i(\(e.g.)k(it)14 b(migh)o(t)f(allo)o(w)i(one)f(to)f(answ)o(er)g(the)h(question)h(\\Ho)o (w)e(m)o(uc)o(h)h(time)75 1116 y(is)h(sp)q(en)o(t)g(in)h(the)e(p)q(oin) o(t)h(to)f(p)q(oin)o(t)i(routines)f(when)g(they're)f(called)i(from)e (collectiv)o(e)j(functions)e(?"\),)f(w)o(e)75 1172 y(ha)o(v)o(e)h (decided)j(not)d(to)g(enforce)h(an)o(y)g(restrictions)g(on)f(the)h (author)f(of)h(the)f Fv(MPI)h Fx(library)g(whic)o(h)h(w)o(ould)75 1228 y(o)o(v)o(ercome)11 b(this.)19 b(Therefore)12 b(the)g(author)g(of) f(the)h(pro\014ling)i(library)e(should)h(b)q(e)g(a)o(w)o(are)e(of)g (this)i(problem,)75 1285 y(and)f(guard)g(against)g(it)h(herself.)19 b(In)13 b(a)f(single)i(threaded)e(w)o(orld)g(this)h(is)g(easily)g(ac)o (hiev)o(ed)g(through)f(use)g(of)75 1341 y(a)j(static)f(v)m(ariable)j (in)f(the)f(pro\014ling)h(co)q(de)f(whic)o(h)h(remem)o(b)q(ers)f(if)h (y)o(ou)e(are)h(already)g(inside)i(a)e(pro\014ling)75 1398 y(routine.)25 b(It)17 b(b)q(ecomes)g(more)f(complex)i(in)f(a)g(m)o (ulti-threaded)h(en)o(vironmen)o(t)f(\(as)f(do)q(es)h(the)f(meaning)75 1454 y(of)f(the)g(times)h(recorded)f(!\))75 1574 y Fv(Link)o(er)g(o)q (ddities)75 1660 y Fx(The)g(Unix)g(link)o(er)g(traditionally)h(op)q (erates)e(in)i(one)e(pass)g(:)20 b(the)14 b(e\013ect)g(of)g(this)h(is)g (that)f(functions)h(from)75 1717 y(libraries)f(are)e(only)h(included)j (in)d(the)g(image)f(if)h(they)g(are)f(needed)i(at)e(the)h(time)g(the)f (library)h(is)h(scanned.)75 1773 y(When)j(com)o(bined)g(with)g(w)o(eak) f(sym)o(b)q(ols,)g(or)g(m)o(ultiple)i(de\014nitions)g(of)e(the)h(same)f (function,)h(this)g(can)75 1829 y(cause)e(o)q(dd)h(\(and)f(unexp)q (ected\))i(e\013ects.)166 1886 y(Consider,)24 b(for)e(instance,)j(an)d (implemen)o(tation)i(of)e Fv(MPI)g Fx(in)h(whic)o(h)g(the)g(F)l(ortran) e(binding)j(is)75 1942 y(ac)o(hiev)o(ed)19 b(b)o(y)e(using)i(wrapp)q (er)f(functions)g(on)g(top)g(of)f(the)h(C)f(implemen)o(tation.)29 b(The)18 b(author)g(of)f(the)75 1999 y(pro\014le)e(library)h(then)e (assumes)h(that)f(it)g(is)h(reasonable)g(only)g(to)f(pro)o(vide)h (pro\014le)g(functions)h(for)e(the)g(C)75 2055 y(binding,)20 b(since)e(F)l(ortran)e(will)j(ev)o(en)o(tually)g(call)f(these,)g(and)g (the)f(cost)g(of)g(the)h(wrapp)q(ers)f(is)h(assumed)75 2112 y(to)f(b)q(e)g(small.)27 b(Ho)o(w)o(ev)o(er,)16 b(if)i(the)f(wrapp)q(er)g(functions)h(are)f(not)g(in)h(the)f (pro\014ling)i(library)l(,)f(then)g(none)75 2168 y(of)c(the)g (pro\014led)i(en)o(try)e(p)q(oin)o(ts)g(will)i(b)q(e)f(unde\014ned)h (when)f(the)g(pro\014ling)g(library)g(is)g(called.)21 b(Therefore)75 2225 y(none)d(of)f(the)g(pro\014ling)i(co)q(de)f(will)h (b)q(e)f(included)i(in)e(the)g(image.)26 b(When)18 b(the)f(standard)g Fv(MPI)g Fx(library)75 2281 y(is)f(scanned,)f(the)g(F)l(ortran)f(wrapp) q(ers)h(will)i(b)q(e)e(resolv)o(ed,)h(and)f(will)h(also)f(pull)i(in)f (the)f(base)g(v)o(ersions)g(of)75 2338 y(the)g Fv(MPI)g Fx(functions.)21 b(The)15 b(o)o(v)o(erall)h(e\013ect)f(is)h(that)e(the) h(co)q(de)h(will)h(link)g(successfully)l(,)g(but)e(will)i(not)e(b)q(e) 75 2394 y(pro\014led.)166 2451 y(T)l(o)i(o)o(v)o(ercome)f(this)h(w)o(e) g(m)o(ust)f(ensure)h(that)g(the)g(F)l(ortran)e(wrapp)q(er)i(functions)h (are)f(included)i(in)75 2507 y(the)d(pro\014ling)i(v)o(ersion)e(of)g (the)g(library)l(.)23 b(W)l(e)17 b(ensure)f(that)g(this)g(is)h(p)q (ossible)h(b)o(y)e(requiring)h(that)f(these)75 2563 y(b)q(e)g (separable)f(from)g(the)g(rest)f(of)h(the)g(base)g Fv(MPI)g Fx(library)l(.)21 b(This)15 b(allo)o(ws)g(them)g(to)g(b)q(e)g Fu(ar)p Fx(ed)h(out)e(of)h(the)75 2620 y(base)g(library)h(and)g(in)o (to)f(the)g(pro\014ling)i(one.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 203 211 203 210 bop 75 -100 a Fm(8.5.)34 b(MUL)l(TIPLE)16 b(LEVELS)g(OF)f (INTER)o(CEPTION)739 b Fx(203)75 45 y Fn(8.5)59 b(Multiple)20 b(levels)f(of)h(interception)75 147 y Fx(The)15 b(sc)o(heme)h(giv)o(en) g(here)f(do)q(es)h(not)f(directly)h(supp)q(ort)f(the)h(nesting)g(of)e (pro\014ling)j(functions,)e(since)i(it)75 203 y(pro)o(vides)f(only)g(a) g(single)g(alternativ)o(e)g(name)g(for)f(eac)o(h)h Fv(MPI)f Fx(function.)22 b(Consideration)16 b(w)o(as)e(giv)o(en)j(to)75 259 y(an)e(implemen)o(tation)i(whic)o(h)f(w)o(ould)g(allo)o(w)g(m)o (ultiple)h(lev)o(els)g(of)e(call)h(in)o(terception,)h(ho)o(w)o(ev)o(er) d(w)o(e)h(w)o(ere)75 316 y(unable)j(to)e(construct)g(an)g(implemen)o (tation)i(of)e(this)h(whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o (wing)i(disadv)m(an-)75 372 y(tages)143 466 y Ft(\017)23 b Fx(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 560 y Ft(\017)23 b Fx(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o(en) g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 654 y(Since)i(one)f(of)g(the)g(ob)s(jectiv)o(es)g(of)f Fv(MPI)h Fx(is)g(to)f(p)q(ermit)i(e\016cien)o(t,)f(lo)o(w)g(latency)h (implemen)o(tations,)g(and)75 710 y(it)j(is)f(not)g(the)h(business)g (of)f(a)g(standard)g(to)g(require)h(a)f(particular)h(implemen)o(tation) g(language,)g(w)o(e)75 767 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h (outlined)h(ab)q(o)o(v)o(e.)166 823 y(Note,)f(ho)o(w)o(ev)o(er,)f(that) h(it)h(is)f(p)q(ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o(v)o(e)g (to)g(implemen)o(t)i(a)e(m)o(ulti-lev)o(el)75 880 y(system,)k(since)g (the)g(function)g(called)i(b)o(y)d(the)h(user)f(ma)o(y)g(call)i(man)o (y)e(di\013eren)o(t)g(pro\014ling)i(functions)75 936 y(b)q(efore)15 b(calling)i(the)f(underlying)h Fv(MPI)d Fx(function.)166 993 y(Unfortunately)g(suc)o(h)h(an)f(implemen)o (tation)i(ma)o(y)d(require)j(more)d(co)q(op)q(eration)i(b)q(et)o(w)o (een)g(the)f(dif-)75 1049 y(feren)o(t)g(pro\014ling)h(libraries)h(than) e(is)g(required)h(for)f(the)g(single)h(lev)o(el)h(implemen)o(tation)f (detailed)h(ab)q(o)o(v)o(e.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 204 212 204 211 bop 75 377 a Fr(Biblio)q(graph)m(y)98 600 y Fx([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 656 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 713 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 806 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 863 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 919 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1013 y([3])j(Purushotham)16 b(V.)f(Bangalore,)g(Nathan)g(E.)g(Doss,)f (and)i(An)o(thon)o(y)f(Skjellum.)22 b(MPI++:)f(Issues)168 1070 y(and)16 b(Features.)j(In)d Fl(OON-SKI)f('94)p Fx(,)g(page)g(in)h (press,)f(1994.)98 1163 y([4])21 b(A.)14 b(Beguelin,)j(J.)d(Dongarra,)e (A.)i(Geist,)h(R.)f(Manc)o(hek,)g(and)g(V.)g(Sunderam.)19 b(Visualization)e(and)168 1220 y(debugging)g(in)f(a)f(heterogeneous)g (en)o(vironmen)o(t.)20 b Fl(IEEE)15 b(Computer)p Fx(,)h(26\(6\):88{95,) 11 b(June)17 b(1993.)98 1314 y([5])k(Luc)14 b(Bomans)f(and)h(Rolf)f (Hemp)q(el.)18 b(The)c(Argonne/GMD)e(macros)g(in)i(F)o(OR)l(TRAN)g(for) e(p)q(ortable)168 1370 y(parallel)20 b(programming)e(and)h(their)f (implemen)o(tation)i(on)e(the)h(In)o(tel)g(iPSC/2.)29 b Fl(Par)n(al)r(lel)19 b(Com-)168 1427 y(puting)p Fx(,)c(15:119{132,)d (1990.)98 1520 y([6])21 b(R.)15 b(Butler)g(and)f(E.)g(Lusk.)k(User's)c (guide)h(to)f(the)g(p4)g(programming)g(system.)k(T)l(ec)o(hnical)e(Rep) q(ort)168 1577 y(TM-ANL{92/17,)e(Argonne)h(National)h(Lab)q(oratory)l (,)e(1992.)98 1671 y([7])21 b(Ralph)j(Butler)f(and)g(Ewing)g(Lusk.)41 b(Monitors,)24 b(messages,)f(and)g(clusters:)35 b(the)22 b(p4)g(parallel)168 1727 y(programming)15 b(system.)21 b Fl(Journal)c(of)g(Par)n(al)r(lel)f(Computing)p Fx(,)f(1994.)20 b(to)15 b(app)q(ear)h(\(Also)g(Argonne)168 1784 y(National)11 b(Lab)q(oratory)e(Mathematics)h(and)g(Computer)g(Science)i(Division)g (preprin)o(t)f(P362-0493\).)98 1877 y([8])21 b(Robin)d(Calkin,)e(Rolf)h (Hemp)q(el,)g(Hans-Christian)g(Hopp)q(e,)g(and)f(P)o(eter)f(Wypior.)22 b(P)o(ortable)16 b(pro-)168 1934 y(gramming)11 b(with)h(the)f(parmacs)g (message{passing)g(library)l(.)k Fl(Par)n(al)r(lel)d(Computing,)h(Sp)n (e)n(cial)f(issue)168 1990 y(on)k(message{p)n(assing)f(interfac)n(es)p Fx(,)f(to)g(app)q(ear.)98 2084 y([9])21 b(S.)c(Chittor)g(and)h(R.)f(J.) g(En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d (mesh{connected)i(w)o(ormhole{)168 2141 y(routed)f(net)o(w)o(orks)e (for)i(in)o(terpro)q(cessor)f(comm)o(unication)i(in)f(m)o (ulticomputers.)25 b(In)17 b Fl(Pr)n(o)n(c)n(e)n(e)n(dings)168 2197 y(of)g(the)f(1990)i(Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p Fx(,)c(pages)j(647{656,)e(1990.)75 2291 y([10])21 b(S.)f(Chittor)g(and) g(R.)g(J.)f(En)o(b)q(o)q(dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g (of)f(mapping)i(on)e(the)h(comm)o(unica-)168 2347 y(tion)g(p)q (erformance)g(of)f(large)g(m)o(ulticomputers.)34 b(In)20 b Fl(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f(1991)h(International)168 2404 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n (essing,)f(vol.)i(II)f(\(Softwar)n(e\))p Fx(,)f(pages)h(I)q(I{1)h({)f (I)q(I{4,)g(1991.)75 2498 y([11])21 b(J.)e(Dongarra,)f(A.)g(Geist,)h (R.)g(Manc)o(hek,)g(and)g(V.)f(Sunderam.)31 b(In)o(tegrated)18 b(PVM)g(framew)o(ork)168 2554 y(supp)q(orts)c(heterogeneous)g(net)o(w)o (ork)f(computing.)18 b Fl(Computers)e(in)e(Physics)p Fx(,)f(7\(2\):166{75,)e(April)168 2610 y(1993.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 205 213 205 212 bop 75 -100 a Fm(BIBLIOGRAPHY)1371 b Fx(205)75 45 y([12])21 b(J.)c(J.)g(Dongarra,)f(R.)h(Hemp)q(el,)i(A.)e(J.)g(G.)f (Hey)l(,)i(and)f(D.)f(W.)h(W)l(alk)o(er.)25 b(A)17 b(prop)q(osal)g(for) g(a)g(user-)168 102 y(lev)o(el,)c(message)d(passing)h(in)o(terface)g (in)h(a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(hnical)g (Rep)q(ort)168 158 y(TM-12231,)g(Oak)j(Ridge)g(National)g(Lab)q (oratory)l(,)e(F)l(ebruary)h(1993.)75 249 y([13])21 b(Nathan)e(Doss,)f (William)i(Gropp,)e(Ewing)h(Lusk,)h(and)e(An)o(thon)o(y)g(Skjellum.)31 b(A)19 b(mo)q(del)g(imple-)168 305 y(men)o(tation)c(of)g(MPI.)20 b(T)l(ec)o(hnical)d(rep)q(ort,)d(Argonne)i(National)f(Lab)q(oratory)l (,)f(1993.)75 396 y([14])21 b(Edin)o(burgh)g(P)o(arallel)f(Computing)g (Cen)o(tre,)g(Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fl(CHIMP)19 b(Conc)n(epts)p Fx(,)168 452 y(June)e(1991.)75 543 y([15])k(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f (Univ)o(ersit)o(y)h(of)g(Edin)o(burgh.)22 b Fl(CHIMP)15 b(V)m(ersion)h(1.0)168 599 y(Interfac)n(e)p Fx(,)e(Ma)o(y)g(1992.)75 690 y([16])21 b(D.)d(F)l(eitelson.)28 b(Comm)o(unicators:)d(Ob)s (ject-based)18 b(m)o(ultipart)o(y)g(in)o(teractions)g(for)g(parallel)h (pro-)168 747 y(gramming.)j(T)l(ec)o(hnical)c(Rep)q(ort)f(91-12,)e (Dept.)g(Computer)h(Science,)i(The)e(Hebrew)g(Univ)o(ersit)o(y)168 803 y(of)f(Jerusalem,)h(No)o(v)o(em)o(b)q(er)f(1991.)75 894 y([17])21 b(Hub)q(ertus)i(F)l(rank)o(e,)f(P)o(eter)f(Ho)q(c)o(hsc)o (hild,)k(Pratap)c(P)o(attnaik,)h(and)g(Marc)e(Snir.)40 b(An)22 b(e\016cien)o(t)168 950 y(implemen)o(tation)i(of)f(MPI.)42 b(In)23 b Fl(1994)h(International)e(Confer)n(enc)n(e)f(on)i(Par)n(al)r (lel)g(Pr)n(o)n(c)n(essing)p Fx(,)168 1007 y(1994.)75 1097 y([18])e(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o (eyton,)f(and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f(PICL:)g(a)168 1154 y(p)q(ortable)f(instrumen)o (ted)f(comm)o(unication)h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e (TM-11616,)f(Oak)h(Ridge)168 1210 y(National)k(Lab)q(oratory)l(,)e (Octob)q(er)i(1990.)75 1301 y([19])21 b(William)j(D.)c(Gropp)h(and)h (Barry)e(Smith.)39 b(Chameleon)22 b(parallel)h(programming)e(to)q(ols)g (users)168 1357 y(man)o(ual.)f(T)l(ec)o(hnical)d(Rep)q(ort)f (ANL-93/23,)e(Argonne)h(National)h(Lab)q(oratory)l(,)e(Marc)o(h)h (1993.)75 1448 y([20])21 b(O.)c(Kr\177)-23 b(amer)16 b(and)h(H.)g(M)q(\177)-24 b(uhlen)o(b)q(ein.)26 b(Mapping)17 b(strategies)g(in)g(message{based)f(m)o(ultipro)q(cessor)168 1504 y(systems.)k Fl(Par)n(al)r(lel)15 b(Computing)p Fx(,)g(9:213{225,)d(1989.)75 1595 y([21])21 b(nCUBE)16 b(Corp)q(oration.)j Fl(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i (r2.0)p Fx(,)f(Decem)o(b)q(er)f(1990.)75 1686 y([22])21 b(P)o(arasoft)10 b(Corp)q(oration,)g(P)o(asadena,)h(CA.)i Fl(Expr)n(ess)f(User's)f(Guide)p Fx(,)h(v)o(ersion)f(3.2.5)f(edition,)i (1992.)75 1776 y([23])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g(system.)22 b(In)17 b Fl(Pr)n(o)n(c)n(e)n(e)n (dings)e(of)i(the)g(Thir)n(d)g(Confer)n(enc)n(e)e(on)168 1833 y(Hyp)n(er)n(cub)n(e)g(Concurr)n(ent)f(Computers)i(and)f(Applic)n (ations)p Fx(,)e(pages)h(384{390.)d(A)o(CM)j(Press,)f(1988.)75 1923 y([24])21 b(A.)14 b(Skjellum)i(and)e(A.)g(Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f(comm)o (unication)h(library)168 1980 y(atop)g(the)f(reactiv)o(e)h(k)o(ernel.) 20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g (editors,)g Fl(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 2036 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g (Confer)n(enc)n(e)p Fx(,)e(pages)i(767{776.)d(IEEE)168 2093 y(Press,)d(1990.)75 2184 y([25])21 b(A.)12 b(Skjellum,)i(S.)e (Smith,)h(C.)e(Still,)j(A.)d(Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q (co)q(de)g(message)f(passing)168 2240 y(system.)20 b(T)l(ec)o(hnical)d (rep)q(ort,)d(La)o(wrence)i(Liv)o(ermore)f(National)h(Lab)q(oratory)l (,)e(Septem)o(b)q(er)i(1992.)75 2331 y([26])21 b(An)o(thon)o(y)12 b(Skjellum,)i(Nathan)e(E.)g(Doss,)g(and)g(Purushotham)g(V.)g (Bangalore.)j(W)l(riting)e(Libraries)168 2387 y(in)i(MPI.)j(In)d(An)o (thon)o(y)f(Skjellum)h(and)f(Donna)g(S.)g(Reese,)h(editors,)f Fl(Pr)n(o)n(c)n(e)n(e)n(dings)f(of)i(the)g(Sc)n(alable)168 2444 y(Par)n(al)r(lel)h(Libr)n(aries)f(Confer)n(enc)n(e)p Fx(,)d(pages)j(166{173.)e(IEEE)i(Computer)g(So)q(ciet)o(y)g(Press,)g (Octob)q(er)168 2500 y(1993.)75 2591 y([27])21 b(An)o(thon)o(y)c (Skjellum,)h(Stev)o(en)f(G.)e(Smith,)j(Nathan)e(E.)g(Doss,)g(Alvin)i(P) e(.)g(Leung,)h(and)g(Manfred)168 2647 y(Morari.)32 b(The)19 b(Design)h(and)f(Ev)o(olution)h(of)f(Zip)q(co)q(de.)33 b Fl(Par)n(al)r(lel)19 b(Computing)p Fx(,)h(1994.)31 b(\(In)o(vited)168 2704 y(P)o(ap)q(er,)15 b(to)g(app)q(ear)g(in)h(Sp)q (ecial)h(Issue)f(on)f(Message)g(P)o(assing\).)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 206 214 206 213 bop 75 -100 a Fx(206)1368 b Fm(BIBLIOGRAPHY)75 45 y Fx([28])21 b(An)o(thon)o(y)d(Skjellum,)j(Stev)o(en)d(G.)g(Smith,)h (Nathan)f(E.)g(Doss,)g(Charles)h(H.)f(Still,)i(Alvin)g(P)l(.)e(Le-)168 102 y(ung,)g(and)f(Manfred)g(Morari.)24 b(Zip)q(co)q(de:)h(A)17 b(Portable)g(Comm)o(unication)h(La)o(y)o(er)e(for)h(High)g(P)o(er-)168 158 y(formance)g(Multicomputing.)28 b(T)l(ec)o(hnical)20 b(Rep)q(ort)d(UCRL-JC-106725)g(\(revised)h(9/92,)f(12/93,)168 214 y(4/94\),)f(La)o(wrence)i(Liv)o(ermore)g(National)f(Lab)q(oratory)l (,)g(Marc)o(h)f(1991.)25 b(T)l(o)17 b(app)q(ear)h(in)g(Concur-)168 271 y(rency:)j(Practice)15 b(&)h(Exp)q(erience.)75 365 y([29])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g (passing)g(in)i(a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec) o(h-)168 421 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National)g (Lab)q(oratory)l(,)e(August)h(1992.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 207 215 207 214 bop 75 356 a Fo(Annex)33 b(A)75 564 y Fr(Language)41 b(Binding)75 805 y Fn(A.1)59 b(Intro)r(duction)75 906 y Fx(In)15 b(this)f(section)h(w)o(e)f(summarize)g(the)h(sp)q(eci\014c)h (bindings)g(for)d(b)q(oth)h(F)l(ortran)f(and)h(C.)g(W)l(e)g(presen)o(t) g(\014rst)75 963 y(the)h(C)g(bindings,)i(then)e(the)h(F)l(ortran)e (bindings.)22 b(Listings)16 b(are)f(alphab)q(etical)i(within)f(c)o (hapter.)75 1106 y Fn(A.2)59 b(De\014ned)19 b(Constants)g(fo)n(r)i(C)f (and)f(F)n(o)n(rtran)75 1207 y Fx(These)e(are)g(required)h(de\014ned)h (constan)o(ts,)d(to)g(b)q(e)i(de\014ned)h(in)e(the)h(\014les)g Fu(mpi.h)e Fx(\(for)g(C\))h(and)g Fu(mpif.h)75 1264 y Fx(\(for)d(F)l(ortran\).)75 1370 y Fu(/*)24 b(return)f(codes)g(\(both)g (C)h(and)f(Fortran\))g(*/)75 1427 y(MPI_SUCCESS)75 1483 y(MPI_ERR_BUFFER)75 1539 y(MPI_ERR_COUNT)75 1596 y(MPI_ERR_TYPE)75 1652 y(MPI_ERR_TAG)75 1709 y(MPI_ERR_COMM)75 1765 y(MPI_ERR_RANK)75 1822 y(MPI_ERR_REQUEST)75 1878 y(MPI_ERR_ROOT)75 1935 y(MPI_ERR_GROUP)75 1991 y(MPI_ERR_OP)75 2048 y(MPI_ERR_TOPOLOGY)75 2104 y(MPI_ERR_DIMS)75 2160 y(MPI_ERR_ARG)75 2217 y(MPI_ERR_UNKNOWN)75 2273 y(MPI_ERR_TRUNCATE)75 2330 y(MPI_ERR_OTHER)75 2386 y(MPI_ERR_INTERN)75 2443 y(MPI_ERR_LASTCODE)75 2556 y(/*)h(assorted)e (constants)h(\(both)g(C)h(and)f(Fortran\))g(*/)75 2612 y(MPI_BOTTOM)75 2669 y(MPI_PROC_NULL)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 208 216 208 215 bop 75 -100 a Fx(208)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI_ANY_SOURCE)75 102 y(MPI_ANY_TAG)75 158 y(MPI_UNDEFINED)75 214 y(MPI_UB)75 271 y(MPI_LB)75 384 y(/*)24 b(status)f(size)g(and)g(reserved)g(index)g (values)g(\(Fortran\))g(*/)75 440 y(MPI_STATUS_SIZE)75 497 y(MPI_SOURCE)75 553 y(MPI_TAG)75 666 y(/*)h(Error-handling)e (specifiers)g(\(C)i(and)f(Fortran\))g(*/)75 723 y(MPI_ERRORSFATAL)75 779 y(MPI_ERRORSRETURN)75 892 y(/*)h(Maximum)e(sizes)i(for)f(strings)g (*/)75 948 y(MPI_MAX_PROCESSOR_NAME)75 1061 y(/*)h(elementary)e (datatypes)h(\(C\))g(*/)75 1118 y(MPI_CHAR)75 1174 y(MPI_SHORT)75 1231 y(MPI_INT)75 1287 y(MPI_LONG)75 1344 y(MPI_UNSIGNED_CHAR)75 1400 y(MPI_UNSIGNED_SHORT)75 1456 y(MPI_UNSIGNED)75 1513 y(MPI_UNSIGNED_LONG)75 1569 y(MPI_FLOAT)75 1626 y(MPI_DOUBLE)75 1682 y(MPI_LONG_DOUBLE)75 1739 y(MPI_BYTE)75 1795 y(MPI_PACKED)75 2021 y(/*)h(elementary)e(datatypes)h(\(Fortran\))f(*/)75 2077 y(MPI_INTEGER)75 2134 y(MPI_REAL)75 2190 y(MPI_DOUBLE_PRECISION)75 2247 y(MPI_COMPLEX)75 2303 y(MPI_DOUBLE_COMPLEX)75 2360 y(MPI_LOGICAL)75 2416 y(MPI_CHARACTER)75 2473 y(MPI_BYTE)75 2586 y(/*)i(optional)e(datatypes)h(\(Fortran\))g(*/)75 2698 y(MPI_INTEGER1)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 209 217 209 216 bop 75 -100 a Fm(A.2.)29 b(DEFINED)15 b(CONST)l(ANTS)h(F)o(OR)f (C)g(AND)g(F)o(OR)l(TRAN)590 b Fx(209)75 45 y Fu(MPI_INTEGER2)75 102 y(MPI_INTEGER4)75 158 y(MPI_REAL2)75 214 y(MPI_REAL4)75 271 y(MPI_REAL8)75 384 y(/*)24 b(optional)e(datatypes)h(\(C\))g(*/)75 440 y(MPI_LONG_LONG_INT)75 553 y(/*)h(reserved)e(communicators)g(\(C)i (and)f(Fortran\))g(*/)75 610 y(MPI_COMM_WORLD)75 666 y(MPI_COMM_SELF)75 779 y(/*)h(results)e(of)i(communicator)e(and)i (group)f(comparisons)f(*/)75 892 y(MPI_IDENT)75 948 y(MPI_CONGRUENT)75 1005 y(MPI_SIMILAR)75 1061 y(MPI_UNEQUAL)75 1174 y(/*)i(environmental)e (inquiry)h(keys)g(\(C)g(and)h(Fortran\))f(*/)75 1231 y(MPI_TAG_UB)75 1287 y(MPI_IO)75 1344 y(MPI_HOST)75 1456 y(/*)h(collective)e(operations)h(\(C)g(and)h(Fortran\))e(*/)75 1513 y(MPI_MAX)75 1569 y(MPI_MIN)75 1626 y(MPI_SUM)75 1682 y(MPI_PROD)75 1739 y(MPI_MAXLOC)75 1795 y(MPI_MINLOC)75 1852 y(MPI_BAND)75 1908 y(MPI_BOR)75 1965 y(MPI_BXOR)75 2021 y(MPI_LAND)75 2077 y(MPI_LOR)75 2134 y(MPI_LXOR)75 2247 y(/*)i(Null)f(handles)g(*/)75 2303 y(MPI_GROUP_NULL)75 2360 y(MPI_COMM_NULL)75 2416 y(MPI_DATATYPE_NULL)75 2473 y(MPI_REQUEST_NULL)75 2529 y(MPI_OP_NULL)75 2586 y(MPI_ERRHANDLER_NULL) 75 2698 y(/*)h(Empty)f(group)g(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 210 218 210 217 bop 75 -100 a Fx(210)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI_GROUP_EMPTY)75 158 y(/*)24 b(topologies)e(\(C)i(and)f(Fortran\))g(*/)75 214 y(MPI_GRAPH)75 271 y(MPI_CART)166 434 y Fx(The)15 b(follo)o(wing)h(are)f(de\014ned)i(C)e(t)o(yp)q(e)g(de\014nitions,)i (also)e(included)j(in)e(the)f(\014le)h Fu(mpi.h)p Fx(.)75 540 y Fu(/*)24 b(opaque)f(types)g(\(C\))g(*/)75 596 y(MPI_Aint)75 709 y(/*)h(type)f(of)g(user-defined)g(functions)f(*/)75 766 y(MPI_Uop)75 822 y(MPI_Handler_function)75 879 y(MPI_Copy_function) 75 935 y(MPI_Delete_function)75 1048 y(/*)i(handles)e(to)i(assorted)f (structures)f(\(C\))i(*/)75 1104 y(MPI_Status)75 1161 y(MPI_Group)75 1217 y(MPI_Comm)75 1274 y(MPI_Datatype)75 1330 y(MPI_Request)75 1387 y(MPI_Op)75 1500 y(/*)g(prototypes)e(for)i (user-defined)e(functions)g(*/)75 1556 y(typedef)h(int)47 b(MPI_copy_function\(MPI_Comm)21 b(*oldcomm,)h(*newcomm,)h(int)g (*keyval,)552 1613 y(void)h(*extra_state\))75 1669 y(typedef)f(int)47 b(MPI_delete_function\(MPI_Comm)20 b(*comm,)j(int)h(*keyval,)839 1725 y(void)f(*extra_state\)})75 1782 y(typedef)g(void)g (MPI_Handler_function\(MPI_Com)o(m)e(*,)j(int)f(*,)h(...\);)75 1838 y(typedef)f(void)g(MPI_Uop\()g(void)g(*invec,)g(void)h(*inoutvec,) e(int)h(*len,)600 1895 y(MPI_Datatype)f(*datatype\);)75 2095 y Fn(A.3)59 b(C)20 b(bindings)g(fo)n(r)h(P)n(oint-to-P)n(oint)e (Communication)75 2196 y Fx(These)d(are)e(presen)o(ted)i(here)g(in)g (the)f(order)g(of)g(their)g(app)q(earance)h(in)g(the)g(c)o(hapter.)75 2252 y Fu(int)23 b(MPI)p 245 2252 15 2 v 17 w(Send\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 2252 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 2309 y(int)h(tag,)f(MPI)p 683 2309 V 17 w(Comm)g(comm\))75 2395 y(int)g(MPI)p 245 2395 V 17 w(Recv\(void*)g(buf,)g(int)g(count,)h (MPI)p 979 2395 V 16 w(Datatype)f(datatype,)g(int)g(source,)393 2452 y(int)h(tag,)f(MPI)p 683 2452 V 17 w(Comm)g(comm,)g(MPI)p 1034 2452 V 17 w(Status)g(*status\))75 2538 y(int)g(MPI)p 245 2538 V 17 w(Get)p 334 2538 V 17 w(count\(MPI)p 567 2538 V 16 w(Status)g(status,)g(MPI)p 1013 2538 V 17 w(Datatype)f (datatype,)h(int)h(*count\))75 2625 y(int)f(MPI)p 245 2625 V 17 w(Bsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2625 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2681 y(int)h(tag,)f(MPI)p 683 2681 V 17 w(Comm)g(comm\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 211 219 211 218 bop 75 -100 a Fm(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b Fx(211)75 45 y Fu(int)23 b(MPI)p 245 45 15 2 v 17 w(Ssend\(void*)g(buf,)g(int)g (count,)g(MPI)p 1002 45 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 102 y(int)h(tag,)f(MPI)p 683 102 V 17 w(Comm)g(comm\))75 188 y(int)g(MPI)p 245 188 V 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 188 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 244 y(int)h(tag,)f(MPI)p 683 244 V 17 w(Comm)g(comm\))75 331 y(int)g(MPI)p 245 331 V 17 w(Buffer)p 406 331 V 17 w(attach\()f(void*)i(buffer,)f(int)g(size\))75 417 y(int)g(MPI)p 245 417 V 17 w(Buffer)p 406 417 V 17 w(detach\()f(void**)i(buffer,)e (int*)i(size\))75 504 y(int)f(MPI)p 245 504 V 17 w(Isend\(void*)g(buf,) g(int)g(count,)g(MPI)p 1002 504 V 17 w(Datatype)g(datatype,)g(int)g (dest,)393 560 y(int)h(tag,)f(MPI)p 683 560 V 17 w(Comm)g(comm,)g(MPI)p 1034 560 V 17 w(Request)g(*request\))75 647 y(int)g(MPI)p 245 647 V 17 w(Ibsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 647 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 703 y(int)g(tag,)f(MPI)p 683 703 V 17 w(Comm)g(comm,)g(MPI)p 1034 703 V 17 w(Request)g(*request\))75 790 y(int)g(MPI)p 245 790 V 17 w(Issend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 790 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 846 y(int)g(tag,)f(MPI)p 683 846 V 17 w(Comm)g(comm,)g(MPI)p 1034 846 V 17 w(Request)g(*request\))75 933 y(int)g(MPI)p 245 933 V 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 933 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 989 y(int)g(tag,)f(MPI)p 683 989 V 17 w(Comm)g(comm,)g(MPI)p 1034 989 V 17 w(Request)g(*request\))75 1075 y(int)g(MPI)p 245 1075 V 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1075 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 1132 y(int)h(tag,)f(MPI)p 683 1132 V 17 w(Comm)g(comm,)g(MPI)p 1034 1132 V 17 w(Request)g(*request\))75 1218 y(int)g(MPI)p 245 1218 V 17 w(Wait\(MPI)p 454 1218 V 16 w(Request)g(*request,)g(MPI)p 972 1218 V 17 w(Status)g(*status\))75 1305 y(int)g(MPI)p 245 1305 V 17 w(Test\(MPI)p 454 1305 V 16 w(Request)g(*request,)g(int)g (*flag,)g(MPI)p 1234 1305 V 17 w(Status)g(*status\))75 1391 y(int)g(MPI)p 245 1391 V 17 w(Request)p 430 1391 V 16 w(free\(MPI)p 638 1391 V 17 w(Request)f(*request\))75 1478 y(int)h(MPI)p 245 1478 V 17 w(Waitany\(int)g(count,)g(MPI)p 788 1478 V 16 w(Request)g(*array)p 1139 1478 V 17 w(of)p 1204 1478 V 17 w(requests,)f(int)i(*index,)393 1534 y(MPI)p 468 1534 V 17 w(Status)f(*status\))75 1621 y(int)g(MPI)p 245 1621 V 17 w(Testany\(int)g(count,)g(MPI)p 788 1621 V 16 w(Request)g(*array)p 1139 1621 V 17 w(of)p 1204 1621 V 17 w(requests,)f(int)i(*index,)393 1677 y(int)g(*flag,)f(MPI)p 731 1677 V 16 w(Status)g(*status\))75 1764 y(int)g(MPI)p 245 1764 V 17 w(Waitall\(int)g(count,)g(MPI)p 788 1764 V 16 w(Request)g(*array)p 1139 1764 V 17 w(of)p 1204 1764 V 17 w(requests,)393 1820 y(MPI)p 468 1820 V 17 w(Status)g(*array)p 796 1820 V 16 w(of)p 860 1820 V 17 w(statuses\))75 1907 y(int)g(MPI)p 245 1907 V 17 w(Testall\(int)g (count,)g(MPI)p 788 1907 V 16 w(Request)g(*array)p 1139 1907 V 17 w(of)p 1204 1907 V 17 w(requests,)f(int)i(*flag,)393 1963 y(MPI)p 468 1963 V 17 w(Status)f(*array)p 796 1963 V 16 w(of)p 860 1963 V 17 w(statuses\))75 2049 y(int)g(MPI)p 245 2049 V 17 w(Waitsome\(int)f(incount,)h(MPI)p 859 2049 V 17 w(Request)g(*array)p 1211 2049 V 16 w(of)p 1275 2049 V 17 w(requests,)g(int)g(*outcount,)393 2106 y(int)h(*array)p 636 2106 V 16 w(of)p 700 2106 V 17 w(indices,)f(MPI)p 1004 2106 V 16 w(Status)g(*array)p 1331 2106 V 17 w(of)p 1396 2106 V 17 w(statuses\))75 2192 y(int)g(MPI)p 245 2192 V 17 w(Testsome\(int)f(incount,)h(MPI)p 859 2192 V 17 w(Request)g(*array)p 1211 2192 V 16 w(of)p 1275 2192 V 17 w(requests,)g(int)g(*outcount,)393 2249 y(int)h(*array)p 636 2249 V 16 w(of)p 700 2249 V 17 w(indices,)f(MPI)p 1004 2249 V 16 w(Status)g(*array)p 1331 2249 V 17 w(of)p 1396 2249 V 17 w(statuses\))75 2335 y(int)g(MPI)p 245 2335 V 17 w(Iprobe\(int)g(source,)g(int)g(tag,)g(MPI)p 1002 2335 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 2392 y(MPI)p 468 2392 V 17 w(Status)g(*status\))75 2478 y(int)g(MPI)p 245 2478 V 17 w(Probe\(int)g(source,)g(int)g(tag,)h(MPI)p 979 2478 V 16 w(Comm)g(comm,)f(MPI)p 1330 2478 V 17 w(Status)g (*status\))75 2565 y(int)g(MPI)p 245 2565 V 17 w(Cancel\(MPI)p 502 2565 V 16 w(Request)g(*request\))75 2651 y(int)g(MPI)p 245 2651 V 17 w(Test)p 358 2651 V 17 w(cancelled\(MPI)p 687 2651 V 15 w(Status)g(status,)g(int)h(*flag\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 212 220 212 219 bop 75 -100 a Fx(212)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(int)23 b(MPI)p 245 45 15 2 v 17 w(Send)p 358 45 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1091 45 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 102 y(int)g(tag,)f(MPI)p 683 102 V 17 w(Comm)g(comm,)g(MPI)p 1034 102 V 17 w(Request)g(*request\))75 188 y(int)g(MPI)p 245 188 V 17 w(Bsend)p 382 188 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 188 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 244 y(int)g(tag,)f(MPI)p 683 244 V 17 w(Comm)g(comm,)g(MPI)p 1034 244 V 17 w(Request)g(*request\))75 331 y(int)g(MPI)p 245 331 V 17 w(Ssend)p 382 331 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 331 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 387 y(int)g(tag,)f(MPI)p 683 387 V 17 w(Comm)g(comm,)g(MPI)p 1034 387 V 17 w(Request)g(*request\))75 474 y(int)g(MPI)p 245 474 V 17 w(Rsend)p 382 474 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 474 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 530 y(int)g(tag,)f(MPI)p 683 530 V 17 w(Comm)g(comm,)g(MPI)p 1034 530 V 17 w(Request)g(*request\))75 617 y(int)g(MPI)p 245 617 V 17 w(Recv)p 358 617 V 17 w(init\(void*)f(buf,)i(int)f(count,) g(MPI)p 1091 617 V 17 w(Datatype)g(datatype,)f(int)i(source,)393 673 y(int)g(tag,)f(MPI)p 683 673 V 17 w(Comm)g(comm,)g(MPI)p 1034 673 V 17 w(Request)g(*request\))75 760 y(int)g(MPI)p 245 760 V 17 w(Start\(MPI)p 478 760 V 16 w(Request)g(*request\))75 846 y(int)g(MPI)p 245 846 V 17 w(Startall\(int)f(count,)h(MPI)p 811 846 V 17 w(Request)g(*array)p 1163 846 V 17 w(of)p 1228 846 V 16 w(requests\))75 933 y(int)g(MPI)p 245 933 V 17 w(Sendrecv\(void)f(*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 933 V 17 w(Datatype)f(sendtype,)393 989 y(int)i(dest,)f(int)g (sendtag,)g(void)g(*recvbuf,)g(int)g(recvcount,)393 1045 y(MPI)p 468 1045 V 17 w(Datatype)g(recvtype,)f(int)i(source,)f(MPI)p 1297 1045 V 16 w(Datatype)g(recvtag,)393 1102 y(MPI)p 468 1102 V 17 w(Comm)g(comm,)g(MPI)p 819 1102 V 17 w(Status)g (*status\))75 1188 y(int)g(MPI)p 245 1188 V 17 w(Sendrecv)p 454 1188 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 1188 V 17 w(Datatype)g(datatype,)393 1245 y(int)h(dest,)f(int)g (sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 1245 V 16 w(Comm)h(comm,)393 1301 y(MPI)p 468 1301 V 17 w(Status)f(*status\))75 1388 y(int)g(MPI)p 245 1388 V 17 w(Type)p 358 1388 V 17 w(contiguous\(int)f(count,)h(MPI)p 972 1388 V 17 w(Datatype)f(oldtype,)393 1444 y(MPI)p 468 1444 V 17 w(Datatype)h(*newtype\))75 1531 y(int)g(MPI)p 245 1531 V 17 w(Type)p 358 1531 V 17 w(vector\(int)f(count,)h(int)h (blocklength,)e(int)i(stride,)393 1587 y(MPI)p 468 1587 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 1587 V 17 w(Datatype)h (*newtype\))75 1674 y(int)g(MPI)p 245 1674 V 17 w(Type)p 358 1674 V 17 w(hvector\(int)f(count,)h(int)h(blocklength,)e(MPI)p 1306 1674 V 17 w(Aint)h(stride,)393 1730 y(MPI)p 468 1730 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 1730 V 17 w(Datatype)h(*newtype\))75 1816 y(int)g(MPI)p 245 1816 V 17 w(Type)p 358 1816 V 17 w(indexed\(int)f(count,)h(int)h(*array)p 1068 1816 V 16 w(of)p 1132 1816 V 17 w(blocklengths,)393 1873 y(int)g(*array)p 636 1873 V 16 w(of)p 700 1873 V 17 w(displacements,)e(MPI)p 1147 1873 V 17 w(Datatype)g(oldtype,)393 1929 y(MPI)p 468 1929 V 17 w(Datatype)h(*newtype\))75 2016 y(int)g(MPI)p 245 2016 V 17 w(Type)p 358 2016 V 17 w(hindexed\(int)f(count,)h(int)h(*array)p 1092 2016 V 16 w(of)p 1156 2016 V 17 w(blocklengths,)393 2072 y(MPI)p 468 2072 V 17 w(Aint)f(*array)p 748 2072 V 17 w(of)p 813 2072 V 17 w(displacements,)e(MPI)p 1259 2072 V 17 w(Datatype)i(oldtype,)393 2129 y(MPI)p 468 2129 V 17 w(Datatype)g(*newtype\))75 2215 y(int)g(MPI)p 245 2215 V 17 w(Type)p 358 2215 V 17 w(struct\(int)f(count,)h(int)h(*array)p 1044 2215 V 16 w(of)p 1108 2215 V 17 w(blocklengths,)393 2272 y(MPI)p 468 2272 V 17 w(Aint)f(*array)p 748 2272 V 17 w(of)p 813 2272 V 17 w(displacements,)e(MPI)p 1259 2272 V 17 w(Datatype)i(*array)p 1635 2272 V 16 w(of)p 1699 2272 V 17 w(types,)393 2328 y(MPI)p 468 2328 V 17 w(Datatype)g(*newtype\))75 2415 y(int)g(MPI)p 245 2415 V 17 w(Address\(void*)f(location,)h(MPI)p 907 2415 V 17 w(Aint)g(*address\))75 2501 y(int)g(MPI)p 245 2501 V 17 w(Type)p 358 2501 V 17 w(extent\(MPI)p 615 2501 V 16 w(Datatype)f(datatype,)h(MPI)p 1156 2501 V 17 w(Aint)g(*extent\)) 75 2588 y(int)g(MPI)p 245 2588 V 17 w(Type)p 358 2588 V 17 w(size\(MPI)p 567 2588 V 16 w(Datatype)g(datatype,)f(MPI)p 1108 2588 V 17 w(Aint)h(*size\))75 2674 y(int)g(MPI)p 245 2674 V 17 w(Type)p 358 2674 V 17 w(count\(MPI)p 591 2674 V 16 w(Datatype)g(datatype,)f(int)i(*count\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 213 221 213 220 bop 75 -100 a Fm(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b Fx(213)75 45 y Fu(int)23 b(MPI)p 245 45 15 2 v 17 w(Type)p 358 45 V 17 w(lb\(MPI)p 519 45 V 16 w(Datatype)g(datatype,)g(int*)g (displacement\))75 132 y(int)g(MPI)p 245 132 V 17 w(Type)p 358 132 V 17 w(ub\(MPI)p 519 132 V 16 w(Datatype)g(datatype,)g(int*)g (displacement\))75 218 y(int)g(MPI)p 245 218 V 17 w(Type)p 358 218 V 17 w(commit\(MPI)p 615 218 V 16 w(Datatype)f(*datatype\))75 304 y(int)h(MPI)p 245 304 V 17 w(Type)p 358 304 V 17 w(free\(MPI)p 567 304 V 16 w(Datatype)g(*datatype\))75 391 y(int)g(MPI)p 245 391 V 17 w(Get)p 334 391 V 17 w(elements\(MPI)p 639 391 V 16 w(Status)g(status,)f(MPI)p 1084 391 V 17 w(Datatype)h(datatype,)g(int)g(*count\))75 477 y(int)g(MPI)p 245 477 V 17 w(Pack\(void*)g(inbuf,)g(int)g(incount,)g(MPI)p 1074 477 V 17 w(Datatype)g(datatype,)f(void)h(*outbuf,)393 534 y(int)h(outsize,)e(int)i(*position,)e(MPI)p 1136 534 V 17 w(Comm)h(comm\))75 620 y(int)g(MPI)p 245 620 V 17 w(Unpack\(void*)f(inbuf,)h(int)h(insize,)f(int)g(*position,)g (void)g(*outbuf,)393 677 y(int)h(outcount,)e(MPI)p 802 677 V 17 w(Datatype)h(datatype,)f(MPI)p 1344 677 V 17 w(Comm)i(comm\))75 763 y(int)f(MPI)p 245 763 V 17 w(Pack)p 358 763 V 17 w(size\(int)g(incount,)f(MPI)p 876 763 V 17 w(Datatype)h(datatype,)g(MPI)p 1419 763 V 16 w(Comm)h(comm,)393 820 y(int)g(*size\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 214 222 214 221 bop 75 -100 a Fx(214)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fn(A.4)59 b(C)20 b(Bindings)g(fo)n (r)h(Collective)e(Communication)75 147 y Fu(int)k(MPI)p 245 147 15 2 v 17 w(Barrier\(MPI)p 526 147 V 16 w(Comm)g(comm)h(\))75 233 y(int)f(MPI)p 245 233 V 17 w(Bcast\(void*)g(buffer,)f(int)i(count,) f(MPI)p 1074 233 V 17 w(Datatype)g(datatype,)f(int)i(root,)393 289 y(MPI)p 468 289 V 17 w(Comm)f(comm)h(\))75 376 y(int)f(MPI)p 245 376 V 17 w(Gather\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1217 376 V 17 w(Datatype)h(sendtype,)393 432 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 432 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 489 y(MPI)p 468 489 V 17 w(Comm)f(comm\))75 575 y(int)g(MPI)p 245 575 V 17 w(Gatherv\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 575 V 17 w(Datatype)h(sendtype,)393 632 y(void*)g(recvbuf,)g(int)h (*recvcounts,)e(int)h(*displs,)393 688 y(MPI)p 468 688 V 17 w(Datatype)g(recvtype,)f(int)i(root,)f(MPI)p 1249 688 V 17 w(Comm)g(comm\))75 775 y(int)g(MPI)p 245 775 V 17 w(Scatter\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 775 V 17 w(Datatype)h(sendtype,)393 831 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 831 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 888 y(MPI)p 468 888 V 17 w(Comm)f(comm\))75 974 y(int)g(MPI)p 245 974 V 17 w(Scatterv\(void*)f(sendbuf,)h(int)g(*sendcounts,)g(int)g (*displs,)393 1030 y(MPI)p 468 1030 V 17 w(Datatype)g(sendtype,)f (void*)i(recvbuf,)e(int)i(recvcount,)393 1087 y(MPI)p 468 1087 V 17 w(Datatype)f(recvtype,)f(int)i(root,)f(MPI)p 1249 1087 V 17 w(Comm)g(comm\))75 1173 y(int)g(MPI)p 245 1173 V 17 w(Allgather\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1289 1173 V 17 w(Datatype)f(sendtype,)393 1230 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1230 V 17 w(Datatype)h(recvtype,)393 1286 y(MPI)p 468 1286 V 17 w(Comm)g(comm\))75 1373 y(int)g(MPI)p 245 1373 V 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1373 V 16 w(Datatype)g(sendtype,)393 1429 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1486 y(MPI)p 468 1486 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1486 V 17 w(Comm)h(comm\))75 1572 y(int)g(MPI)p 245 1572 V 17 w(Alltoall\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1572 V 17 w(Datatype)f(sendtype,)393 1629 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1629 V 17 w(Datatype)h(recvtype,)393 1685 y(MPI)p 468 1685 V 17 w(Comm)g(comm\))75 1772 y(int)g(MPI)p 245 1772 V 17 w(Alltoallv\(void*)f(sendbuf,)h(int)g(*sendcounts,)f(int) i(*sdispls,)393 1828 y(MPI)p 468 1828 V 17 w(Datatype)f(sendtype,)f (void*)i(recvbuf,)e(int)i(*recvcounts,)393 1884 y(int)g(*rdispls,)e (MPI)p 802 1884 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1884 V 17 w(Comm)i(comm\))75 1971 y(int)f(MPI)p 245 1971 V 17 w(Reduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)h(count,)393 2027 y(MPI)p 468 2027 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2027 V 17 w(Op)i(op,)f(int)h(root,)f(MPI)p 1505 2027 V 17 w(Comm)g(comm\))75 2114 y(int)g(MPI)p 245 2114 V 17 w(Op)p 310 2114 V 17 w(create\(MPI)p 567 2114 V 16 w(Uop)g(function,)g(int)g(commute,)g(MPI)p 1299 2114 V 17 w(Op)h(*op\))75 2200 y(int)f(MPI)p 245 2200 V 17 w(op)p 310 2200 V 17 w(free\()g(MPI)p 542 2200 V 17 w(Op)h(*op\))75 2287 y(int)f(MPI)p 245 2287 V 17 w(Allreduce\(void*)f(sendbuf,)h(void*) g(recvbuf,)g(int)g(count,)393 2343 y(MPI)p 468 2343 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2343 V 17 w(Op)i(op,)f(MPI)p 1266 2343 V 17 w(Comm)g(comm\))75 2430 y(int)g(MPI)p 245 2430 V 17 w(Reduce)p 406 2430 V 17 w(scatter\(void*)f(sendbuf,)g (void*)i(recvbuf,)e(int)i(*recvcounts,)393 2486 y(MPI)p 468 2486 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2486 V 17 w(Op)i(op,)f(MPI)p 1266 2486 V 17 w(Comm)g(comm\))75 2573 y(int)g(MPI)p 245 2573 V 17 w(Scan\(void*)g(sendbuf,)f(void*)i (recvbuf,)e(int)i(count,)393 2629 y(MPI)p 468 2629 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2629 V 17 w(Op)i(op,)f(MPI)p 1266 2629 V 17 w(Comm)g(comm)h(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 215 223 215 222 bop 75 -100 a Fm(A.5.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(GR)o (OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)160 b Fx(215)75 45 y Fn(A.5)59 b(C)20 b(Bindings)g(fo)n(r)h(Groups,)e(Contexts,)f(and)h (Communicato)n(rs)75 147 y Fu(int)k(MPI)p 245 147 15 2 v 17 w(Group)p 382 147 V 17 w(size\(MPI)p 591 147 V 16 w(Group)g(group,)g(int)g(*size\))75 233 y(int)g(MPI)p 245 233 V 17 w(Group)p 382 233 V 17 w(rank\(MPI)p 591 233 V 16 w(Group)g(group,)g(int)g(*rank\))75 319 y(int)g(MPI)p 245 319 V 17 w(Group)p 382 319 V 17 w(translate)p 615 319 V 16 w(ranks)g(\(MPI)p 870 319 V 16 w(Group)h(group1,)f(int)g(n,)h (int)f(*ranks1,)393 376 y(MPI)p 468 376 V 17 w(Group)g(group2,)g(int)g (*ranks2\))75 462 y(int)g(MPI)p 245 462 V 17 w(Group)p 382 462 V 17 w(compare\(MPI)p 663 462 V 15 w(Group)h(group1,MPI)p 1062 462 V 15 w(Group)g(group2,)e(int)i(*result\))75 549 y(int)f(MPI)p 245 549 V 17 w(Comm)p 358 549 V 17 w(group\(MPI)p 591 549 V 16 w(Comm)g(comm,)g(MPI)p 941 549 V 17 w(Group)g(*group\))75 635 y(int)g(MPI)p 245 635 V 17 w(Group)p 382 635 V 17 w(union\(MPI)p 615 635 V 16 w(Group)g(group1,)g(MPI)p 1037 635 V 17 w(Group)g(group2,)g(MPI)p 1460 635 V 16 w(Group)h(*newgroup\))75 722 y(int)f(MPI)p 245 722 V 17 w(Group)p 382 722 V 17 w(intersection\(MPI)p 783 722 V 15 w(Group)g(group1,)g(MPI)p 1204 722 V 17 w(Group)g(group2,)393 778 y(MPI)p 468 778 V 17 w(Group)g(*newgroup\))75 865 y(int)g(MPI)p 245 865 V 17 w(Group)p 382 865 V 17 w(difference\(MPI)p 735 865 V 15 w(Group)g(group1,)g(MPI)p 1156 865 V 17 w(Group)g(group2,)393 921 y(MPI)p 468 921 V 17 w(Group)g(*newgroup\))75 1008 y(int)g(MPI)p 245 1008 V 17 w(Group)p 382 1008 V 17 w(incl\(MPI)p 591 1008 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1008 V 17 w(Group)h(*newgroup\))75 1094 y(int)g(MPI)p 245 1094 V 17 w(Group)p 382 1094 V 17 w(excl\(MPI)p 591 1094 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1094 V 17 w(Group)h(*newgroup\))75 1181 y(int)g(MPI)p 245 1181 V 17 w(Group)p 382 1181 V 17 w(range)p 519 1181 V 16 w(incl\(MPI)p 727 1181 V 16 w(Group)g(group,)h(int)f(n,)h(int)f (**ranges,)393 1237 y(MPI)p 468 1237 V 17 w(Group)g(*newgroup\))75 1323 y(int)g(MPI)p 245 1323 V 17 w(Group)p 382 1323 V 17 w(range)p 519 1323 V 16 w(excl\(MPI)p 727 1323 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 1380 y(MPI)p 468 1380 V 17 w(Group)g(*newgroup\))75 1466 y(int)g(MPI)p 245 1466 V 17 w(Group)p 382 1466 V 17 w(free\(MPI)p 591 1466 V 16 w(Group)g(*group\))75 1553 y(int)g(MPI)p 245 1553 V 17 w(Comm)p 358 1553 V 17 w(size\(MPI)p 567 1553 V 16 w(Comm)g(comm,)g(int)h(*size\))75 1639 y(int)f(MPI)p 245 1639 V 17 w(Comm)p 358 1639 V 17 w(rank\(MPI)p 567 1639 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 1726 y(int)f(MPI)p 245 1726 V 17 w(Comm)p 358 1726 V 17 w(compare\(MPI)p 639 1726 V 16 w(Comm)g(comm1,)g(comm2,)g(int)g(*result\))75 1812 y(int)g(MPI)p 245 1812 V 17 w(Comm)p 358 1812 V 17 w(dup\(MPI)p 543 1812 V 16 w(Comm)g(comm,)h(MPI)p 894 1812 V 16 w(Comm)g(*newcomm\))75 1899 y(int)f(MPI)p 245 1899 V 17 w(Comm)p 358 1899 V 17 w(create\(MPI)p 615 1899 V 16 w(Comm)g(comm,)g(MPI)p 965 1899 V 17 w(Group)g(group,)g (MPI)p 1364 1899 V 17 w(Comm)g(*newcomm\))75 1985 y(int)g(MPI)p 245 1985 V 17 w(Comm)p 358 1985 V 17 w(split\(MPI)p 591 1985 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 1985 V 16 w(Comm)g(*newcomm\))75 2072 y(int)f(MPI)p 245 2072 V 17 w(Comm)p 358 2072 V 17 w(free\(MPI)p 567 2072 V 16 w(Comm)g(*comm\))75 2158 y(int)g(MPI)p 245 2158 V 17 w(Comm)p 358 2158 V 17 w(test)p 471 2158 V 16 w(inter\(MPI)p 703 2158 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 2244 y(int)g(MPI)p 245 2244 V 17 w(Comm)p 358 2244 V 17 w(remote)p 519 2244 V 16 w(size\(MPI)p 727 2244 V 16 w(Comm)h(comm,)f(int)g(*size\))75 2331 y(int)g(MPI)p 245 2331 V 17 w(Comm)p 358 2331 V 17 w(remote)p 519 2331 V 16 w(group\(MPI)p 751 2331 V 16 w(Comm)g(comm,)h(MPI)p 1102 2331 V 16 w(Group)g(*group\))75 2417 y(int)f(MPI)p 245 2417 V 17 w(Intercomm)p 478 2417 V 16 w(create\(MPI)p 734 2417 V 16 w(Comm)g(local)p 989 2417 V 17 w(comm,)g(int)g(local)p 1364 2417 V 17 w(leader,)393 2474 y(MPI)p 468 2474 V 17 w(Comm)g(peer)p 700 2474 V 17 w(comm,)g(int)h(remote)p 1100 2474 V 16 w(leader,)f(int)g(tag,)393 2530 y(MPI)p 468 2530 V 17 w(Comm)g(*newintercomm\))75 2617 y(int)g(MPI)p 245 2617 V 17 w(Intercomm)p 478 2617 V 16 w(merge\(MPI)p 710 2617 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 2673 y(MPI)p 468 2673 V 17 w(Comm)g(*newintracomm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 216 224 216 223 bop 75 -100 a Fx(216)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(int)23 b(MPI)p 245 45 15 2 v 17 w(Keyval)p 406 45 V 17 w(create\(int)f(copy)p 781 45 V 17 w(fn\(\),)h(int)g(delete)p 1180 45 V 17 w(fn\(\),)g(int)g (*keyval,)393 102 y(void*)g(extra)p 659 102 V 17 w(state\))75 188 y(int)g(MPI)p 245 188 V 17 w(Keyval)p 406 188 V 17 w(free\(int)f(*keyval\))75 274 y(int)h(MPI)p 245 274 V 17 w(Attr)p 358 274 V 17 w(put\(MPI)p 543 274 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 274 V 16 w(val\))75 361 y(int)g(MPI)p 245 361 V 17 w(Attr)p 358 361 V 17 w(get\(MPI)p 543 361 V 16 w(Comm)g(comm,)h (int)f(keyval,)g(void)g(**attribute)p 1491 361 V 16 w(val,)g(int)h (*flag\))75 447 y(int)f(MPI)p 245 447 V 17 w(Attr)p 358 447 V 17 w(delete\(MPI)p 615 447 V 16 w(Comm)g(comm,)g(int)h(keyval\)) 75 621 y Fn(A.6)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro)r(cess)e(T)-5 b(op)r(ologies)75 722 y Fu(int)23 b(MPI)p 245 722 V 17 w(Cart)p 358 722 V 17 w(create\(MPI)p 615 722 V 16 w(Comm)g(comm)p 846 722 V 17 w(old,)g(int)g(ndims,)g(int)h(*dims,)f(int)g(*periods,)393 779 y(int)h(reorder,)e(MPI)p 778 779 V 17 w(Comm)h(*comm)p 1034 779 V 17 w(cart\))75 865 y(int)g(MPI)p 245 865 V 17 w(Dims)p 358 865 V 17 w(create\(int)f(nnodes,)h(int)h(ndims,)f(int)g (*dims\))75 951 y(int)g(MPI)p 245 951 V 17 w(Graph)p 382 951 V 17 w(create\(MPI)p 639 951 V 16 w(Comm)g(comm)p 870 951 V 16 w(old,)h(int)f(nnodes,)g(int)h(*index,)f(int)g(*edges,)393 1008 y(int)h(reorder,)e(MPI)p 778 1008 V 17 w(Comm)h(*comm)p 1034 1008 V 17 w(graph\))75 1094 y(int)g(MPI)p 245 1094 V 17 w(Topo)p 358 1094 V 17 w(test\(MPI)p 567 1094 V 16 w(Comm)g(comm,)g(int)h(*status\))75 1181 y(int)f(MPI)p 245 1181 V 17 w(Graphdims)p 478 1181 V 16 w(get\(MPI)p 662 1181 V 16 w(Comm)h(comm,)f(int)g(*nnodes,)g(int)h(*nedges\))75 1267 y(int)f(MPI)p 245 1267 V 17 w(Graph)p 382 1267 V 17 w(get\(MPI)p 567 1267 V 16 w(Comm)g(comm,)g(int)h(maxindex,)f(int)g (maxedges,)g(int)g(*index,)393 1324 y(int)h(*edges\))75 1410 y(int)f(MPI)p 245 1410 V 17 w(Cartdim)p 430 1410 V 16 w(get\(MPI)p 614 1410 V 17 w(Comm)g(comm,)g(int)h(*ndims\))75 1497 y(int)f(MPI)p 245 1497 V 17 w(Cart)p 358 1497 V 17 w(get\(MPI)p 543 1497 V 16 w(Comm)g(comm,)h(int)f(maxdims,)g(int)g (*dims,)g(int)h(*periods,)393 1553 y(int)g(*coords\))75 1640 y(int)f(MPI)p 245 1640 V 17 w(Cart)p 358 1640 V 17 w(rank\(MPI)p 567 1640 V 16 w(Comm)g(comm,)g(int)h(*coords,)f(int)g (*rank\))75 1726 y(int)g(MPI)p 245 1726 V 17 w(Cart)p 358 1726 V 17 w(coords\(MPI)p 615 1726 V 16 w(Comm)g(comm,)g(int)h (rank,)f(int)g(maxdims,)g(int)g(*coords\))75 1812 y(int)g(MPI)p 245 1812 V 17 w(Graph)p 382 1812 V 17 w(neighbors)p 615 1812 V 16 w(count\(MPI)p 847 1812 V 16 w(Comm)g(comm,)g(int)h(rank,)f (int)g(*nneighbors\))75 1899 y(int)g(MPI)p 245 1899 V 17 w(Graph)p 382 1899 V 17 w(neighbors\(MPI)p 711 1899 V 15 w(Comm)g(comm,)h(int)f(rank,)g(int)h(*maxneighbors,)393 1955 y(int)g(*neighbors\))75 2042 y(int)f(MPI)p 245 2042 V 17 w(Cart)p 358 2042 V 17 w(shift\(MPI)p 591 2042 V 16 w(Comm)g(comm,)g(int)h(direction,)e(int)i(disp,)f(int)g(*rank)p 1681 2042 V 17 w(source,)393 2098 y(int)h(*rank)p 612 2098 V 16 w(dest\))75 2185 y(int)f(MPI)p 245 2185 V 17 w(Cart)p 358 2185 V 17 w(sub\(MPI)p 543 2185 V 16 w(Comm)g(comm,)h(int) f(*remain)p 1085 2185 V 16 w(dims,)g(MPI)p 1316 2185 V 17 w(Comm)h(*newcomm\))75 2271 y(int)f(MPI)p 245 2271 V 17 w(Cart)p 358 2271 V 17 w(map\(MPI)p 543 2271 V 16 w(Comm)g(comm,)h(int)f(ndims,)g(int)h(*dims,)f(int)g(*periods,)393 2328 y(int)h(*newrank\))75 2414 y(int)f(MPI)p 245 2414 V 17 w(Graph)p 382 2414 V 17 w(map\(MPI)p 567 2414 V 16 w(Comm)g(comm,)g(int)h(nnodes,)f(int)g(*index,)g(int)h(*edges,)393 2471 y(int)g(*newrank\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 217 225 217 224 bop 75 -100 a Fm(A.7.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(ENVIR)o (ONMENT)l(AL)g(INQUIR)l(Y)582 b Fx(217)75 45 y Fn(A.7)59 b(C)20 b(bindings)g(fo)n(r)h(Environmental)d(Inquiry)75 147 y Fu(int)23 b(MPI)p 245 147 15 2 v 17 w(Get)p 334 147 V 17 w(processor)p 567 147 V 16 w(name\(int)g(len,)g(char)g (*name\))75 233 y(int)g(MPI)p 245 233 V 17 w(Errhandler)p 502 233 V 16 w(create\(MPI)p 758 233 V 16 w(Handler)p 942 233 V 16 w(function)g(*function,)393 289 y(MPI)p 468 289 V 17 w(Errhandler)f(*errhandler\))75 376 y(int)h(MPI)p 245 376 V 17 w(Errhandler)p 502 376 V 16 w(set\(MPI)p 686 376 V 16 w(Comm)h(comm,)f(MPI)p 1037 376 V 17 w(Errhandler)f (errhandler\))75 462 y(int)h(MPI)p 245 462 V 17 w(Errhandler)p 502 462 V 16 w(get\(MPI)p 686 462 V 16 w(Comm)h(comm,)f(MPI)p 1037 462 V 17 w(Errhandler)f(*errhandler\))75 549 y(int)h(MPI)p 245 549 V 17 w(Errhandler)p 502 549 V 16 w(free\(MPI)p 710 549 V 16 w(Errhandler)g(*errhandler\))75 635 y(int)g(MPI)p 245 635 V 17 w(Error)p 382 635 V 17 w(string\(int)f(errorcode,)h(int)g (len,)g(char)h(*string\))75 722 y(int)f(MPI)p 245 722 V 17 w(Error)p 382 722 V 17 w(class\(int)f(errorcode,)h(int)g (*errorclass\))75 808 y(double)g(MPI)p 317 808 V 17 w(Wtime\(\))75 895 y(double)g(MPI)p 317 895 V 17 w(Wtick\(\))75 981 y(int)g(MPI)p 245 981 V 17 w(Init\(int)g(*argc,)g(char)g(***argv\))75 1068 y(int)g(MPI)p 245 1068 V 17 w(Finalize\(\))75 1154 y(int)g(MPI)p 245 1154 V 17 w(Initialized\(int)f(*flag\))75 1241 y(int)h(MPI)p 245 1241 V 17 w(Abort\(MPI)p 478 1241 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 1414 y Fn(A.8)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro\014ling)75 1515 y Fu(int)i(MPI)p 245 1515 V 17 w(Pcontrol\(const)f(int)i(level,)f Fp(:)8 b(:)g(:)d Fu(\))75 1688 y Fn(A.9)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(P)n(oint-to-P)n(oint)f(Communication)75 1790 y Fu(MPI)p 150 1790 V 17 w(SEND\(BUF,)i(COUNT,)h(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(IERROR\))170 1846 y()g(BUF\(*\))170 1903 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 1989 y(MPI)p 150 1989 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,) g(TAG,)g(COMM,)g(STATUS,)g(IERROR\))170 2046 y()g(BUF\(*\))170 2102 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 2102 V 16 w(STATUS)p 1623 2102 V 17 w(SIZE\),)170 2159 y(IERROR)75 2245 y(MPI)p 150 2245 V 17 w(GET)p 239 2245 V 17 w(COUNT\(STATUS,)f(DATATYPE,)g(COUNT,)h (IERROR\))170 2302 y(INTEGER)g(STATUS\(MPI)p 604 2302 V 16 w(STATUS)p 764 2302 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h(IERROR) 75 2388 y(MPI)p 150 2388 V 17 w(BSEND\(BUF,)f(COUNT,)h(DATATYPE,)g (DEST,)g(TAG,)g(COMM,)h(IERROR\))170 2444 y()f(BUF\(*\))170 2501 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 2587 y(MPI)p 150 2587 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 2644 y()f(BUF\(*\))170 2700 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 218 226 218 225 bop 75 -100 a Fx(218)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(RSEND\(BUF,)22 b(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)h (IERROR\))170 102 y()f(BUF\(*\))170 158 y(INTEGER)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 244 y(MPI)p 150 244 V 17 w(BUFFER)p 311 244 V 16 w(ATTACH\()g(BUFFER,)g(SIZE,)g (IERROR\))170 301 y()g(BUFFER\(*\))170 357 y(INTEGER)g(SIZE,)h (IERROR)75 444 y(MPI)p 150 444 V 17 w(BUFFER)p 311 444 V 16 w(DETACH\()f(BUFFER,)g(SIZE,)g(IERROR\))170 500 y()g(BUFFER\(*\))170 557 y(INTEGER)g(SIZE,)h(IERROR)75 643 y(MPI)p 150 643 V 17 w(ISEND\(BUF,)e(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(REQUEST,)e(IERROR\))170 700 y()h(BUF\(*\))170 756 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 843 y(MPI)p 150 843 V 17 w(IBSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 899 y()g(BUF\(*\))170 955 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,) g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1042 y(MPI)p 150 1042 V 17 w(ISSEND\(BUF,)g(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 1098 y()g(BUF\(*\))170 1155 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1241 y(MPI)p 150 1241 V 17 w(IRSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1298 y()g(BUF\(*\))170 1354 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1441 y(MPI)p 150 1441 V 17 w(IRECV\(BUF,)g(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 1497 y()g(BUF\(*\))170 1554 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR)75 1640 y(MPI)p 150 1640 V 17 w(WAIT\(REQUEST,)f (STATUS,)h(IERROR\))170 1696 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1696 V 16 w(STATUS)p 979 1696 V 16 w(SIZE\),)g(IERROR)75 1783 y(MPI)p 150 1783 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g (IERROR\))170 1839 y(LOGICAL)g(FLAG)170 1896 y(INTEGER)g(REQUEST,)g (STATUS\(MPI)p 819 1896 V 16 w(STATUS)p 979 1896 V 16 w(SIZE\),)g(IERROR)75 1982 y(MPI)p 150 1982 V 17 w(REQUEST)p 335 1982 V 16 w(FREE\(REQUEST,)f(IERROR\))170 2039 y(INTEGER)h (REQUEST,)g(IERROR)75 2125 y(MPI)p 150 2125 V 17 w(WAITANY\(COUNT,)f (ARRAY)p 645 2125 V 16 w(OF)p 709 2125 V 17 w(REQUESTS,)h(INDEX,)g (STATUS,)g(IERROR\))170 2182 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2182 V 17 w(OF)p 716 2182 V 17 w(REQUESTS\(*\),)f(INDEX,)h (STATUS\(MPI)p 1450 2182 V 16 w(STATUS)p 1610 2182 V 16 w(SIZE\),)170 2238 y(IERROR)75 2325 y(MPI)p 150 2325 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 2325 V 16 w(OF)p 709 2325 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g(STATUS,)g(IERROR\))170 2381 y(LOGICAL)g(FLAG)170 2438 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2438 V 17 w(OF)p 716 2438 V 17 w(REQUESTS\(*\),)f(INDEX,)h (STATUS\(MPI)p 1450 2438 V 16 w(STATUS)p 1610 2438 V 16 w(SIZE\),)170 2494 y(IERROR)75 2580 y(MPI)p 150 2580 V 17 w(WAITALL\(COUNT,)f(ARRAY)p 645 2580 V 16 w(OF)p 709 2580 V 17 w(REQUESTS,)h(ARRAY)p 1085 2580 V 16 w(OF)p 1149 2580 V 17 w(STATUSES,)g(IERROR\))170 2637 y(INTEGER)g(COUNT,)g (ARRAY)p 651 2637 V 17 w(OF)p 716 2637 V 17 w(REQUESTS\(*\),)170 2693 y(ARRAY)p 293 2693 V 17 w(OF)p 358 2693 V 17 w(STATUSES\(MPI)p 663 2693 V 15 w(STATUS)p 822 2693 V 17 w(SIZE,*\),)g(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 219 227 219 226 bop 75 -100 a Fm(A.9.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(POINT-TO-POINT)h(COMMUNICA)l(TION)185 b Fx(219)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(TESTALL\(COUNT,)22 b(ARRAY)p 645 45 V 16 w(OF)p 709 45 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 45 V 16 w(OF)p 1292 45 V 17 w(STATUSES,)g(IERROR\))170 102 y(LOGICAL)g(FLAG)170 158 y(INTEGER)g(COUNT,)g(ARRAY)p 651 158 V 17 w(OF)p 716 158 V 17 w(REQUESTS\(*\),)170 214 y(ARRAY)p 293 214 V 17 w(OF)p 358 214 V 17 w(STATUSES\(MPI)p 663 214 V 15 w(STATUS)p 822 214 V 17 w(SIZE,*\),)g(IERROR)75 301 y(MPI)p 150 301 V 17 w(WAITSOME\(INCOUNT,)e(ARRAY)p 716 301 V 17 w(OF)p 781 301 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 301 V 16 w(OF)p 1459 301 V 17 w(INDICES,)393 357 y(ARRAY)p 516 357 V 17 w(OF)p 581 357 V 17 w(STATUSES,)f(IERROR\))170 414 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 414 V 17 w(OF)p 764 414 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 414 V 17 w(OF)p 1514 414 V 17 w(INDICES\(*\),)170 470 y(ARRAY)p 293 470 V 17 w(OF)p 358 470 V 17 w(STATUSES\(MPI)p 663 470 V 15 w(STATUS)p 822 470 V 17 w(SIZE,*\),)h(IERROR)75 557 y(MPI)p 150 557 V 17 w(TESTSOME\(INCOUNT,)e(ARRAY)p 716 557 V 17 w(OF)p 781 557 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 557 V 16 w(OF)p 1459 557 V 17 w(INDICES,)393 613 y(ARRAY)p 516 613 V 17 w(OF)p 581 613 V 17 w(STATUSES,)f(IERROR\))170 670 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 670 V 17 w(OF)p 764 670 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 670 V 17 w(OF)p 1514 670 V 17 w(INDICES\(*\),)170 726 y(ARRAY)p 293 726 V 17 w(OF)p 358 726 V 17 w(STATUSES\(MPI)p 663 726 V 15 w(STATUS)p 822 726 V 17 w(SIZE,*\),)h(IERROR)75 813 y(MPI)p 150 813 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 869 y(LOGICAL)g(FLAG)170 925 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 925 V 16 w(STATUS)p 1218 925 V 16 w(SIZE\),)g(IERROR)75 1012 y(MPI)p 150 1012 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g (IERROR\))170 1068 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1068 V 16 w(STATUS)p 1218 1068 V 16 w(SIZE\),)g(IERROR)75 1155 y(MPI)p 150 1155 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1211 y(INTEGER)h(REQUEST,)g(IERROR)75 1298 y(MPI)p 150 1298 V 17 w(TEST)p 263 1298 V 16 w(CANCELLED\(STATUS,)f(FLAG,)h (IERROR\))170 1354 y(LOGICAL)g(FLAG)170 1411 y(INTEGER)g(STATUS\(MPI)p 604 1411 V 16 w(STATUS)p 764 1411 V 17 w(SIZE\),)g(IERROR)75 1497 y(MPI)p 150 1497 V 17 w(SEND)p 263 1497 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1554 y()g(BUF\(*\))170 1610 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1696 y(MPI)p 150 1696 V 17 w(BSEND)p 287 1696 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1753 y()g(BUF\(*\))170 1809 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1896 y(MPI)p 150 1896 V 17 w(SSEND)p 287 1896 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1952 y()g(BUF\(*\))170 2009 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 2095 y(MPI)p 150 2095 V 17 w(RSEND)p 287 2095 V 16 w(INIT\(BUF,)h(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2152 y()g(BUF\(*\))170 2208 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 2295 y(MPI)p 150 2295 V 17 w(RECV)p 263 2295 V 16 w(INIT\(BUF,)h(COUNT,)g(DATATYPE,) g(SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2351 y()g(BUF\(*\))170 2408 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 2494 y(MPI)p 150 2494 V 17 w(START\(REQUEST,)f(IERROR\))170 2550 y(INTEGER)h(REQUEST,)g (IERROR)75 2637 y(MPI)p 150 2637 V 17 w(STARTALL\(COUNT,)f(ARRAY)p 669 2637 V 16 w(OF)p 733 2637 V 17 w(REQUESTS,)h(IERROR\))170 2693 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2693 V 17 w(OF)p 716 2693 V 17 w(REQUESTS\(*\),)f(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 220 228 220 227 bop 75 -100 a Fx(220)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(SENDRECV\(SENDBUF,)21 b(SENDCOUNT,)i(SENDTYPE,)f(DEST,)i (SENDTAG,)e(RECVBUF,)393 102 y(RECVCOUNT,)h(RECVTYPE,)f(SOURCE,)h (RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 158 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 214 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h(RECVTYPE,)170 271 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 271 V 16 w(STATUS)p 1122 271 V 16 w(SIZE\),)h(IERROR)75 357 y(MPI)p 150 357 V 17 w(SENDRECV)p 359 357 V 16 w(REPLACE\(BUF,)e (COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)393 414 y(COMM,)g(STATUS,)g(IERROR\))170 470 y()g(BUF\(*\))170 527 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g (RECVTAG,)g(COMM,)170 583 y(STATUS\(MPI)p 413 583 V 16 w(STATUS)p 573 583 V 17 w(SIZE\),)g(IERROR)75 670 y(MPI)p 150 670 V 17 w(TYPE)p 263 670 V 16 w(CONTIGUOUS\(COUNT,)f(OLDTYPE,)h (NEWTYPE,)f(IERROR\))170 726 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(NEWTYPE,)g (IERROR)75 813 y(MPI)p 150 813 V 17 w(TYPE)p 263 813 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h (IERROR\))170 869 y(INTEGER)g(COUNT,)g(BLOCKLENGTH,)g(STRIDE,)g (OLDTYPE,)f(NEWTYPE,)h(IERROR)75 955 y(MPI)p 150 955 V 17 w(TYPE)p 263 955 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)f (OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1012 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 1098 y(MPI)p 150 1098 V 17 w(TYPE)p 263 1098 V 16 w(INDEXED\(COUNT,)f (ARRAY)p 757 1098 V 17 w(OF)p 822 1098 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 1098 V 16 w(OF)p 1357 1098 V 17 w(DISPLACEMENTS,)393 1155 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1211 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1211 V 17 w(OF)p 716 1211 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1211 V 17 w(OF)p 1323 1211 V 17 w(DISPLACEMENTS\(*\),)170 1268 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1354 y(MPI)p 150 1354 V 17 w(TYPE)p 263 1354 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 1354 V 17 w(OF)p 846 1354 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 1354 V 16 w(OF)p 1381 1354 V 17 w(DISPLACEMENTS,)393 1411 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1467 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1467 V 17 w(OF)p 716 1467 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1467 V 17 w(OF)p 1323 1467 V 17 w(DISPLACEMENTS\(*\),)170 1524 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1610 y(MPI)p 150 1610 V 17 w(TYPE)p 263 1610 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 1610 V 17 w(OF)p 798 1610 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 1610 V 16 w(OF)p 1333 1610 V 17 w(DISPLACEMENTS,)393 1666 y(ARRAY)p 516 1666 V 17 w(OF)p 581 1666 V 17 w(TYPES,)h(NEWTYPE,)f (IERROR\))170 1723 y(INTEGER)h(COUNT,)g(ARRAY)p 651 1723 V 17 w(OF)p 716 1723 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 1723 V 17 w(OF)p 1323 1723 V 17 w(DISPLACEMENTS\(*\),)170 1779 y(ARRAY)p 293 1779 V 17 w(OF)p 358 1779 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)75 1866 y(MPI)p 150 1866 V 17 w(ADDRESS\(LOCATION,)e (ADDRESS,)i(IERROR\))170 1922 y()g(LOCATION\(*\))170 1979 y(INTEGER)g(ADDRESS,)g(IERROR)75 2065 y(MPI)p 150 2065 V 17 w(TYPE)p 263 2065 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h (IERROR\))170 2122 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)75 2208 y(MPI)p 150 2208 V 17 w(TYPE)p 263 2208 V 16 w(SIZE\(DATATYPE,)f (SIZE,)i(IERROR\))170 2265 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)75 2351 y(MPI)p 150 2351 V 17 w(TYPE)p 263 2351 V 16 w(COUNT\(DATATYPE,)f (COUNT,)h(IERROR\))170 2408 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)75 2494 y(MPI)p 150 2494 V 17 w(TYPE)p 263 2494 V 16 w(LB\()h(DATATYPE,)e (DISPLACEMENT,)h(IERROR\))170 2550 y(INTEGER)g(DATATYPE,)g (DISPLACEMENT,)f(IERROR)75 2637 y(MPI)p 150 2637 V 17 w(TYPE)p 263 2637 V 16 w(UB\()i(DATATYPE,)e(DISPLACEMENT,)h(IERROR\)) 170 2693 y(INTEGER)g(DATATYPE,)g(DISPLACEMENT,)f(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 221 229 221 228 bop 75 -100 a Fm(A.10.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(COLLECTIVE)g(COMMUNICA)l(TION)257 b Fx(221)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(COMMIT\(DATATYPE,)22 b(IERROR\))170 102 y(INTEGER)h (DATATYPE,)g(IERROR)75 188 y(MPI)p 150 188 V 17 w(TYPE)p 263 188 V 16 w(FREE\(DATATYPE,)f(IERROR\))170 244 y(INTEGER)h (DATATYPE,)g(IERROR)75 331 y(MPI)p 150 331 V 17 w(GET)p 239 331 V 17 w(ELEMENTS\(STATUS,)e(DATATYPE,)i(COUNT,)g(IERROR\))170 387 y(INTEGER)g(STATUS\(MPI)p 604 387 V 16 w(STATUS)p 764 387 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h(IERROR)75 474 y(MPI)p 150 474 V 17 w(PACK\(INBUF,)f(INCOUNT,)h(DATATYPE,)g (OUTBUF,)f(OUTCOUNT,)h(POSITION,)g(COMM,)393 530 y(IERROR\))170 587 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 643 y(INTEGER)g(INCOUNT,)g (DATATYPE,)g(OUTCOUNT,)f(POSITION,)h(COMM,)g(IERROR)75 730 y(MPI)p 150 730 V 17 w(UNPACK\(INBUF,)f(INSIZE,)h(POSITION,)f (OUTBUF,)h(OUTCOUNT,)g(DATATYPE,)g(SOURCE,)393 786 y(COMM,)g(IERROR\)) 170 843 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 899 y(INTEGER)g(INSIZE,)g(POSITION,)g(OUTCOUNT,)g(DATATYPE,)f(COMM,)h (IERROR)75 985 y(MPI)p 150 985 V 17 w(PACK)p 263 985 V 16 w(SIZE\(INCOUNT,)f(DATATYPE,)h(COMM,)g(SIZE,)g(IERROR\))170 1042 y(INTEGER)g(INCOUNT,)g(DATATYPE,)g(COMM,)g(SIZE,)g(IERROR)75 1215 y Fn(A.10)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Collective)e (Communication)75 1317 y Fu(MPI)p 150 1317 V 17 w(BARRIER\(COMM,)j (IERROR\))170 1373 y(INTEGER)h(COMM,)h(IERROR)75 1460 y(MPI)p 150 1460 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,)g (COMM,)g(IERROR\))170 1516 y()g(BUFFER\(*\))170 1572 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 1659 y(MPI)p 150 1659 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,) h(RECVBUF,)f(RECVCOUNT,)h(RECVTYPE,)393 1715 y(ROOT,)g(COMM,)h (IERROR\))170 1772 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 1828 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (ROOT,)g(COMM,)g(IERROR)75 1915 y(MPI)p 150 1915 V 17 w(GATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f(RECVCOUNTS,)h (DISPLS,)393 1971 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 2028 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2084 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h (RECVTYPE,)g(ROOT,)170 2141 y(COMM,)h(IERROR)75 2227 y(MPI)p 150 2227 V 17 w(SCATTER\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)g (RECVBUF,)f(RECVCOUNT,)h(RECVTYPE,)393 2283 y(ROOT,)g(COMM,)h(IERROR\)) 170 2340 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2396 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (ROOT,)g(COMM,)g(IERROR)75 2483 y(MPI)p 150 2483 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i(DISPLS,)g(SENDTYPE,)f(RECVBUF,)h (RECVCOUNT,)393 2539 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 2596 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 222 230 222 229 bop 75 -100 a Fx(222)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fu(INTEGER)23 b(SENDCOUNTS\(*\),)f (DISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 102 y(COMM,)h(IERROR)75 188 y(MPI)p 150 188 15 2 v 17 w(ALLGATHER\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,) g(RECVTYPE,)393 244 y(COMM,)g(IERROR\))170 301 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 357 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g(IERROR)75 444 y(MPI)p 150 444 V 17 w(ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(RECVBUF,)h(RECVCOUNTS,)f(DISPLS,)393 500 y(RECVTYPE,)h(COMM,)g(IERROR\))170 557 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 613 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f (RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g(COMM,)170 670 y(IERROR)75 756 y(MPI)p 150 756 V 17 w(ALLTOALL\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 813 y(COMM,)g(IERROR\))170 869 y()g(SENDBUF\(*\),)g(RECVBUF\(*\)) 170 925 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (COMM,)g(IERROR)75 1012 y(MPI)p 150 1012 V 17 w(ALLTOALLV\(SENDBUF,)e (SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h(RECVBUF,)g(RECVCOUNTS,)393 1068 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g(IERROR\))170 1125 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1181 y(INTEGER)g (SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNTS\(*\),)g (RDISPLS\(*\),)170 1238 y(RECVTYPE,)h(COMM,)g(IERROR)75 1324 y(MPI)p 150 1324 V 17 w(REDUCE\(SENDBUF,)f(RECVBUF,)g(COUNT,)h (DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR\))170 1381 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1437 y(INTEGER)g(COUNT,)g (DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR)75 1524 y(MPI)p 150 1524 V 17 w(OP)p 215 1524 V 17 w(CREATE\()g(FUNCTION,)f(COMMUTE,)h (OP,)g(IERROR\))170 1580 y(EXTERNAL)g(FUNCTION)170 1636 y(LOGICAL)g(COMMUTE)170 1693 y(INTEGER)g(OP,)h(IERROR)75 1779 y(MPI)p 150 1779 V 17 w(OP)p 215 1779 V 17 w(FREE\()f(OP,)g (IERROR\))170 1836 y(INTEGER)g(OP,)h(IERROR)75 1922 y(MPI)p 150 1922 V 17 w(ALLREDUCE\(SENDBUF,)d(RECVBUF,)i(COUNT,)g(DATATYPE,)g (OP,)g(COMM,)g(IERROR\))170 1979 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2035 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)75 2122 y(MPI)p 150 2122 V 17 w(REDUCE)p 311 2122 V 16 w(SCATTER\(SENDBUF,)e(RECVBUF,)h(RECVCOUNTS,)f(DATATYPE,)h (OP,)g(COMM,)393 2178 y(IERROR\))170 2235 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2291 y(INTEGER)g(RECVCOUNTS,)g(DATATYPE,)f(OP,)i (COMM,)f(IERROR)75 2377 y(MPI)p 150 2377 V 17 w(SCAN\(SENDBUF,)f (RECVBUF,)h(COUNT,)g(DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 2434 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2490 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 223 231 223 230 bop 75 -100 a Fm(A.11.)27 b(F)o(OR)l(TRAN)11 b(BINDINGS)g(F)o(OR)g(GR)o(OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)p Fx(223)75 45 y Fn(A.11)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i (Groups,)f(Contexts,)d(and)j(Communicato)n(rs)75 147 y Fu(MPI)p 150 147 15 2 v 17 w(GROUP)p 287 147 V 16 w(SIZE\(GROUP,)j (SIZE,)g(IERROR\))170 203 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 289 y(MPI)p 150 289 V 17 w(GROUP)p 287 289 V 16 w(RANK\(GROUP,)f(RANK,) g(IERROR\))170 346 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)75 432 y(MPI)p 150 432 V 17 w(GROUP)p 287 432 V 16 w(TRANSLATE)p 519 432 V 16 w(RANKS\(GROUP1,)e(N,)i(RANKS1,)f(GROUP2,)g(RANKS2,)g (IERROR\))170 489 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1\(*\),)e(GROUP2,)h (RANKS2\(*\),)g(IERROR)75 575 y(MPI)p 150 575 V 17 w(GROUP)p 287 575 V 16 w(COMPARE\(GROUP1,)f(GROUP2,)h(RESULT,)g(IERROR\))170 632 y(INTEGER)g(GROUP1,)g(GROUP2,)g(RESULT,)g(IERROR)75 718 y(MPI)p 150 718 V 17 w(COMM)p 263 718 V 16 w(GROUP\(COMM,)g(GROUP,) g(IERROR\))170 775 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)75 861 y(MPI)p 150 861 V 17 w(GROUP)p 287 861 V 16 w(UNION\(GROUP1,)f (GROUP2,)h(NEWGROUP,)g(IERROR\))170 918 y(INTEGER)g(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 1004 y(MPI)p 150 1004 V 17 w(GROUP)p 287 1004 V 16 w(INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\)) 170 1060 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1147 y(MPI)p 150 1147 V 17 w(GROUP)p 287 1147 V 16 w (DIFFERENCE\(GROUP1,)f(GROUP2,)h(NEWGROUP,)f(IERROR\))170 1203 y(INTEGER)h(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1290 y(MPI)p 150 1290 V 17 w(GROUP)p 287 1290 V 16 w(INCL\(GROUP,)g(N,) g(RANKS,)g(NEWGROUP,)g(IERROR\))170 1346 y(INTEGER)g(GROUP,)g(N,)h (RANKS\(*\),)f(NEWGROUP,)f(IERROR)75 1433 y(MPI)p 150 1433 V 17 w(GROUP)p 287 1433 V 16 w(EXCL\(GROUP,)h(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 1489 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)75 1576 y(MPI)p 150 1576 V 17 w(GROUP)p 287 1576 V 16 w(RANGE)p 423 1576 V 17 w(INCL\(GROUP,)g(N,)i(RANGES,)f (NEWGROUP,)f(IERROR\))170 1632 y(INTEGER)h(GROUP,)g(N,)h (RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1719 y(MPI)p 150 1719 V 17 w(GROUP)p 287 1719 V 16 w(RANGE)p 423 1719 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1775 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1862 y(MPI)p 150 1862 V 17 w(GROUP)p 287 1862 V 16 w(FREE\(GROUP,)g (IERROR\))170 1918 y(INTEGER)g(GROUP,)g(IERROR)75 2004 y(MPI)p 150 2004 V 17 w(COMM)p 263 2004 V 16 w(SIZE\(COMM,)g(SIZE,)g (IERROR\))170 2061 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 2147 y(MPI)p 150 2147 V 17 w(COMM)p 263 2147 V 16 w(RANK\(COMM,)g (RANK,)g(IERROR\))170 2204 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 2290 y(MPI)p 150 2290 V 17 w(COMM)p 263 2290 V 16 w(COMPARE\(COMM1,)f (COMM2,)h(RESULT,)g(IERROR\))170 2347 y(INTEGER)g(COMM1,)g(COMM2,)g (RESULT,)g(IERROR)75 2433 y(MPI)p 150 2433 V 17 w(COMM)p 263 2433 V 16 w(DUP\(COMM,)g(NEWCOMM,)g(IERROR\))170 2490 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)75 2576 y(MPI)p 150 2576 V 17 w(COMM)p 263 2576 V 16 w(CREATE\(COMM,)h(GROUP,)g (NEWCOMM,)f(IERROR\))170 2633 y(INTEGER)h(COMM,)h(GROUP,)f(NEWCOMM,)f (IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 224 232 224 231 bop 75 -100 a Fx(224)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(SPLIT\(COMM,)23 b(COLOR,)g(KEY,)g (NEWCOMM,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g (NEWCOMM,)g(IERROR)75 188 y(MPI)p 150 188 V 17 w(COMM)p 263 188 V 16 w(FREE\(COMM,)g(IERROR\))170 244 y(INTEGER)g(COMM,)h (IERROR)75 331 y(MPI)p 150 331 V 17 w(COMM)p 263 331 V 16 w(TEST)p 375 331 V 17 w(INTER\(COMM,)e(FLAG,)i(IERROR\))170 387 y(INTEGER)f(COMM,)h(IERROR)170 444 y(LOGICAL)f(FLAG)75 530 y(MPI)p 150 530 V 17 w(COMM)p 263 530 V 16 w(REMOTE)p 423 530 V 17 w(SIZE\(COMM,)f(SIZE,)h(IERROR\))170 587 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 673 y(MPI)p 150 673 V 17 w(COMM)p 263 673 V 16 w(REMOTE)p 423 673 V 17 w(GROUP\(COMM,)f(GROUP,)h(IERROR\))170 730 y(INTEGER)g(COMM,)h(GROUP,)f (IERROR)75 816 y(MPI)p 150 816 V 17 w(INTERCOMM)p 383 816 V 16 w(CREATE\(LOCAL)p 687 816 V 15 w(COMM,)g(LOCAL)p 965 816 V 17 w(LEADER,)g(PEER)p 1269 816 V 16 w(COMM,)h(REMOTE)p 1573 816 V 16 w(LEADER,)f(TAG,)393 873 y(NEWINTERCOMM,)f(IERROR\))170 929 y(INTEGER)h(LOCAL)p 484 929 V 17 w(COMM,)g(LOCAL)p 764 929 V 17 w(LEADER,)f(PEER)p 1067 929 V 17 w(COMM,)h(REMOTE)p 1371 929 V 17 w(LEADER,)g(TAG,)170 985 y(NEWINTERCOMM,)g(IERROR)75 1072 y(MPI)p 150 1072 V 17 w(INTERCOMM)p 383 1072 V 16 w(MERGE\(INTERCOMM,)e(HIGH,)j(INTRACOMM,)e(IERROR\))170 1128 y(INTEGER)h(INTERCOMM,)g(INTRACOMM,)f(IERROR)170 1185 y(LOGICAL)h(HIGH)75 1271 y(MPI)p 150 1271 V 17 w(KEYVAL)p 311 1271 V 16 w(CREATE\(COPY)p 591 1271 V 16 w(FN,)g(DELETE)p 846 1271 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1269 1271 V 16 w(STATE,)h(IERROR\))170 1328 y(EXTERNAL)f(COPY)p 484 1328 V 17 w(FN,)g(DELETE)p 740 1328 V 17 w(FN)170 1384 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 1384 V 17 w(STATE,)g(IERROR)75 1471 y(FUNCTION)g(COPY)p 389 1471 V 16 w(FN\(OLDCOMM,)g(NEWCOMM,)g (KEYVAL,)f(EXTRA)p 1217 1471 V 17 w(STATE\))170 1527 y(INTEGER)h(OLDCOMM,)g(NEWCOMM,)g(KEYVAL,)g(EXTRA)p 1105 1527 V 16 w(STATE)75 1614 y(FUNCTION)g(DELETE)p 437 1614 V 16 w(FN\(COMM,)g(KEYVAL,)g(EXTRA)p 979 1614 V 16 w(STATE\))170 1670 y(INTEGER)g(COMM,)h(KEYVAL,)e(EXTRA)p 818 1670 V 17 w(STATE)75 1756 y(MPI)p 150 1756 V 17 w(KEYVAL)p 311 1756 V 16 w(FREE\(KEYVAL,)g(IERROR\))170 1813 y(INTEGER)h(KEYVAL,)g (IERROR)75 1899 y(MPI)p 150 1899 V 17 w(ATTR)p 263 1899 V 16 w(PUT\(COMM,)g(KEYVAL,)g(ATTRIBUTE)p 925 1899 V 16 w(VAL,)g(IERROR\))170 1956 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 1956 V 16 w(VAL,)i(IERROR)75 2042 y(MPI)p 150 2042 V 17 w(ATTR)p 263 2042 V 16 w(GET\(COMM,)f(KEYVAL,)g(ATTRIBUTE)p 925 2042 V 16 w(VAL,)g(FLAG,)g(IERROR\))170 2099 y(INTEGER)g(COMM,)h (KEYVAL,)e(ATTRIBUTE)p 914 2099 V 16 w(VAL,)i(IERROR)170 2155 y(LOGICAL)f(FLAG)75 2242 y(MPI)p 150 2242 V 17 w(ATTR)p 263 2242 V 16 w(DELETE\(COMM,)g(KEYVAL,)g(IERROR\))170 2298 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)75 2471 y Fn(A.12)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro)r(cess)e(T)-5 b(op)r(ologies)75 2573 y Fu(MPI)p 150 2573 V 17 w(CART)p 263 2573 V 16 w(CREATE\(COMM)p 543 2573 V 16 w(OLD,)23 b(NDIMS,)g(DIMS,)h(PERIODS,)e(REORDER,)h(COMM)p 1514 2573 V 17 w(CART,)g(IERROR\))170 2629 y(INTEGER)g(COMM)p 460 2629 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(COMM)p 1074 2629 V 17 w(CART,)g(IERROR)170 2686 y(LOGICAL)g(PERIODS\(*\),)g (REORDER)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 225 233 225 232 bop 75 -100 a Fm(A.13.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(ENVIR)o(ONMENT)l(AL)g(INQUIR)l(Y)364 b Fx(225)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(DIMS)p 263 45 V 16 w(CREATE\(NNODES,)22 b(NDIMS,)h(DIMS,)h(IERROR\))170 102 y(INTEGER)f(NNODES,)g(NDIMS,)g(DIMS\(*\),)g(IERROR)75 188 y(MPI)p 150 188 V 17 w(GRAPH)p 287 188 V 16 w(CREATE\(COMM)p 567 188 V 16 w(OLD,)g(NNODES,)g(INDEX,)g(EDGES,)g(REORDER,)g(COMM)p 1538 188 V 17 w(GRAPH,)393 244 y(IERROR\))170 301 y(INTEGER)g(COMM)p 460 301 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g(EDGES\(*\),)f(COMM)p 1360 301 V 17 w(GRAPH,)h(IERROR)170 357 y(LOGICAL)g(REORDER)75 444 y(MPI)p 150 444 V 17 w(TOPO)p 263 444 V 16 w(TEST\(COMM,)g(STATUS,) g(IERROR\))170 500 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)75 587 y(MPI)p 150 587 V 17 w(GRAPHDIMS)p 383 587 V 16 w(GET\(COMM,)g (NNODES,)h(NEDGES,)g(IERROR\))170 643 y(INTEGER)g(COMM,)h(NNODES,)e (NEDGES,)h(IERROR)75 730 y(MPI)p 150 730 V 17 w(GRAPH)p 287 730 V 16 w(GET\(COMM,)g(MAXINDEX,)g(MAXEDGES,)f(INDEX,)h(EDGES,)g (IERROR\))170 786 y(INTEGER)g(COMM,)h(MAXINDEX,)e(MAXEDGES,)h (INDEX\(*\),)f(EDGES\(*\),)h(IERROR)75 873 y(MPI)p 150 873 V 17 w(CARTDIM)p 335 873 V 16 w(GET\(COMM,)g(NDIMS,)g(IERROR\))170 929 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)75 1015 y(MPI)p 150 1015 V 17 w(CART)p 263 1015 V 16 w(GET\(COMM,)g(MAXDIMS,)g(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 1072 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(COORDS\(*\),)g(IERROR)170 1128 y(LOGICAL)g(PERIODS\(*\)) 75 1215 y(MPI)p 150 1215 V 17 w(CART)p 263 1215 V 16 w(RANK\(COMM,)g(COORDS,)g(RANK,)g(IERROR\))170 1271 y(INTEGER)g(COMM,)h (COORDS\(*\),)e(RANK,)h(IERROR)75 1358 y(MPI)p 150 1358 V 17 w(CART)p 263 1358 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g (COORDS,)f(IERROR\))170 1414 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f (COORDS\(*\),)h(IERROR)75 1501 y(MPI)p 150 1501 V 17 w(GRAPH)p 287 1501 V 16 w(NEIGHBORS)p 519 1501 V 16 w(COUNT\(COMM,)g (RANK,)g(NNEIGHBORS,)f(IERROR\))170 1557 y(INTEGER)h(COMM,)h(RANK,)f (NNEIGHBORS,)f(IERROR)75 1644 y(MPI)p 150 1644 V 17 w(GRAPH)p 287 1644 V 16 w(NEIGHBORS\(COMM,)g(RANK,)h(MAXNEIGHBORS,)f(NEIGHBORS,)h (IERROR\))170 1700 y(INTEGER)g(COMM,)h(RANK,)f(MAXNEIGHBORS,)f (NEIGHBORS\(*\),)g(IERROR)75 1786 y(MPI)p 150 1786 V 17 w(CART)p 263 1786 V 16 w(SHIFT\(COMM,)h(DIRECTION,)f(DISP,)h(RANK)p 1067 1786 V 17 w(SOURCE,)g(RANK)p 1371 1786 V 17 w(DEST,)g(IERROR\))170 1843 y(INTEGER)g(COMM,)h(DIRECTION,)e(DISP,)h(RANK)p 1009 1843 V 17 w(SOURCE,)g(RANK)p 1313 1843 V 16 w(DEST,)h(IERROR)75 1929 y(MPI)p 150 1929 V 17 w(CART)p 263 1929 V 16 w(SUB\(COMM,)f (REMAIN)p 662 1929 V 16 w(DIMS,)h(NEWCOMM,)e(IERROR\))170 1986 y(INTEGER)h(COMM,)h(NEWCOMM,)e(IERROR)170 2042 y(LOGICAL)h(REMAIN) p 508 2042 V 17 w(DIMS\(*\))75 2129 y(MPI)p 150 2129 V 17 w(CART)p 263 2129 V 16 w(MAP\(COMM,)g(NDIMS,)g(DIMS,)g(PERIODS,)g (NEWRANK,)g(IERROR\))170 2185 y(INTEGER)g(COMM,)h(NDIMS,)f(DIMS\(*\),)f (NEWRANK,)h(IERROR)170 2242 y(LOGICAL)g(PERIODS\(*\))75 2328 y(MPI)p 150 2328 V 17 w(GRAPH)p 287 2328 V 16 w(MAP\(COMM,)g (NNODES,)g(INDEX,)g(EDGES,)g(NEWRANK,)g(IERROR\))170 2385 y(INTEGER)g(COMM,)h(NNODES,)e(INDEX\(*\),)h(EDGES\(*\),)g (NEWRANK,)f(IERROR)75 2558 y Fn(A.13)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Environmental)d(Inquiry)75 2659 y Fu(MPI)p 150 2659 V 17 w(GET)p 239 2659 V 17 w(PROCESSOR)p 472 2659 V 15 w(NAME\(LEN,)23 b(NAME,)g(IERROR\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 226 234 226 233 bop 75 -100 a Fx(226)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fu(CHARACTER*\(*\))23 b(NAME)170 102 y(INTEGER)g(LEN,)h(IERROR)75 188 y(MPI)p 150 188 15 2 v 17 w(ERRHANDLER)p 407 188 V 16 w(CREATE\(FUNCTION,)d (HANDLER,)i(IERROR\))170 244 y(EXTERNAL)g(FUNCTION)170 301 y(INTEGER)g(ERRHANDLER,)g(IERROR)75 387 y(MPI)p 150 387 V 17 w(ERRHANDLER)p 407 387 V 16 w(SET\(COMM,)f(ERRHANDLER,)h (IERROR\))170 444 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)75 530 y(MPI)p 150 530 V 17 w(ERRHANDLER)p 407 530 V 16 w(GET\(COMM,)g(ERRHANDLER,)h(IERROR\))170 587 y(INTEGER)g(COMM,)h (ERRHANDLER,)e(IERROR)75 673 y(MPI)p 150 673 V 17 w(ERRHANDLER)p 407 673 V 16 w(FREE\(ERRHANDLER,)f(IERROR\))170 730 y(INTEGER)i (ERRHANDLER,)g(IERROR)75 816 y(MPI)p 150 816 V 17 w(ERROR)p 287 816 V 16 w(STRING\(ERRORCODE,)f(LEN,)h(STRING,)g(IERROR\))170 873 y(INTEGER)g(ERRORCODE,)g(LEN,)g(IERROR)170 929 y(CHARACTER\(*\))g (STRING)75 1015 y(MPI)p 150 1015 V 17 w(ERROR)p 287 1015 V 16 w(CLASS\(ERRORCODE,)f(ERRORCLASS,)g(IERROR\))170 1072 y(INTEGER)h(ERRORCODE,)g(ERRORCLASS,)f(IERROR)75 1158 y(DOUBLE)h(PRECISION)g(MPI)p 556 1158 V 16 w(WTIME\(\))75 1245 y(DOUBLE)g(PRECISION)g(MPI)p 556 1245 V 16 w(WTICK\(\))75 1331 y(MPI)p 150 1331 V 17 w(INIT\(IERROR\))170 1388 y(INTEGER)g(IERROR)75 1474 y(MPI)p 150 1474 V 17 w(FINALIZE\(IERROR\)) 170 1531 y(INTEGER)g(IERROR)75 1617 y(MPI)p 150 1617 V 17 w(INITIALIZED\(FLAG,)e(IERROR\))170 1674 y(LOGICAL)i(FLAG)170 1730 y(INTEGER)g(IERROR)75 1816 y(MPI)p 150 1816 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1873 y(INTEGER)g(COMM,)h (ERRORCODE,)e(IERROR)75 2046 y Fn(A.14)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro\014ling)75 2148 y Fu(MPI)p 150 2148 V 17 w(PCONTROL\(level\))170 2204 y(INTEGER)i(LEVEL,)g Fp(:)8 b(:)g(:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 227 235 227 234 bop 75 283 a Fr(MPI)40 b(F)-10 b(unction)40 b(Index)75 490 y Fx(MPI)p 167 490 14 2 v 16 w(ABOR)l(T,)16 b(197)75 547 y(MPI)p 167 547 V 16 w(ADDRESS,)g(67)75 603 y(MPI)p 167 603 V 16 w(ALLGA)l(THER,)g(107)75 660 y(MPI)p 167 660 V 16 w(ALLGA)l(THER)-5 b(V,)16 b(108)75 716 y(MPI)p 167 716 V 16 w(ALLREDUCE,)g(122)75 773 y(MPI)p 167 773 V 16 w(ALL)l(TO)o(ALL,)h(109)75 829 y(MPI)p 167 829 V 16 w(ALL)l(TO)o(ALL)-5 b(V,)17 b(110)75 886 y(MPI)p 167 886 V 16 w(A)l(TTR)p 312 886 V 17 w(DELETE,)e(169)75 942 y(MPI)p 167 942 V 16 w(A)l(TTR)p 312 942 V 17 w(GET,)f(169)75 999 y(MPI)p 167 999 V 16 w(A)l(TTR)p 312 999 V 17 w(PUT,)h(168)75 1096 y(MPI)p 167 1096 V 16 w(BARRIER,)i(93)75 1153 y(MPI)p 167 1153 V 16 w(BCAST,)e(93)75 1209 y(MPI)p 167 1209 V 16 w(BSEND,)g(27)75 1266 y(MPI)p 167 1266 V 16 w(BSEND)p 340 1266 V 17 w(INIT,)g(53)75 1322 y(MPI)p 167 1322 V 16 w(BUFFER)p 373 1322 V 17 w(A)l(TT)l(A)o(CH,)f(33)75 1379 y(MPI)p 167 1379 V 16 w(BUFFER)p 373 1379 V 17 w(DET)l(A)o(CH,)g (33)75 1477 y(MPI)p 167 1477 V 16 w(CANCEL,)h(51)75 1533 y(MPI)p 167 1533 V 16 w(CAR)l(T)p 312 1533 V 17 w(COORDS,)h(182)75 1590 y(MPI)p 167 1590 V 16 w(CAR)l(T)p 312 1590 V 17 w(CREA)l(TE,)f(177)75 1646 y(MPI)p 167 1646 V 16 w(CAR)l(T)p 312 1646 V 17 w(GET,)f(181)75 1703 y(MPI)p 167 1703 V 16 w(CAR)l(T)p 312 1703 V 17 w(MAP)l(,)h(186)75 1759 y(MPI)p 167 1759 V 16 w(CAR)l(T)p 312 1759 V 17 w(RANK,)h(182)75 1815 y(MPI)p 167 1815 V 16 w(CAR)l(T)p 312 1815 V 17 w(SHIFT,)f(184)75 1872 y(MPI)p 167 1872 V 16 w(CAR)l(T)p 312 1872 V 17 w(SUB,)g(185)75 1928 y(MPI)p 167 1928 V 16 w(CAR)l(TDIM)p 405 1928 V 17 w(GET,)f(181)75 1985 y(MPI)p 167 1985 V 16 w(COMM)p 335 1985 V 16 w(COMP)l(ARE,)i(142)75 2041 y(MPI)p 167 2041 V 16 w(COMM)p 335 2041 V 16 w(CREA)l(TE,)f(143)75 2098 y(MPI)p 167 2098 V 16 w(COMM)p 335 2098 V 16 w(DUP)l(,)g(143)75 2154 y(MPI)p 167 2154 V 16 w(COMM)p 335 2154 V 16 w(FREE,)g(145)75 2211 y(MPI)p 167 2211 V 16 w(COMM)p 335 2211 V 16 w(GR)o(OUP)l(,)g(137) 75 2267 y(MPI)p 167 2267 V 16 w(COMM)p 335 2267 V 16 w(RANK,)h(142)75 2324 y(MPI)p 167 2324 V 16 w(COMM)p 335 2324 V 16 w(REMOTE)p 556 2324 V 17 w(GR)o(OUP)l(,)f(155)75 2380 y(MPI)p 167 2380 V 16 w(COMM)p 335 2380 V 16 w(REMOTE)p 556 2380 V 17 w(SIZE,)g(155)75 2436 y(MPI)p 167 2436 V 16 w(COMM)p 335 2436 V 16 w(SIZE,)g(141)75 2493 y(MPI)p 167 2493 V 16 w(COMM)p 335 2493 V 16 w(SPLIT,)h(144)75 2549 y(MPI)p 167 2549 V 16 w(COMM)p 335 2549 V 16 w(TEST)p 473 2549 V 16 w(INTER,)g(154)75 2647 y(MPI)p 167 2647 V 16 w(DIMS)p 301 2647 V 17 w(CREA)l(TE,)f(177)1048 490 y(MPI)p 1140 490 V 16 w(ERRHANDLER)p 1482 490 V 18 w(CREA)l(TE,)g(192) 1048 547 y(MPI)p 1140 547 V 16 w(ERRHANDLER)p 1482 547 V 18 w(FREE,)g(193)1048 603 y(MPI)p 1140 603 V 16 w(ERRHANDLER)p 1482 603 V 18 w(GET,)f(193)1048 660 y(MPI)p 1140 660 V 16 w(ERRHANDLER)p 1482 660 V 18 w(SET,)h(192)1048 716 y(MPI)p 1140 716 V 16 w(ERR)o(OR)p 1320 716 V 18 w(CLASS,)g(194)1048 773 y(MPI)p 1140 773 V 16 w(ERR)o(OR)p 1320 773 V 18 w(STRING,)g(193)1048 871 y(MPI)p 1140 871 V 16 w(FINALIZE,)h(196)1048 968 y(MPI)p 1140 968 V 16 w(GA)l(THER,)f(94)1048 1025 y(MPI)p 1140 1025 V 16 w(GA)l(THER)-5 b(V,)15 b(95)1048 1081 y(MPI)p 1140 1081 V 16 w(GET)p 1256 1081 V 16 w(COUNT,)g(21)1048 1138 y(MPI)p 1140 1138 V 16 w(GET)p 1256 1138 V 16 w(ELEMENTS,)g(73) 1048 1194 y(MPI)p 1140 1194 V 16 w(GET)p 1256 1194 V 16 w(PR)o(OCESSOR)p 1552 1194 V 18 w(NAME,)f(190)1048 1251 y(MPI)p 1140 1251 V 16 w(GRAPH)p 1324 1251 V 17 w(CREA)l(TE,)h(178)1048 1307 y(MPI)p 1140 1307 V 16 w(GRAPH)p 1324 1307 V 17 w(GET,)f(181)1048 1364 y(MPI)p 1140 1364 V 16 w(GRAPH)p 1324 1364 V 17 w(MAP)l(,)g(187)1048 1420 y(MPI)p 1140 1420 V 16 w(GRAPH)p 1324 1420 V 17 w(NEIGHBORS,)i(183)1048 1477 y(MPI)p 1140 1477 V 16 w(GRAPH)p 1324 1477 V 17 w(NEIGHBORS)p 1617 1477 V 17 w(COUNT,)10 b(182)1048 1533 y(MPI)p 1140 1533 V 16 w(GRAPHDIMS)p 1442 1533 V 17 w(GET,)k(180)1048 1590 y(MPI)p 1140 1590 V 16 w(GR)o(OUP)p 1324 1590 V 17 w(COMP)l(ARE,)h(136)1048 1646 y(MPI)p 1140 1646 V 16 w(GR)o(OUP)p 1324 1646 V 17 w(DIFFERENCE,)f(138)1048 1702 y(MPI)p 1140 1702 V 16 w(GR)o(OUP)p 1324 1702 V 17 w(EX)o(CL,)g(139)1048 1759 y(MPI)p 1140 1759 V 16 w(GR)o(OUP)p 1324 1759 V 17 w(FREE,)g(140)1048 1815 y(MPI)p 1140 1815 V 16 w(GR)o(OUP)p 1324 1815 V 17 w(INCL,)h(138)1048 1872 y(MPI)p 1140 1872 V 16 w(GR)o(OUP)p 1324 1872 V 17 w(INTERSECTION,)h(137)1048 1928 y(MPI)p 1140 1928 V 16 w(GR)o(OUP)p 1324 1928 V 17 w(RANGE)p 1509 1928 V 16 w(EX)o(CL,)f(140)1048 1985 y(MPI)p 1140 1985 V 16 w(GR)o(OUP)p 1324 1985 V 17 w(RANGE)p 1509 1985 V 16 w(INCL,)h(139)1048 2041 y(MPI)p 1140 2041 V 16 w(GR)o(OUP)p 1324 2041 V 17 w(RANK,)f(135)1048 2098 y(MPI)p 1140 2098 V 16 w(GR)o(OUP)p 1324 2098 V 17 w(SIZE,)g(135)1048 2154 y(MPI)p 1140 2154 V 16 w(GR)o(OUP)p 1324 2154 V 17 w(TRANSLA)l(TE)p 1622 2154 V 17 w(RANKS,)c(136)1048 2211 y(MPI)p 1140 2211 V 16 w(GR)o(OUP)p 1324 2211 V 17 w(UNION,)k(137)1048 2308 y(MPI)p 1140 2308 V 16 w(IBSEND,)g(37)1048 2365 y(MPI)p 1140 2365 V 16 w(INIT,)h(196)1048 2421 y(MPI)p 1140 2421 V 16 w(INITIALIZED,)g(196)1048 2478 y(MPI)p 1140 2478 V 16 w(INTER)o(COMM)p 1454 2478 V 16 w(CREA)l(TE,)f(157)1048 2534 y(MPI)p 1140 2534 V 16 w(INTER)o(COMM)p 1454 2534 V 16 w(MER)o(GE,)g(157)1048 2591 y(MPI)p 1140 2591 V 16 w(IPR)o(OBE,)h(48)1048 2647 y(MPI)p 1140 2647 V 16 w(IRECV,)g(38)1048 2704 y(MPI)p 1140 2704 V 16 w(IRSEND,)g(38)941 2828 y(227)p eop %%Page: 228 236 228 235 bop 75 -100 a Fx(228)1326 b Fm(MPI)15 b(F)l(unction)h(Index)75 45 y Fx(MPI)p 167 45 14 2 v 16 w(ISEND,)g(36)75 102 y(MPI)p 167 102 V 16 w(ISSEND,)g(37)75 196 y(MPI)p 167 196 V 16 w(KEYV)-5 b(AL)p 374 196 V 18 w(CREA)l(TE,)15 b(166)75 253 y(MPI)p 167 253 V 16 w(KEYV)-5 b(AL)p 374 253 V 18 w(FREE,)14 b(168)75 348 y(MPI)p 167 348 V 16 w(OP)p 249 348 V 17 w(CREA)l(TE,)h(118)75 404 y(MPI)p 167 404 V 16 w(OP)p 249 404 V 17 w(FREE,)g(120)75 499 y(MPI)p 167 499 V 16 w(P)l(A)o(CK,)g(83)75 556 y(MPI)p 167 556 V 16 w(P)l(A)o(CK)p 311 556 V 17 w(SIZE,)g(85)75 612 y(MPI)p 167 612 V 16 w(PCONTR)o(OL,)h(199)75 669 y(MPI)p 167 669 V 16 w(PR)o(OBE,)g(49)75 764 y(MPI)p 167 764 V 16 w(RECV,)f(19)75 820 y(MPI)p 167 820 V 16 w(RECV)p 314 820 V 17 w(INIT,)h(55)75 877 y(MPI)p 167 877 V 16 w(REDUCE,)f(111)75 933 y(MPI)p 167 933 V 16 w(REDUCE)p 380 933 V 17 w(SCA)l(TTER,)g(123)75 989 y(MPI)p 167 989 V 16 w(REQUEST)p 405 989 V 18 w(FREE,)f(41)75 1046 y(MPI)p 167 1046 V 16 w(RSEND,)i(28)75 1102 y(MPI)p 167 1102 V 16 w(RSEND)p 341 1102 V 17 w(INIT,)g(54)75 1197 y(MPI)p 167 1197 V 16 w(SCAN,)f(124)75 1254 y(MPI)p 167 1254 V 16 w(SCA)l(TTER,)h(103)75 1310 y(MPI)p 167 1310 V 16 w(SCA)l(TTER)-5 b(V,)16 b(104)75 1367 y(MPI)p 167 1367 V 16 w(SEND,)f(16)75 1423 y(MPI)p 167 1423 V 16 w(SEND)p 308 1423 V 17 w(INIT,)g(53)75 1480 y(MPI)p 167 1480 V 16 w(SENDRECV,)g(57)75 1536 y(MPI)p 167 1536 V 16 w(SENDRECV)p 439 1536 V 17 w(REPLA)o(CE,)g(58)75 1592 y(MPI)p 167 1592 V 16 w(SSEND,)g(27)75 1649 y(MPI)p 167 1649 V 16 w(SSEND)p 333 1649 V 17 w(INIT,)h(54)75 1705 y(MPI)p 167 1705 V 16 w(ST)l(AR)l(T,)g(55)75 1762 y(MPI)p 167 1762 V 16 w(ST)l(AR)l(T)l(ALL,)h(56)75 1857 y(MPI)p 167 1857 V 16 w(TEST,)e(40)75 1913 y(MPI)p 167 1913 V 16 w(TEST)p 305 1913 V 17 w(CANCELLED,)g(52)75 1970 y(MPI)p 167 1970 V 16 w(TEST)l(ALL,)h(45)75 2026 y(MPI)p 167 2026 V 16 w(TEST)l(ANY,)f(44)75 2083 y(MPI)p 167 2083 V 16 w(TESTSOME,)g(46)75 2139 y(MPI)p 167 2139 V 16 w(TOPO)p 317 2139 V 17 w(TEST,)g(180)75 2195 y(MPI)p 167 2195 V 16 w(TYPE)p 312 2195 V 17 w(COMMIT,)f(70)75 2252 y(MPI)p 167 2252 V 16 w(TYPE)p 312 2252 V 17 w(CONTIGUOUS,)h(60)75 2308 y(MPI)p 167 2308 V 16 w(TYPE)p 312 2308 V 17 w(COUNT,)g(69)75 2365 y(MPI)p 167 2365 V 16 w(TYPE)p 312 2365 V 17 w(EXTENT,)f(68)75 2421 y(MPI)p 167 2421 V 16 w(TYPE)p 312 2421 V 17 w(FREE,)g(71)75 2478 y(MPI)p 167 2478 V 16 w(TYPE)p 312 2478 V 17 w(HINDEXED,)g(65)75 2534 y(MPI)p 167 2534 V 16 w(TYPE)p 312 2534 V 17 w(HVECTOR,)h(63)75 2591 y(MPI)p 167 2591 V 16 w(TYPE)p 312 2591 V 17 w(INDEXED,)f(64)75 2647 y(MPI)p 167 2647 V 16 w(TYPE)p 312 2647 V 17 w(LB,)h(70)75 2704 y(MPI)p 167 2704 V 16 w(TYPE)p 312 2704 V 17 w(SIZE,)g(68)1048 45 y(MPI)p 1140 45 V 16 w(TYPE)p 1285 45 V 16 w(STR)o(UCT,)g(66)1048 102 y(MPI)p 1140 102 V 16 w(TYPE)p 1285 102 V 16 w(UB,)g(70)1048 158 y(MPI)p 1140 158 V 16 w(TYPE)p 1285 158 V 16 w(VECTOR,)g(61)1048 256 y(MPI)p 1140 256 V 16 w(UNP)l(A)o(CK,)g(84)1048 354 y(MPI)p 1140 354 V 16 w(W)-5 b(AIT,)15 b(39)1048 410 y(MPI)p 1140 410 V 16 w(W)-5 b(AIT)l(ALL,)16 b(44)1048 467 y(MPI)p 1140 467 V 16 w(W)-5 b(AIT)l(ANY,)15 b(43)1048 523 y(MPI)p 1140 523 V 16 w(W)-5 b(AITSOME,)15 b(46)1048 580 y(MPI)p 1140 580 V 16 w(WTICK,)g(195)1048 636 y(MPI)p 1140 636 V 16 w(WTIME,)f(195)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .