%!PS-Adobe-2.0 %%Creator: dvips 5.515 Copyright 1986, 1993 Radical Eye Software %%Title: draft.dvi %%CreationDate: Fri May 28 11:39:18 1993 %%Pages: 100 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips draft.dvi -o draft.ps %DVIPSSource: TeX output 1993.05.28:1137 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /@rigin{isls{[ 0 -1 1 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{Resolution hsize -72 div mul 0 TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{ /nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{ /sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0] N df-tail}B /E{pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get}B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{ 128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]} if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{ cc 1 add D}B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet TeXDict begin 40258437 52099154 1000 300 300 (/amd/iliamna/projects/dsrg/C/staff/otto/papers/Editor/MPI/draft3/draft.dvi) @start /Fa 3 52 df<03000700FF000700070007000700070007000700070007000700 07000700070007000700070007007FF00C157E9412>49 D<0F8030E040708030C038E038 4038003800700070006000C00180030006000C08080810183FF07FF0FFF00D157E9412> I<0FE030306018701C701C001C00180038006007E000300018000C000E000EE00EE00EC0 0C401830300FE00F157F9412>I E /Fb 1 106 df<01C003C003C0018000000000000000 00000000001C00270047004700870087000E000E001C001C001C00380038803880708071 0032001C000A1C7E9B0E>105 D E /Fc 7 118 df<01F007080C08181C38383000700070 00E000E000E000E000E000E008E010602030C01F000E127B9113>99 D<0003C0000670000C70001C60001C00001C0000380000380000380000380000380003FF 8000700000700000700000700000700000E00000E00000E00000E00000E00001C00001C0 0001C00001C00001C000038000038000038000030000030000070000C60000E60000CC00 007800001425819C0D>102 D<01800380010000000000000000000000000000001C0026 00470047008E008E000E001C001C001C0038003800710071007100720072003C00091C7C 9B0D>105 D<3C3C002646004687004707008E07008E07000E07000E07001C0E001C0E00 1C0E001C1C00381C40381C40383840383880701900300E0012127C9117>110 D<01E007180C0C180C380C300E700E700EE01CE01CE01CE018E038E030E06060C031801E 000F127B9115>I<00C001C001C001C00380038003800380FFE00700070007000E000E00 0E000E001C001C001C001C00384038403840388019000E000B1A7D990E>116 D<1E0300270700470700470700870E00870E000E0E000E0E001C1C001C1C001C1C001C1C 003838803838801838801839001C5900078E0011127C9116>I E /Fd 9 56 df34 DI<00001C00003C0000F80001E00003C0000780000F00000E 00001E00003C00003C00003C000078000078000078000078000078000078000078000078 000078000078000078000078000078000078000078000078000078000078000078000078 000078000078000078000078000078000078000078000078000078000078000078000078 0000780000780000780000780000780000780000F00000F00000F00001E00001E00003C0 000380000700000E00001C0000780000E00000E000007800001C00000E00000700000380 0003C00001E00001E00000F00000F00000F0000078000078000078000078000078000078 000078000078000078000078000078000078000078000078000078000078000078000078 000078000078000078000078000078000078000078000078000078000078000078000078 00007800007800007800007800007800007800007800007800003C00003C00003C00001E 00000E00000F000007800003C00001E00000F800003C00001C167C7B8121>40 D50 DII<00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001CFFFCFF FCFFFC0E4A80811C>III E /Fe 1 4 df<020002000200C218F2783AE00F800F803AE0F278C2180200 020002000D0E7E8E12>3 D E /Ff 2 36 df<0000000400000000020000000002000000 00010000000000800000000040FFFFFFFFF8FFFFFFFFF800000000400000000080000000 0100000000020000000002000000000400250E7E902A>33 D<0300030003000300030003 000300030003000300030003000300030003000300030003000300030003000300030003 00030003000300030003008304631813200B4007800300030001000E257D9C15>35 D E /Fg 52 122 df<00E001E0038007000E001C001C0038003800700070007000E000E0 00E000E000E000E000E000E000E000700070007000380038001C001C000E000700038001 E000E00B217A9C16>40 DI<387C7E7E3E0E1E1C78F060070B798416>44 D<70F8F8F8700505788416>46 D<03E0000FF8001FFC001E3C00380E00780F0070070070 0700E00380E00380E00380E00380E00380E00380E00380E00380F0078070070070070078 0F003C1E001E3C001FFC000FF80003E00011197E9816>48 D<01800380038007800F807F 80FF807380038003800380038003800380038003800380038003800380038003807FF87F FC7FF80E197C9816>I<7FFF00FFFF80FFFF80000000000000000000000000000000FFFF 80FFFF807FFF00110B7E9116>61 D<00E00001F00001F00001B00001B00003B80003B800 03B800031800071C00071C00071C00071C00071C000E0E000E0E000FFE000FFE001FFF00 1C07001C07001C07007F1FC0FF1FE07F1FC013197F9816>65 D<7FF800FFFE007FFF001C 0F001C07801C03801C03801C03801C07801C07001FFF001FFE001FFE001C1F001C03801C 03C01C01C01C01C01C01C01C01C01C03C01C07807FFF80FFFF007FFC0012197F9816>I< 01F18007FB800FFF801F0F803C0780380380700380700380F00000E00000E00000E00000 E00000E00000E00000E00000F000007003807003803803803C07001F0F000FFE0007FC00 01F00011197E9816>I<7FF800FFFE007FFF001C0F001C07801C03C01C01C01C01C01C01 E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C01C01C01C01C03C01C07 801C0F807FFF00FFFE007FF8001319809816>I<7FFFC0FFFFC07FFFC01C01C01C01C01C 01C01C01C01C00001C00001C1C001C1C001FFC001FFC001FFC001C1C001C1C001C00001C 00E01C00E01C00E01C00E01C00E07FFFE0FFFFE07FFFE013197F9816>II<03E30007FF000FFF001E1F003C0F00380700700700700700F00000E00000E000 00E00000E00000E03F80E07FC0E03F80F00700700700700700380F003C0F001E1F000FFF 0007F70003E70012197E9816>I<7F1FC0FFBFE07F1FC01C07001C07001C07001C07001C 07001C07001C07001FFF001FFF001FFF001C07001C07001C07001C07001C07001C07001C 07001C07001C07007F1FC0FFBFE07F1FC013197F9816>II<7F0FE0FF8FF07F0FE01C07801C0F001C0E001C1C001C3C001C7800 1CF0001CE0001DF0001FF0001FF8001F38001E1C001C1C001C0E001C0E001C07001C0700 1C03807F07E0FF8FF07F07E01419809816>75 DII<7E1FC0FF3FE07F1FC01D07001D87001D87001D87001DC7001DC7001CC7001CC7 001CE7001CE7001CE7001C67001C67001C77001C77001C37001C37001C37001C17007F1F 00FF9F007F0F0013197F9816>I<1FFC003FFE007FFF00780F00F00780E00380E00380E0 0380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380F0 0780F00780780F007FFF003FFE001FFC0011197E9816>I<7FF800FFFE007FFF001C0F80 1C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE001FF8001C0000 1C00001C00001C00001C00001C00001C00007F0000FF80007F000012197F9816>I<1FFC 003FFE007FFF00780F00F00780E00380E00380E00380E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380E0E380E1E380F0F780787F007FFF003FFE001FFC 00001C00001E00000E00000F00000700000700111F7E9816>I<7FE000FFF8007FFC001C 1E001C0F001C07001C07001C07001C07001C0F001C1E001FFC001FF8001FFC001C1C001C 0E001C0E001C0E001C0E001C0E201C0E701C0E707F07E0FF87E07F03C014197F9816>I< 07E3001FFF003FFF00781F00F00700E00700E00700E00000F000007800003F80001FF000 07FC0000FE00000F00000700000380000380600380E00380E00700F80F00FFFE00FFFC00 C7F00011197E9816>I<7FFFE0FFFFE0FFFFE0E0E0E0E0E0E0E0E0E0E0E0E000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00007FC000FFE0007FC0013197F9816>I<7F07F0FF8FF87F07F01C01C01C01C01C 01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C 01C01C01C00E03800E038007070007FF0003FE0000F8001519809816>III<7F1F807F3F807F1F800E1E000E1C00073C0007380003 B80003F00001F00001E00000E00001E00001F00003F00003B80007B800071C00071C000E 0E000E0E001C07007F1FC0FF1FE07F1FC013197F9816>II91 D93 D<1FE0003FF0007FF800783C00300E00000E00000E0003FE001FFE003E0E00700E00E00E 00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<7E0000FE00007E0000 0E00000E00000E00000E00000E3E000EFF000FFF800F83C00F00E00E00E00E00700E0070 0E00700E00700E00700E00700E00E00F01E00F83C00FFF800EFF00063C001419809816> I<03F80FFC1FFE3C1E780C7000E000E000E000E000E000F000700778073E0E1FFC0FF803 F010127D9116>I<03E00FF81FFC3C1E780E7007E007FFFFFFFFFFFFE000E00070077807 3C0F1FFE0FFC03F010127D9116>101 D<001F00007F8000FF8001E78001C30001C00001 C0007FFF00FFFF00FFFF0001C00001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C00001C0003FFE007FFF003FFE0011197F9816>I<03E3C007F7E00FFFE0 1C1CC0380E00380E00380E00380E00380E001C1C000FF8001FF0001BE000380000180000 1FFC001FFF003FFF807803C0E000E0E000E0E000E0E000E07001C07C07C03FFF800FFE00 03F800131C7F9116>I<7E0000FE00007E00000E00000E00000E00000E00000E3C000EFE 000FFF000F87800F03800E03800E03800E03800E03800E03800E03800E03800E03800E03 800E03807FC7F0FFE7F87FC7F01519809816>I<018003C003C001800000000000000000 7FC07FC07FC001C001C001C001C001C001C001C001C001C001C001C001C07FFFFFFF7FFF 101A7D9916>I108 DI<7E3C00FEFE007FFF000F87800F03800E03800E03800E03800E03800E 03800E03800E03800E03800E03800E03807FC7F0FFE7F87FC7F01512809116>I<03E000 0FF8001FFC003C1E00780F00700700E00380E00380E00380E00380E00380F00780700700 780F003C1E001FFC000FF80003E00011127E9116>I<7E3E00FEFF007FFF800F83C00F00 E00E00E00E00700E00700E00700E00700E00700E00700E00E00F01E00F83C00FFF800EFF 000E3C000E00000E00000E00000E00000E00000E00007FC000FFE0007FC000141B809116 >I114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007F8003C600EE00EF00EF81EFFFCFFF8C7 E00F127D9116>I<0300000700000700000700000700007FFF00FFFF00FFFF0007000007 000007000007000007000007000007000007010007038007038007038007870003FE0001 FC0000F80011177F9616>I<7E1F80FE3F807E1F800E03800E03800E03800E03800E0380 0E03800E03800E03800E03800E03800E03800E0F800FFFF007FBF803E3F01512809116> I119 D<7F1FC0FF9FE07F1FC01C07000E07000E0E000E0E00070E00071C00071C00039C00039C 0003980001B80001B80000F00000F00000F00000E00000E00000E00001C00079C0007BC0 007F80003F00003C0000131B7F9116>121 D E /Fh 84 126 df<70F8F8F8F8F8F8F8F8 F8F8F8F8F8F8F8F870000000000070F8F8F870051C779B18>33 D<030600078F00078F00 078F00078F00078F00078F007FFFC0FFFFE0FFFFE07FFFC00F1E000F1E000F1E000F1E00 0F1E000F1E007FFFC0FFFFE0FFFFE07FFFC01E3C001E3C001E3C001E3C001E3C001E3C00 0C1800131C7E9B18>35 D<3803007C07807C0780EE0F80EE0F00EE0F00EE1F00EE1E00EE 1E00EE3E007C3C007C3C00387C0000780000780000F80000F00001F00001E00001E00003 E00003C00003C00007C0000783800787C00F87C00F0EE00F0EE01F0EE01E0EE01E0EE03E 0EE03C07C03C07C018038013247E9F18>37 D<01C00007E0000FF0000E70001C38001C38 001C38001C38001C73F01C73F01CE3F00FE3800FC7000F87000F07001F0E003F0E007B8E 0073DC00E1DC00E0F800E0F800E07070E0787070FC707FFFE03FCFE00F03C0141C7F9B18 >I<387C7C7E3E0E0E0E1C1C38F8F0C0070E789B18>I<007000F001E003C007800F001E00 1C00380038007000700070007000E000E000E000E000E000E000E000E000700070007000 7000380038001C001E000F00078003C001F000F000700C24799F18>I<6000F00078003C 001E000F000780038001C001C000E000E000E000E0007000700070007000700070007000 7000E000E000E000E001C001C0038007800F001E003C007800F00060000C247C9F18>I< 01C00001C00001C00001C000C1C180F1C780F9CF807FFF001FFC0007F00007F0001FFC00 7FFF00F9CF80F1C780C1C18001C00001C00001C00001C00011147D9718>I<00600000F0 0000F00000F00000F00000F00000F00000F0007FFFC0FFFFE0FFFFE07FFFC000F00000F0 0000F00000F00000F00000F00000F00000600013147E9718>I<1C3E7E7F3F1F070E1E7C F860080C788518>I<7FFF00FFFF80FFFF807FFF0011047D8F18>I<3078FCFC7830060677 8518>I<000300000780000780000F80000F00001F00001E00001E00003E00003C00007C 0000780000780000F80000F00001F00001E00003E00003C00003C00007C0000780000F80 000F00000F00001F00001E00003E00003C00003C00007C0000780000F80000F00000F000 0060000011247D9F18>I<01F00007FC000FFE001F1F001C07003803807803C07001C070 01C0E000E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0F001E07001C070 01C07803C03803801C07001F1F000FFE0007FC0001F000131C7E9B18>I<018003800380 07800F803F80FF80FB804380038003800380038003800380038003800380038003800380 03800380038003807FFCFFFE7FFC0F1C7B9B18>I<03F0000FFE003FFF007C0F807003C0 E001C0F000E0F000E06000E00000E00000E00001C00001C00003C0000780000F00001E00 003C0000780000F00001E00007C0000F80001E00E03C00E07FFFE0FFFFE07FFFE0131C7E 9B18>I<07F8001FFE003FFF007807807803C07801C03001C00001C00003C0000380000F 0003FF0003FE0003FF000007800003C00001C00000E00000E00000E0F000E0F000E0F001 C0F003C07C07803FFF001FFE0003F800131C7E9B18>I<001F00003F0000770000770000 E70001E70001C7000387000787000707000E07001E07003C0700380700780700F00700FF FFF8FFFFF8FFFFF8000700000700000700000700000700000700007FF000FFF8007FF015 1C7F9B18>I<1FFF803FFF803FFF80380000380000380000380000380000380000380000 3800003BF8003FFE003FFF003C07801803C00001C00000E00000E06000E0F000E0F000E0 E001C07003C07C0F803FFF001FFC0003F000131C7E9B18>I<007E0001FF0007FF800F83 C01E03C01C03C0380180380000700000700000E1F800E7FE00FFFF00FE0780F803C0F001 C0F000E0E000E0F000E07000E07000E07000E03801C03C03C01E07800FFF0007FE0001F8 00131C7E9B18>II<03F8000FFE00 1FFF003E0F803803807001C07001C07001C07001C03803803C07801FFF0007FC000FFE00 1F1F003C07807001C0F001E0E000E0E000E0E000E0E000E07001C07803C03E0F801FFF00 0FFE0003F800131C7E9B18>I<03F0000FFC001FFE003C0F00780780700380E001C0E001 C0E001C0E001E0E001E07001E07803E03C0FE01FFFE00FFEE003F0E00000E00001C00001 C00001C0300380780780780F00783E003FFC001FF00007C000131C7E9B18>I<183C7E7E 3C180000000000000000183C7E7E3E1E0E1C3C78F060071A789318>59 D<000300000780001F80003F00007E0001FC0003F00007E0001FC0003F00007E0000FC00 00FC00007E00003F00001FC00007E00003F00001FC00007E00003F00001F800007800003 0011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE0000000000000000000000000FFFFE0FF FFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00007E00003F00001FC00007E000 03F00001FC00007E00003F00001F80001F80003F00007E0001FC0003F00007E0001FC000 3F00007E0000FC0000F0000060000011187D9918>I<00700000F80000F80000D80000D8 0001DC0001DC0001DC00018C00038E00038E00038E00038E000306000707000707000707 000707000FFF800FFF800FFF800E03800E03801C01C01C01C07F07F0FF8FF87F07F0151C 7F9B18>65 DI<00F8E003FEE007FFE00F07 E01E03E03C01E03800E07000E07000E0700000E00000E00000E00000E00000E00000E000 00E00000E000007000007000E07000E03800E03C00E01E01C00F07C007FF8003FE0000F8 00131C7E9B18>I<7FF800FFFE007FFF001C0F801C03C01C03C01C01E01C00E01C00E01C 00F01C00701C00701C00701C00701C00701C00701C00701C00701C00F01C00E01C00E01C 01E01C01C01C03C01C0F807FFF00FFFE007FF800141C7F9B18>III<01F1C003FDC00F FFC01F0FC01C03C03803C03801C07001C07001C0700000E00000E00000E00000E00000E0 0000E00FF0E01FF0E00FF07001C07001C07003C03803C03803C01C07C01F0FC00FFFC003 FDC001F1C0141C7E9B18>I<7F07F0FF8FF87F07F01C01C01C01C01C01C01C01C01C01C0 1C01C01C01C01C01C01C01C01FFFC01FFFC01FFFC01C01C01C01C01C01C01C01C01C01C0 1C01C01C01C01C01C01C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>I<7FFF00FFFF 807FFF0001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0 0001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF 00FFFF807FFF00111C7D9B18>I<7F07F0FF87F87F07F01C03C01C07801C07001C0E001C 1E001C3C001C38001C70001CF0001DF0001DF0001FB8001FB8001F1C001E1C001C0E001C 0E001C07001C07001C03801C03801C01C07F03F0FF87F87F03F0151C7F9B18>75 D<7FE000FFE0007FE0000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00700E00700E00700E00 700E00707FFFF0FFFFF07FFFF0141C7F9B18>II<7E07F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C01CC1C01CE1C01CE1C0 1CE1C01C61C01C71C01C71C01C31C01C39C01C39C01C39C01C19C01C19C01C1DC01C0DC0 1C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18>I<0FF8003FFE007FFF00780F007007 00F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380F00780700700780F007FFF003FFE000FF800111C 7D9B18>II<0FF8003FFE007FFF00780F00 700700F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380 E00380E00380E00380E00380E1E380E1E380F0E78070F700787F007FFF003FFE000FFC00 001C00001E00000E00000F0000070000070011227D9B18>I<7FF800FFFE007FFF001C0F 801C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE001FFE001C0F 001C07001C03801C03801C03801C03801C03801C039C1C039C1C039C7F01F8FF81F87F00 F0161C7F9B18>I<03F3801FFF803FFF807C0F80700780E00380E00380E00380E0000070 00007800003F00001FF00007FE0000FF00000F800003C00001C00000E00000E06000E0E0 00E0E001E0F001C0F80780FFFF80FFFE00E7F800131C7E9B18>I<7FFFF8FFFFF8FFFFF8 E07038E07038E07038E07038007000007000007000007000007000007000007000007000 00700000700000700000700000700000700000700000700000700000700007FF0007FF00 07FF00151C7F9B18>IIII<7F8FE07F9F E07F8FE00E07000F0700070E00078E00039C0003DC0001F80001F80000F00000F0000070 0000F00000F80001F80001DC00039E00038E00070F000707000E07800E03801E03C07F07 F0FF8FF87F07F0151C7F9B18>II<3FFFE0 7FFFE07FFFE07001C07003C0700780700700000F00001E00001C00003C00007800007000 00F00001E00001C00003C0000780000700000F00001E00E01C00E03C00E07800E07000E0 FFFFE0FFFFE0FFFFE0131C7E9B18>II93 D<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C3807 0E789E18>I<1FE0003FF8007FFC00781E00300E0000070000070000FF0007FF001FFF00 7F0700780700E00700E00700E00700F00F00781F003FFFF01FFBF007E1F014147D9318> I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFFC00FC1 E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F00700F80 E00FC1E00FFFC00EFF80063E00151C809B18>I<01FE0007FF001FFF803E078038030070 0000700000E00000E00000E00000E00000E00000E000007000007001C03801C03E03C01F FF8007FF0001FC0012147D9318>I<001F80003F80001F80000380000380000380000380 00038003E3800FFB801FFF803C1F80380F80700780700380E00380E00380E00380E00380 E00380E00380700780700780380F803C1F801FFFF00FFBF803E3F0151C7E9B18>I<01F0 0007FC001FFE003E0F00380780700380700380E001C0E001C0FFFFC0FFFFC0FFFFC0E000 007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80007FC000 FFE000E1E001C0C001C00001C00001C0007FFFC0FFFFC0FFFFC001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF007F FF007FFF00131C7F9B18>I<01E1F007FFF80FFFF81E1E301C0E00380700380700380700 3807003807001C0E001E1E001FFC001FF80039E0003800001C00001FFE001FFFC03FFFE0 7801F0700070E00038E00038E00038E000387800F07E03F01FFFC00FFF8001FC00151F7F 9318>I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFF C00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00 E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B18>I<03800007C00007C00007C00003 80000000000000000000000000007FC000FFC0007FC00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C000FFFF00FFFF80FF FF00111D7C9C18>I<0038007C007C007C003800000000000000000FFC1FFC0FFC001C00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C6038F078FFF07FE03F800E277E9C18>II<7FE000FFE0007FE00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E0007FFFC0FFFFE07FFFC0131C7E9B18>I<7CE0E000FFFBF8007FFFF800 1F1F1C001E1E1C001E1E1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C007F1F1F00FFBFBF807F1F1F0019148193 18>I<7E3E00FEFF807FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E0 0E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC1714809318>I<01F0 000FFE001FFF003E0F803803807001C07001C0E000E0E000E0E000E0E000E0E000E0F001 E07001C07803C03C07803E0F801FFF000FFE0001F00013147E9318>I<7E3E00FEFF807F FFC00FC1E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F 00700F80E00FC1E00FFFC00EFF800E3E000E00000E00000E00000E00000E00000E00000E 00007FC000FFE0007FC000151E809318>I<7F87E0FF9FF07FBFF803F87803F03003E000 03C00003C0000380000380000380000380000380000380000380000380000380007FFE00 FFFF007FFE0015147F9318>114 D<07F7003FFF007FFF00780F00E00700E00700E00700 7C00007FE0001FFC0003FE00001F00600780E00380E00380F00380F80F00FFFF00FFFC00 E7F00011147D9318>I<0180000380000380000380000380007FFFC0FFFFC0FFFFC00380 000380000380000380000380000380000380000380000380000380400380E00380E00380 E001C1C001FFC000FF80003E0013197F9818>I<7E07E0FE0FE07E07E00E00E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E01E00F03E007 FFFC03FFFE01FCFC1714809318>I<7F8FF0FF8FF87F8FF01E03C00E03800E03800E0380 070700070700070700038E00038E00038E00038E0001DC0001DC0001DC0000F80000F800 00700015147F9318>II<7F8FF07F9FF07F8FF0070700078E00039E0001DC0001F80000F80000700000 F00000F80001DC00039E00038E000707000F07807F8FF0FF8FF87F8FF015147F9318>I< 7F8FF0FF8FF87F8FF00E01C00E03800E0380070380070700070700038700038600038E00 01CE0001CE0000CC0000CC0000DC0000780000780000780000700000700000700000F000 00E00079E0007BC0007F80003F00001E0000151E7F9318>I<3FFFF07FFFF07FFFF07001 E07003C0700780000F00001E00003C0000F80001F00003C0000780000F00701E00703C00 70780070FFFFF0FFFFF0FFFFF014147F9318>I<0007E0001FE0007FE000780000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00001E0007FC000FF 8000FF80007FC00001E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E000007800007FE0001FE00007E013247E9F18>I<60F0F0F0F0F0F0F0F0 F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0600424769F18>I<7C00 00FF0000FFC00003C00000E00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000F000007FC0003FE0003FE0007FC000F00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00003C000FFC000FF00007C00001324 7E9F18>I E /Fi 50 122 df<00003FE00000E010000180380003807800030078000700 30000700000007000000070000000E0000000E0000000E000000FFFFE0000E00E0001C01 C0001C01C0001C01C0001C01C0001C038000380380003803800038038000380700003807 00007007000070071000700E2000700E2000700E2000E00E2000E0064000E0038000E000 0000C0000001C0000001C000003180000079800000F3000000620000003C0000001D2982 9F1A>12 D<000100020004000800100020006000C0018001800300070006000E000C001C 0018003800380030007000700060006000E000E000C000C000C000C000C000C000C000C0 00C000C000C000C000C0004000600060002000100010000800102E79A113>40 D<0010000008000004000006000002000003000003000003000001000001800001800001 800001800001800001800001800003800003800003800003000003000003000007000007 00000600000600000E00000C00000C00001C0000180000380000300000700000600000E0 0000C0000180000100000300000600000C0000180000300000600000800000112E80A113 >I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I< 70F8F8F0E005057B840E>I<000F800030E000E07001C0700380300380380700380F0078 0F00780E00781E00781E00703C00F03C00F03C00F03C00F07801E07801E07801E07801C0 7003C0F003C0F00380F00780F00700700700700E00701C003038001870000FC000151F7C 9D17>48 D<000200020006000E003C00DC031C001C003800380038003800700070007000 7000E000E000E000E001C001C001C001C003800380038003800780FFF80F1E7B9D17>I< 001F000061800080E00100E00200700220700420700410700820F00820F00820F00840E0 0881E00703C0000380000700000C00001800006000008000030000040000080040100040 1000802001807E030047FF0041FE0080FC00807800141F7C9D17>I<001F0000718000C0 C00180C00380E00700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C01C03C01C03 C01C07C01C0F800C0F8006378003C700000F00000E00000E00001C00601C00F03800F070 00E0600080C0004380003E0000131F7B9D17>57 D<070F1F1F0E00000000000000000000 70F8F8F0E008147B930E>I<00000200000006000000060000000E0000001E0000001E00 00003F0000002F0000004F0000004F0000008F0000010F0000010F0000020F0000020F00 00040F00000C0F0000080F0000100F0000100F0000200F80003FFF800040078000C00780 0080078001000780010007800200078002000780060007801E000F80FF807FF81D207E9F 22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C003C003C003C 003C003C0078007800780078007800F0007801E000F0078000FFFE0000F00F8000F003C0 01E001C001E001E001E001E001E001E003C001E003C001E003C001E003C001C0078003C0 0780078007800F0007801E000F007800FFFFE0001E1F7D9E20>I<0000FE020007818600 1C004C0038003C0060003C00C0001C01C0001803800018070000180F0000181E0000101E 0000103C0000003C00000078000000780000007800000078000000F0000000F0000000F0 000000F0000000F00000807000008070000080700001003800010038000200180004000C 001800060020000381C00000FE00001F217A9F21>I<01FFFF80001E00E0001E0070001E 0038001E001C003C001C003C000E003C000E003C000E0078000E0078000E0078000E0078 000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01E0007803C0 007003C0007003C000E003C001C0078001C00780038007800E0007801C000F007000FFFF C0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C0004003C00 04003C0004003C00040078080800780800007808000078180000F0300000FFF00000F030 0000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C0004003C000 40078000800780018007800100078007000F001F00FFFFFE001F1F7D9E1F>I<01FFFFFC 001E0038001E0018001E0008001E0008003C0008003C0008003C0008003C000800780010 00780800007808000078080000F0100000F0300000FFF00000F0300001E0200001E02000 01E0200001E0200003C0000003C0000003C0000003C00000078000000780000007800000 078000000F800000FFF800001E1F7D9E1E>I<0000FC040007030C001C00980030007800 E0007801C000380380003003800030070000300E0000301E0000201E0000203C0000003C 00000078000000780000007800000078000000F0000000F000FFF0F0000780F0000780F0 000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E000E00CC0003 83040000FC00001E217A9F23>I<01FFF3FFE0001F003E00001E003C00001E003C00001E 003C00003C007800003C007800003C007800003C007800007800F000007800F000007800 F000007800F00000F001E00000FFFFE00000F001E00000F001E00001E003C00001E003C0 0001E003C00001E003C00003C007800003C007800003C007800003C007800007800F0000 07800F000007800F000007800F00000F801F0000FFF1FFE000231F7D9E22>I<01FFF000 1F00001E00001E00001E00003C00003C00003C00003C0000780000780000780000780000 F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C00003C00007 80000780000780000780000F8000FFF800141F7D9E12>I<01FE00007FC0001E0000FC00 001E0000F80000170001780000170001780000270002F00000270004F00000270004F000 00270008F00000470009E00000470011E00000470021E00000470021E00000870043C000 00838043C00000838083C00000838083C000010381078000010382078000010382078000 0103840780000203840F00000203880F00000203900F00000203900F00000401E01E0000 0401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0002A1F7D9E29> 77 D<01FFFF80001E00E0001E0070001E0038001E003C003C003C003C003C003C003C00 3C003C0078007800780078007800F0007800E000F003C000F00F0000FFFC0000F0000001 E0000001E0000001E0000001E0000003C0000003C0000003C0000003C000000780000007 80000007800000078000000F800000FFF000001E1F7D9E1F>80 D<0007E040001C18C000 3005800060038000C0038001C00180018001000380010003800100038001000380000003 C0000003C0000003F8000001FF800001FFE000007FF000001FF0000001F8000000780000 007800000038000000380020003800200038002000300060007000600060006000E00070 00C000E8038000C606000081F800001A217D9F1A>83 D<0FFFFFF01E0780E01807802010 07802020078020200F0020600F0020400F0020400F0020801E0040001E0000001E000000 1E0000003C0000003C0000003C0000003C00000078000000780000007800000078000000 F0000000F0000000F0000000F0000001E0000001E0000001E0000001E0000003E00000FF FF00001C1F789E21>I<7FFC1FF807C003C00780010007800100078001000F0002000F00 02000F0002000F0002001E0004001E0004001E0004001E0004003C0008003C0008003C00 08003C00080078001000780010007800100078001000F0002000F0002000F0002000F000 4000F0004000700080007001000030020000380400000C18000007E000001D20779E22> II<00F1800389C00707 800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E20F01C 40F01C40703C40705C40308C800F070013147C9317>97 D<07803F800700070007000700 0E000E000E000E001C001C001CF01D0C3A0E3C0E380F380F700F700F700F700FE01EE01E E01EE01CE03CE038607060E031C01F0010207B9F15>I<007E0001C1000300800E07801E 07801C07003C0200780000780000780000F00000F00000F00000F00000F0000070010070 020030040018380007C00011147C9315>I<0000780003F8000070000070000070000070 0000E00000E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C0380 380700780700780700780700F00E00F00E00F00E00F00E20F01C40F01C40703C40705C40 308C800F070015207C9F17>I<007C01C207010E011C013C013802780C7BF07C00F000F0 00F000F0007000700170023804183807C010147C9315>I<00007800019C00033C00033C 000718000700000700000E00000E00000E00000E00000E0001FFE0001C00001C00001C00 001C00003800003800003800003800003800007000007000007000007000007000007000 00E00000E00000E00000E00000C00001C00001C0000180003180007B0000F30000660000 3C00001629829F0E>I<003C6000E27001C1E00380E00700E00F00E00E01C01E01C01E01 C01E01C03C03803C03803C03803C03803C07003C07001C0F001C17000C2E0003CE00000E 00000E00001C00001C00301C00783800F0700060E0003F8000141D7E9315>I<01E0000F E00001C00001C00001C00001C000038000038000038000038000070000070000071E0007 63000E81800F01C00E01C00E01C01C03801C03801C03801C038038070038070038070038 0E10700E20700C20701C20700C40E00CC060070014207D9F17>I<00C001E001E001C000 000000000000000000000000000E003300230043804300470087000E000E000E001C001C 001C003840388030807080310033001C000B1F7C9E0E>I<0001800003C00003C0000380 000000000000000000000000000000000000000000003C00004600008700008700010700 010700020E00000E00000E00000E00001C00001C00001C00001C00003800003800003800 00380000700000700000700000700000E00000E00030E00079C000F180006300003C0000 1228829E0E>I<01E0000FE00001C00001C00001C00001C0000380000380000380000380 000700000700000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC0 0038E000387000387000383840707080707080707080703100E03100601E0013207D9F15 >I<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C00 1C0038003800380038007000700070007100E200E200E200E200640038000A207C9F0C> I<1C0F80F0002630C318004740640C004780680E004700700E004700700E008E00E01C00 0E00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C0708038 038071003803806100380380E10038038062007007006600300300380021147C9325>I< 1C0F802630C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C0 1C01C01C03843803883803083807083803107003303001C016147C931A>I<007C0001C3 000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F007 80F00700700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C04 781C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C03803807803 80700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C 0000FFC000171D809317>I<1C1E002661004783804787804707804703008E00000E0000 0E00000E00001C00001C00001C00001C0000380000380000380000380000700000300000 11147C9313>114 D<00FC030206010C030C070C060C000F800FF007F803FC003E000E70 0EF00CF00CE008401020601F8010147D9313>I<018001C0038003800380038007000700 FFF007000E000E000E000E001C001C001C001C0038003800380038207040704070807080 31001E000C1C7C9B0F>I<0E00C03300E02301C04381C04301C04701C08703800E03800E 03800E03801C07001C07001C07001C07101C0E20180E20180E201C1E200C264007C38014 147C9318>I<0E03803307802307C04383C04301C04700C08700800E00800E00800E0080 1C01001C01001C01001C02001C02001C04001C04001C08000E300003C00012147C9315> I<0E00C1C03300E3C02301C3E04381C1E04301C0E04701C060870380400E0380400E0380 400E0380401C0700801C0700801C0700801C0701001C0701001C0602001C0F02000C0F04 000E13080003E1F0001B147C931E>I<0383800CC4401068E01071E02071E02070C040E0 0000E00000E00000E00001C00001C00001C00001C040638080F38080F38100E5810084C6 0078780013147D9315>I<0E00C03300E02301C04381C04301C04701C08703800E03800E 03800E03801C07001C07001C07001C07001C0E00180E00180E001C1E000C3C0007DC0000 1C00001C00003800F03800F07000E06000C0C0004380003E0000131D7C9316>I E /Fj 58 124 df<000FF83F00007FFDFFC001F81FE3E003E03F87E007C03F87E00F803F 07E00F803F03C00F801F00000F801F00000F801F00000F801F00000F801F00000F801F00 00FFFFFFFC00FFFFFFFC000F801F00000F801F00000F801F00000F801F00000F801F0000 0F801F00000F801F00000F801F00000F801F00000F801F00000F801F00000F801F00000F 801F00000F801F00000F801F00000F801F00000F801F00000F801F00007FF0FFF0007FF0 FFF00023237FA221>11 D<000FF000007FFC0001F80E0003E01F0007C03F000F803F000F 803F000F801E000F800C000F8000000F8000000F8000000F800000FFFFFF00FFFFFF000F 801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F000F 801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F007F F0FFE07FF0FFE01B237FA21F>I<387CFEFFFF7F3B03030706060C1C18702008117CA210> 39 D<387CFEFEFE7C3807077C8610>46 D<00FE0007FFC00F83E01F01F03E00F83E00F8 7C007C7C007C7C007CFC007CFC007EFC007EFC007EFC007EFC007EFC007EFC007EFC007E FC007EFC007EFC007EFC007EFC007E7C007C7C007C7C007C3E00F83E00F81F01F00F83E0 07FFC000FE0017207E9F1C>48 D<00180000780001F800FFF800FFF80001F80001F80001 F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001 F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F8007FFFE07F FFE013207C9F1C>I<03FC000FFF003C1FC07007E07C07F0FE03F0FE03F8FE03F8FE01F8 7C01F83803F80003F80003F00003F00007E00007C0000F80001F00003E00003800007000 00E01801C0180380180700180E00380FFFF01FFFF03FFFF07FFFF0FFFFF0FFFFF015207D 9F1C>I<00FE0007FFC00F07E01E03F03F03F03F81F83F81F83F81F81F03F81F03F00003 F00003E00007C0001F8001FE0001FF000007C00001F00001F80000FC0000FC3C00FE7E00 FEFF00FEFF00FEFF00FEFF00FC7E01FC7801F81E07F00FFFC001FE0017207E9F1C>I<00 00E00001E00003E00003E00007E0000FE0001FE0001FE00037E00077E000E7E001C7E001 87E00307E00707E00E07E00C07E01807E03807E07007E0E007E0FFFFFEFFFFFE0007E000 07E00007E00007E00007E00007E00007E000FFFE00FFFE17207E9F1C>I<1000201E01E0 1FFFC01FFF801FFF001FFE001FF8001BC00018000018000018000018000019FC001FFF00 1E0FC01807E01803E00003F00003F00003F80003F83803F87C03F8FE03F8FE03F8FC03F0 FC03F07007E03007C01C1F800FFF0003F80015207D9F1C>I<001F8000FFE003F07007C0 F00F01F81F01F83E01F83E01F87E00F07C00007C0000FC0800FC7FC0FCFFE0FD80F0FF00 F8FE007CFE007CFC007EFC007EFC007EFC007E7C007E7C007E7C007E3C007C3E007C1E00 F80F00F00783E003FFC000FF0017207E9F1C>I<6000007800007FFFFE7FFFFE7FFFFC7F FFF87FFFF87FFFF0E00060E000C0C00180C00300C00300000600000C00001C0000180000 380000780000780000F00000F00000F00001F00001F00001F00003F00003F00003F00003 F00003F00003F00003F00001E00017227DA11C>I<00FE0003FFC00703E00E00F01C00F0 1C00783C00783E00783F00783F80783FE0F01FF9E01FFFC00FFF8007FFC003FFE007FFF0 1E7FF83C1FFC7807FC7801FEF000FEF0003EF0001EF0001EF0001CF8001C7800383C0038 1F01F00FFFC001FF0017207E9F1C>I<01FE0007FF800F83E01E01F03E00F07C00F87C00 78FC007CFC007CFC007CFC007EFC007EFC007EFC007E7C00FE7C00FE3E01FE1E037E0FFE 7E07FC7E00207E00007C00007C1E007C3F00F83F00F83F00F03F01E01E03C01C0F800FFE 0003F80017207E9F1C>I<387CFEFEFE7C380000000000000000387CFEFEFE7C3807167C 9510>I<000070000000007000000000F800000000F800000000F800000001FC00000001 FC00000003FE00000003FE00000003FE00000006FF000000067F0000000E7F8000000C3F 8000000C3F800000183FC00000181FC00000381FE00000300FE00000300FE00000600FF0 00006007F00000E007F80000FFFFF80000FFFFF800018001FC00018001FC00038001FE00 030000FE00030000FE000600007F000600007F00FFE00FFFF8FFE00FFFF825227EA12A> 65 DI< 0003FE0080001FFF818000FF01E38001F8003F8003E0001F8007C0000F800F800007801F 800007803F000003803F000003807F000001807E000001807E00000180FE00000000FE00 000000FE00000000FE00000000FE00000000FE00000000FE00000000FE000000007E0000 00007E000001807F000001803F000001803F000003801F800003000F8000030007C00006 0003F0000C0001F800380000FF00F000001FFFC0000003FE000021227DA128>IIII<0003FE0040001FFF C0C0007F00F1C001F8003FC003F0000FC007C00007C00FC00003C01F800003C03F000001 C03F000001C07F000000C07E000000C07E000000C0FE00000000FE00000000FE00000000 FE00000000FE00000000FE00000000FE00000000FE000FFFFC7E000FFFFC7F00001FC07F 00001FC03F00001FC03F00001FC01F80001FC00FC0001FC007E0001FC003F0001FC001FC 003FC0007F80E7C0001FFFC3C00003FF00C026227DA12C>III76 DII<0007FC0000003FFF800000FC07E00003F001F80007 E000FC000FC0007E001F80003F001F80003F003F00001F803F00001F807F00001FC07E00 000FC07E00000FC0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE0000 0FE0FE00000FE0FE00000FE0FE00000FE07E00000FC07F00001FC07F00001FC03F00001F 803F80003F801F80003F000FC0007E0007E000FC0003F001F80000FC07E000003FFF8000 0007FC000023227DA12A>II82 D<01FC0407FF8C1F03FC3C007C7C003C78001C78001CF8000CF8000C FC000CFC0000FF0000FFE0007FFF007FFFC03FFFF01FFFF80FFFFC03FFFE003FFE0003FF 00007F00003F00003FC0001FC0001FC0001FE0001EE0001EF0003CFC003CFF00F8C7FFE0 80FF8018227DA11F>I<7FFFFFFF807FFFFFFF807E03F80F807803F807807003F8038060 03F80180E003F801C0E003F801C0C003F800C0C003F800C0C003F800C0C003F800C00003 F800000003F800000003F800000003F800000003F800000003F800000003F800000003F8 00000003F800000003F800000003F800000003F800000003F800000003F800000003F800 000003F800000003F800000003F800000003F800000003F8000003FFFFF80003FFFFF800 22227EA127>I87 D<07FC001FFF803F07C03F03E03F01E03F01F01E01F00001F00001F0003FF003FDF01FC1 F03F01F07E01F0FC01F0FC01F0FC01F0FC01F07E02F07E0CF81FF87F07E03F18167E951B >97 DI<00FF8007FFE00F83F01F03F03E03F07E03F07C01E07C0000FC0000FC0000 FC0000FC0000FC0000FC00007C00007E00007E00003E00301F00600FC0E007FF8000FE00 14167E9519>I<0001FE000001FE0000003E0000003E0000003E0000003E0000003E0000 003E0000003E0000003E0000003E0000003E0000003E0001FC3E0007FFBE000F81FE001F 007E003E003E007E003E007C003E00FC003E00FC003E00FC003E00FC003E00FC003E00FC 003E00FC003E00FC003E007C003E007C003E003E007E001E00FE000F83BE0007FF3FC001 FC3FC01A237EA21F>I<00FE0007FF800F87C01E01E03E01F07C00F07C00F8FC00F8FC00 F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C00007E00003E00181F00300FC07003FF C000FF0015167E951A>I<003F8000FFC001E3E003C7E007C7E00F87E00F83C00F80000F 80000F80000F80000F80000F8000FFFC00FFFC000F80000F80000F80000F80000F80000F 80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F 80007FF8007FF80013237FA211>I<03FC1E0FFF7F1F0F8F3E07CF3C03C07C03E07C03E0 7C03E07C03E07C03E03C03C03E07C01F0F801FFF0013FC003000003000003800003FFF80 1FFFF00FFFF81FFFFC3800FC70003EF0001EF0001EF0001EF0001E78003C7C007C3F01F8 0FFFE001FF0018217E951C>II<1C003F007F007F007F003F001C00000000000000 0000000000000000FF00FF001F001F001F001F001F001F001F001F001F001F001F001F00 1F001F001F001F001F001F00FFE0FFE00B247EA310>I<0038007C00FE00FE00FE007C00 38000000000000000000000000000003FE03FE003E003E003E003E003E003E003E003E00 3E003E003E003E003E003E003E003E003E003E003E003E003E003E003E783EFC3EFC3CFC 7C78F87FE01F800F2E83A311>I108 DII<00FE0007FFC00F83E01E00F03E00F87C007C7C007C7C007CFC007EFC007EFC007E FC007EFC007EFC007EFC007E7C007C7C007C3E00F81F01F00F83E007FFC000FE0017167E 951C>II<00FE030007 FF87000FC1C7001F006F003F003F007E003F007E001F007C001F00FC001F00FC001F00FC 001F00FC001F00FC001F00FC001F00FC001F007E001F007E001F003E003F001F007F000F C1DF0007FF9F0001FC1F0000001F0000001F0000001F0000001F0000001F0000001F0000 001F0000001F000000FFE00000FFE01B207E951E>II<0FF3003FFF00781F00600700E0 0300E00300F00300FC00007FE0007FF8003FFE000FFF0001FF00000F80C00780C00380E0 0380E00380F00700FC0E00EFFC00C7F00011167E9516>I<018000018000018000018000 0380000380000780000780000F80003F8000FFFF00FFFF000F80000F80000F80000F8000 0F80000F80000F80000F80000F80000F80000F80000F81800F81800F81800F81800F8180 0F830007C30003FE0000F80011207F9F16>IIIIII123 D E /Fk 73 124 df<007E1F0001C1B1800303E3C00703C3C00E03C1800E01C0000E01C0 000E01C0000E01C0000E01C0000E01C000FFFFFC000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0007F87FC001A1D809C18>11 D<007E0001C1800301800703 C00E03C00E01800E00000E00000E00000E00000E0000FFFFC00E01C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C07F87F8151D809C17>I<007FC001C1C00303C00703C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C0FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C07FCFF8151D809C17>I<003F07 E00001C09C18000380F018000701F03C000E01E03C000E00E018000E00E000000E00E000 000E00E000000E00E000000E00E00000FFFFFFFC000E00E01C000E00E01C000E00E01C00 0E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E 00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C007FC7FCFF80211D 809C23>I<6060F0F0F8F86868080808080808101010102020404080800D0C7F9C15>34 D<60F0F8680808081010204080050C7C9C0C>39 D<004000800100020006000C000C0018 001800300030007000600060006000E000E000E000E000E000E000E000E000E000E000E0 00E000600060006000700030003000180018000C000C00060002000100008000400A2A7D 9E10>I<800040002000100018000C000C000600060003000300038001800180018001C0 01C001C001C001C001C001C001C001C001C001C001C00180018001800380030003000600 06000C000C00180010002000400080000A2A7E9E10>I<60F0F070101010102020408004 0C7C830C>44 DI<60F0F06004047C830C>I<0001000300060006 0006000C000C000C0018001800180030003000300060006000C000C000C0018001800180 030003000300060006000C000C000C00180018001800300030003000600060006000C000 C00010297E9E15>I<03C00C301818300C300C700E60066006E007E007E007E007E007E0 07E007E007E007E007E007E007E00760066006700E300C300C18180C3007E0101D7E9B15 >I<030007003F00C7000700070007000700070007000700070007000700070007000700 0700070007000700070007000700070007000F80FFF80D1C7C9B15>I<07C01830201C40 0C400EF00FF80FF807F8077007000F000E000E001C001C00380070006000C00180030006 010C01180110023FFE7FFEFFFE101C7E9B15>I<07E01830201C201C781E780E781E381E 001C001C00180030006007E00030001C001C000E000F000F700FF80FF80FF80FF00E401C 201C183007E0101D7E9B15>I<00F0030C06040C0E181E301E300C700070006000E3E0E4 30E818F00CF00EE006E007E007E007E007E007600760077006300E300C18180C3003E010 1D7E9B15>54 D<4000007FFF807FFF007FFF004002008004008004008008000010000010 0000200000600000400000C00000C00001C0000180000180000380000380000380000380 00078000078000078000078000078000078000030000111D7E9B15>I<03C00C30181830 0C700C600EE006E006E007E007E007E007E0076007700F300F18170C2707C70006000600 0E300C780C78187010203030C00F80101D7E9B15>57 D<60F0F060000000000000000000 0060F0F06004127C910C>I<60F0F0600000000000000000000060F0F070101010102020 4080041A7C910C>I<0FE03038401CE00EF00EF00EF00E000C001C0030006000C0008001 800100010001000100010001000000000000000000000003000780078003000F1D7E9C14 >63 D<000600000006000000060000000F0000000F0000000F0000001780000017800000 1780000023C0000023C0000023C0000041E0000041E0000041E0000080F0000080F00001 80F8000100780001FFF80003007C0002003C0002003C0006003E0004001E0004001E000C 001F001E001F00FF80FFF01C1D7F9C1F>65 DI<001F808000E0618001801980070007800E0003801C0003801C000180380001807800 00807800008070000080F0000000F0000000F0000000F0000000F0000000F0000000F000 0000F0000000700000807800008078000080380000801C0001001C0001000E0002000700 04000180080000E03000001FC000191E7E9C1E>IIII<001F 808000E0618001801980070007800E0003801C0003801C00018038000180780000807800 008070000080F0000000F0000000F0000000F0000000F0000000F0000000F000FFF0F000 0F80700007807800078078000780380007801C0007801C0007800E00078007000B800180 118000E06080001F80001C1E7E9C21>III<1FFF00F800780078007800780078007800 780078007800780078007800780078007800780078007800787078F878F878F878F0F040 E021C01F00101D7F9B15>IIIII<003F800000E0E0000380380007001C00 0E000E001C0007003C00078038000380780003C0780003C0700001C0F00001E0F00001E0 F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0700001C0780003C0780003C0 380003803C0007801C0007000E000E0007001C000380380000E0E000003F80001B1E7E9C 20>II<003F800000E0E000038038000700 1C000E000E001C0007003C00078038000380780003C0780003C0700001C0F00001E0F000 01E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0700001C0780003C07800 03C0380003803C0E07801C1107000E208E0007205C0003A0780000F0E020003FE0200000 602000003060000038E000003FC000003FC000001F8000000F001B257E9C20>II<07E0801C1980300580700380600180E00180E00080E00080 E00080F00000F800007C00007FC0003FF8001FFE0007FF0000FF80000F800007C00003C0 0001C08001C08001C08001C0C00180C00180E00300D00200CC0C0083F800121E7E9C17> I<7FFFFFC0700F01C0600F00C0400F0040400F0040C00F0020800F0020800F0020800F00 20000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000001F80 0003FFFC001B1C7F9B1E>II87 D89 D<08081010202040404040808080808080B0B0F8F8787830 300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC000F1C00 381C00701C00601C00E01C40E01C40E01C40603C40304E801F870012127E9115>97 DI<07E00C301878307870306000E0 00E000E000E000E000E00060007004300418080C3007C00E127E9112>I<003F00000700 00070000070000070000070000070000070000070000070000070003E7000C1700180F00 300700700700600700E00700E00700E00700E00700E00700E00700600700700700300700 180F000C370007C7E0131D7E9C17>I<03E00C301818300C700E6006E006FFFEE000E000 E000E00060007002300218040C1803E00F127F9112>I<00F8018C071E061E0E0C0E000E 000E000E000E000E00FFE00E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E007FE00F1D809C0D>I<00038003C4C00C38C01C3880181800381C00381C 00381C00381C001818001C38000C300013C0001000003000001800001FF8001FFF001FFF 803003806001C0C000C0C000C0C000C06001803003001C0E0007F800121C7F9215>II<18003C003C00180000000000000000 00000000000000FC001C001C001C001C001C001C001C001C001C001C001C001C001C001C 001C001C00FF80091D7F9C0C>I<00C001E001E000C00000000000000000000000000000 0FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E060E0F0C0F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180E001 C0E001C0E001C0E001C0E001C0E001C06001807003803003001806000E1C0003F0001212 7F9115>II<03C1000C3300180B00300F00700700700700 E00700E00700E00700E00700E00700E00700600700700700300F00180F000C370007C700 000700000700000700000700000700000700000700003FE0131A7E9116>II<1F9030704030C010C010E010F8007F803FE00FF000F880388018C018C018E010D060 8FC00D127F9110>I<04000400040004000C000C001C003C00FFE01C001C001C001C001C 001C001C001C001C001C101C101C101C101C100C100E2003C00C1A7F9910>IIII<7F8FF00F03800F030007020003840001C80001 D80000F00000700000780000F800009C00010E00020E000607000403801E07C0FF0FF815 12809116>II<7FFC70386038407040F040E041C003C003 8007000F040E041C043C0C380870087038FFF80E127F9112>II E /Fl 32 122 df<0020004001800380030006000E001C001C003C003800380078007800 7800F800F000F000F000F000F000F000F000F000F000F800780078007800380038003C00 1C001C000E000600030003800180004000200B297C9E13>40 D<80004000300038001800 0C000E000700070007800380038003C003C003C003E001E001E001E001E001E001E001E0 01E001E003E003C003C003C0038003800780070007000E000C0018003800300040008000 0B297D9E13>I<78FCFCFEFE7A0202040408083040070E7D850D>44 D<78FCFCFCFC7800000000000078FCFCFCFC7806127D910D>58 D<001FE02000FFF8E003 F80FE007C003E00F8001E01F0000E03E0000E03E0000607E0000607C000060FC000000FC 000000FC000000FC000000FC000000FC000000FC000000FC0000007C0000607E0000603E 0000603E0000C01F0000C00F80018007C0030003F80E0000FFFC00001FE0001B1C7D9B22 >67 DII<000FF008007FFE3801FC07 F807E001F80F8000781F0000783F0000383E0000387E0000187C000018FC000000FC0000 00FC000000FC000000FC000000FC000000FC007FFFFC007FFF7C0001F87E0001F83E0001 F83F0001F81F0001F80F8001F807E001F801FC07F8007FFE78000FF818201C7D9B26>71 D73 D77 D<003FE00001F07C0003C01E000F800F801F0007C01E0003C03E0003E07E0003 F07C0001F07C0001F0FC0001F8FC0001F8FC0001F8FC0001F8FC0001F8FC0001F8FC0001 F8FC0001F87C0001F07E0003F07E0003F03E0003E03F0007E01F0007C00F800F8003C01E 0001F07C00003FE0001D1C7D9B24>79 DI82 D<07F8201FFEE03C07E07801E07000E0F000E0F00060F00060F80000FE0000FFE0007FFE 003FFF003FFF800FFFC007FFE0007FE00003F00001F00000F0C000F0C000F0C000E0E000 E0F001C0FC03C0EFFF0083FC00141C7D9B1B>I85 D<0FF8001C1E003E0F803E07803E07C01C07C00007C0007FC007E7C01F07C03C07C07C07 C0F807C0F807C0F807C0780BC03E13F80FE1F815127F9117>97 DI<03FC000E0E001C1F003C1F00781F00780E00F80000F800 00F80000F80000F80000F800007800007801803C01801C03000E0E0003F80011127E9115 >I<01FC000F07001C03803C01C07801C07801E0F801E0F801E0FFFFE0F80000F80000F8 00007800007C00603C00601E00C00F038001FC0013127F9116>101 D<007F0001E38003C7C00787C00F87C00F83800F80000F80000F80000F80000F8000FFF8 00FFF8000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80 000F80000F80000F80007FF8007FF800121D809C0F>I<03F8F00E0F381E0F381C07303C 07803C07803C07803C07801C07001E0F000E0E001BF8001000001800001800001FFF001F FFC00FFFE01FFFF07801F8F00078F00078F000787000707800F01E03C007FF00151B7F91 18>I<1E003F003F003F003F001E00000000000000000000000000FF00FF001F001F001F 001F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B1E7F9D0E>105 D108 DII<01FC000F07801C01C03C01E07800F07800F0F800F8 F800F8F800F8F800F8F800F8F800F87800F07800F03C01E01E03C00F078001FC0015127F 9118>II114 D<1FD830786018E018E018F000FF807FE07FF01FF807FC007CC01CC01C E01CE018F830CFC00E127E9113>I<0300030003000300070007000F000F003FFCFFFC1F 001F001F001F001F001F001F001F001F001F0C1F0C1F0C1F0C0F08079803F00E1A7F9913 >II121 D E /Fm 59 123 df<0001FF81FE00001FFFEFFF80007F80FF87C000FC00FE 0FE001F801FE0FE003F801FC0FE007F001FC0FE007F001FC07C007F001FC000007F001FC 000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC0000FFFFFFFF F800FFFFFFFFF800FFFFFFFFF80007F001FC000007F001FC000007F001FC000007F001FC 000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC 000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC 000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC00007FFF1FFF E0007FFF1FFFE0007FFF1FFFE0002B2A7FA928>11 D<1C007F007F00FF80FFC0FFC07FC0 7FC01CC000C000C00180018001800300030006000C001800300020000A157BA913>39 D<1C003E007F00FF80FF80FF807F003E001C0009097B8813>46 D<003F800001FFF00007 E0FC000FC07E001F803F001F803F003F001F803F001F807F001FC07F001FC07F001FC07F 001FC0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF 001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE07F001FC07F001FC07F 001FC07F001FC03F001F803F001F801F803F001F803F000FC07E0007E0FC0001FFF00000 3F80001B277DA622>48 D<000E00001E00007E0007FE00FFFE00FFFE00F8FE0000FE0000 FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000 FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000 FE0000FE0000FE0000FE007FFFFE7FFFFE7FFFFE17277BA622>I<00FF800003FFF0000F FFFC001F03FE003800FF007C007F80FE003FC0FF003FC0FF003FE0FF001FE0FF001FE07E 001FE03C003FE000003FE000003FC000003FC000007F8000007F000000FE000000FC0000 01F8000003F0000003E00000078000000F0000001E0000003C00E0007000E000E000E001 C001C0038001C0070001C00FFFFFC01FFFFFC03FFFFFC07FFFFFC0FFFFFF80FFFFFF80FF FFFF801B277DA622>I<007F800003FFF00007FFFC000F81FE001F00FF003F80FF003F80 7F803F807F803F807F801F807F800F007F800000FF000000FF000000FE000001FC000001 F8000007F00000FFC00000FFF0000001FC0000007E0000007F0000007F8000003FC00000 3FC000003FE000003FE03C003FE07E003FE0FF003FE0FF003FE0FF003FC0FF007FC07E00 7F807C007F003F01FE001FFFFC0007FFF00000FF80001B277DA622>I<00000E0000001E 0000003E0000007E000000FE000000FE000001FE000003FE0000077E00000E7E00000E7E 00001C7E0000387E0000707E0000E07E0000E07E0001C07E0003807E0007007E000E007E 000E007E001C007E0038007E0070007E00E0007E00FFFFFFF8FFFFFFF8FFFFFFF80000FE 000000FE000000FE000000FE000000FE000000FE000000FE000000FE00007FFFF8007FFF F8007FFFF81D277EA622>I<0C0003000F803F000FFFFE000FFFFC000FFFF8000FFFF000 0FFFE0000FFFC0000FFE00000E0000000E0000000E0000000E0000000E0000000E000000 0E7FC0000FFFF8000F80FC000E003E000C003F0000001F8000001FC000001FC000001FE0 00001FE018001FE07C001FE0FE001FE0FE001FE0FE001FE0FE001FC0FC001FC078003F80 78003F803C007F001F01FE000FFFF80003FFF00000FF80001B277DA622>I<0007F00000 3FFC0000FFFE0001FC0F0003F01F8007E03F800FC03F801FC03F801F803F803F801F003F 8000007F0000007F0000007F000000FF000000FF0FC000FF3FF800FF707C00FFC03E00FF C03F00FF801F80FF801FC0FF001FC0FF001FE0FF001FE0FF001FE07F001FE07F001FE07F 001FE07F001FE03F001FE03F001FC01F801FC01F803F800FC03F0007E07E0003FFFC0000 FFF000003FC0001B277DA622>I<380000003E0000003FFFFFF03FFFFFF03FFFFFF07FFF FFE07FFFFFC07FFFFF807FFFFF0070000E0070000E0070001C00E0003800E0007000E000 E0000000E0000001C000000380000007800000078000000F0000000F0000001F0000001F 0000003F0000003E0000003E0000007E0000007E0000007E0000007E000000FE000000FE 000000FE000000FE000000FE000000FE000000FE000000FE0000007C0000003800001C29 7CA822>I<003FC00001FFF00003FFFC0007C07E000F003F001E001F001E000F803E000F 803E000F803F000F803F000F803FC00F003FF01F001FFC1E001FFE3C000FFFF80007FFE0 0003FFF80001FFFC0001FFFE0007FFFF000F0FFF801E03FFC03C01FFC07C007FE078001F E0F80007E0F80007E0F80003E0F80003E0F80003E0F80003C07C0003C07C0007803F000F 001FC03E000FFFFC0003FFF800007FC0001B277DA622>I<007F800001FFF00007FFF800 0FE0FC001F807E003F803F007F003F007F001F80FF001F80FF001FC0FF001FC0FF001FC0 FF001FE0FF001FE0FF001FE0FF001FE07F001FE07F003FE03F003FE01F807FE00F807FE0 07C1DFE003FF9FE0007E1FE000001FE000001FC000001FC000001FC000003F801F003F80 3F803F003F803F003F807E003F807C001F01F8001E03F0000FFFE00007FF800001FE0000 1B277DA622>I<01FF000FFFE01E03F03801F87801FCFC01FEFE01FEFE01FEFE01FE7C01 FE3801FC0003F80007F00007C0000F80001F00001E00003C000038000038000078000070 000070000070000070000070000070000000000000000000000000000000000000000070 0000F80001FC0003FE0003FE0003FE0001FC0000F800007000172A7CA920>63 D<000003800000000007C00000000007C0000000000FE0000000000FE0000000000FE000 0000001FF0000000001FF0000000003FF8000000003FF8000000003FF80000000073FC00 00000073FC00000000F3FE00000000E1FE00000000E1FE00000001C0FF00000001C0FF00 000003C0FF80000003807F80000007807FC0000007003FC0000007003FC000000E003FE0 00000E001FE000001E001FF000001C000FF000001FFFFFF000003FFFFFF800003FFFFFF8 0000780007FC0000700003FC0000700003FC0000E00001FE0000E00001FE0001E00001FF 0001C00000FF0001C00000FF00FFFE001FFFFEFFFE001FFFFEFFFE001FFFFE2F297EA834 >65 DI<00003FF001800003FFFE0380000FFFFF8780003FF007DF8000FF8001FF8001FE 00007F8003FC00003F8007F000001F800FF000000F801FE0000007801FE0000007803FC0 000007803FC0000003807FC0000003807F80000003807F8000000000FF8000000000FF80 00000000FF8000000000FF8000000000FF8000000000FF8000000000FF8000000000FF80 00000000FF80000000007F80000000007F80000000007FC0000003803FC0000003803FC0 000003801FE0000003801FE0000007000FF00000070007F000000E0003FC00001E0001FE 00003C0000FF8000F800003FF007E000000FFFFFC0000003FFFF000000003FF800002929 7CA832>IIII<0000 7FE003000003FFFC0700001FFFFF0F00003FF00FFF0000FF8001FF0001FE0000FF0003F8 00003F0007F000003F000FF000001F001FE000000F001FE000000F003FC000000F003FC0 000007007FC0000007007F80000007007F8000000000FF8000000000FF8000000000FF80 00000000FF8000000000FF8000000000FF8000000000FF8000000000FF8000000000FF80 01FFFFF87F8001FFFFF87F8001FFFFF87FC00000FF003FC00000FF003FC00000FF001FE0 0000FF001FE00000FF000FF00000FF0007F00000FF0003F80000FF0001FE0000FF0000FF 8001FF00003FF007BF00001FFFFF1F000003FFFE0F0000007FF003002D297CA836>I73 D76 DII<0000FFE000 000007FFFC0000003FC07F8000007F001FC00001FC0007F00003F80003F80007F00001FC 000FF00001FE001FE00000FF001FE00000FF003FC000007F803FC000007F807FC000007F C07F8000003FC07F8000003FC07F8000003FC0FF8000003FE0FF8000003FE0FF8000003F E0FF8000003FE0FF8000003FE0FF8000003FE0FF8000003FE0FF8000003FE0FF8000003F E0FF8000003FE07F8000003FC07FC000007FC07FC000007FC03FC000007F803FC000007F 801FE00000FF001FE00000FF000FF00001FE0007F00001FC0003F80003F80001FC0007F0 0000FF001FE000003FC07F8000000FFFFE00000000FFE000002B297CA834>II82 D<007F806003FFF0E007FFF9E00F807FE01F001FE03E0007E07C0003E07C0001E0FC0001 E0FC0001E0FC0000E0FE0000E0FE0000E0FF000000FFC000007FFE00007FFFE0003FFFFC 001FFFFE000FFFFF8007FFFFC003FFFFE000FFFFE00007FFF000007FF000000FF8000007 F8000003F8600001F8E00001F8E00001F8E00001F8F00001F0F00001F0F80003F0FC0003 E0FF0007C0FFE01F80F3FFFF00E0FFFE00C01FF0001D297CA826>I<7FFFFFFFFFC07FFF FFFFFFC07FFFFFFFFFC07F803FC03FC07E003FC007C078003FC003C078003FC003C07000 3FC001C0F0003FC001E0F0003FC001E0E0003FC000E0E0003FC000E0E0003FC000E0E000 3FC000E0E0003FC000E000003FC0000000003FC0000000003FC0000000003FC000000000 3FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC000000000 3FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC000000000 3FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC00000007F FFFFE000007FFFFFE000007FFFFFE0002B287EA730>IIII<01FF800007FFF0000F81F8001FC07E001FC07E001FC03F000F80 3F8007003F8000003F8000003F8000003F80000FFF8000FFFF8007FC3F800FE03F803F80 3F803F003F807F003F80FE003F80FE003F80FE003F80FE003F807E007F807F00DF803F83 9FFC0FFF0FFC01FC03FC1E1B7E9A21>97 DI<001FF80000FFFE0003F01F0007E03F800F C03F801F803F803F801F007F800E007F0000007F000000FF000000FF000000FF000000FF 000000FF000000FF000000FF0000007F0000007F0000007F8000003F8001C01F8001C00F C0038007E0070003F01E0000FFFC00001FE0001A1B7E9A1F>I<00003FF80000003FF800 00003FF800000003F800000003F800000003F800000003F800000003F800000003F80000 0003F800000003F800000003F800000003F800000003F800000003F800001FE3F80000FF FBF80003F03FF80007E00FF8000FC007F8001F8003F8003F8003F8007F0003F8007F0003 F8007F0003F800FF0003F800FF0003F800FF0003F800FF0003F800FF0003F800FF0003F8 00FF0003F8007F0003F8007F0003F8007F0003F8003F8003F8001F8003F8000F8007F800 07C00FF80003F03BFF8000FFF3FF80003FC3FF80212A7EA926>I<003FE00001FFF80003 F07E0007C01F000F801F801F800F803F800FC07F000FC07F0007C07F0007E0FF0007E0FF 0007E0FFFFFFE0FFFFFFE0FF000000FF000000FF0000007F0000007F0000007F0000003F 8000E01F8000E00FC001C007E0038003F81F0000FFFE00001FF0001B1B7E9A20>I<0007 F0003FFC00FE3E01F87F03F87F03F07F07F07F07F03E07F00007F00007F00007F00007F0 0007F00007F000FFFFC0FFFFC0FFFFC007F00007F00007F00007F00007F00007F00007F0 0007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F0 0007F00007F0007FFF807FFF807FFF80182A7EA915>I<00FF81F003FFE7F80FC1FE7C1F 80FC7C1F007C383F007E107F007F007F007F007F007F007F007F007F007F007F007F003F 007E001F007C001F80FC000FC1F8001FFFE00018FF800038000000380000003C0000003E 0000003FFFF8001FFFFF001FFFFF800FFFFFC007FFFFE01FFFFFF03E0007F07C0001F8F8 0000F8F80000F8F80000F8F80000F87C0001F03C0001E01F0007C00FC01F8003FFFE0000 7FF0001E287E9A22>II<07000FC01FE03FE03FE03FE01FE00FC0070000000000000000 00000000000000FFE0FFE0FFE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00F E00FE00FE00FE00FE00FE00FE00FE00FE00FE0FFFEFFFEFFFE0F2B7DAA14>I<00070000 0F80001FC0003FE0003FE0003FE0001FC0000F8000070000000000000000000000000000 000000000000000001FFE001FFE001FFE0000FE0000FE0000FE0000FE0000FE0000FE000 0FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE000 0FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE07C0FE0FE0FE0FE 0FC0FE1F80FE1F007C3E003FFC000FF000133784AA15>IIIII<003FE00001FFFC0003F07E000FC01F801F800FC0 3F800FE03F0007E07F0007F07F0007F07F0007F0FF0007F8FF0007F8FF0007F8FF0007F8 FF0007F8FF0007F8FF0007F8FF0007F87F0007F07F0007F03F800FE03F800FE01F800FC0 0FC01F8007F07F0001FFFC00003FE0001D1B7E9A22>II<001FC0380000FFF0780003F838F80007E00DF8000FC007F8001FC0 07F8003F8003F8007F8003F8007F8003F8007F0003F800FF0003F800FF0003F800FF0003 F800FF0003F800FF0003F800FF0003F800FF0003F8007F0003F8007F0003F8007F8003F8 003F8003F8001F8007F8000FC007F80007E01FF80003F07BF80000FFF3F800003FC3F800 000003F800000003F800000003F800000003F800000003F800000003F800000003F80000 0003F800000003F80000003FFF8000003FFF8000003FFF8021277E9A24>II<03FE300FFFF01E03F03800F0700070F00070F00070F80070FC 0000FFE0007FFE007FFF803FFFE01FFFF007FFF800FFF80003FC0000FC60007CE0003CF0 003CF00038F80038FC0070FF01E0F7FFC0C1FF00161B7E9A1B>I<007000007000007000 00700000F00000F00000F00001F00003F00003F00007F0001FFFF0FFFFF0FFFFF007F000 07F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F000 07F03807F03807F03807F03807F03807F03803F03803F87001F86000FFC0001F8015267F A51B>III< FFFE7FFC0FFEFFFE7FFC0FFEFFFE7FFC0FFE0FE007E000E007F003F001C007F003F001C0 07F807F803C003F807F8038003F807F8038001FC0EFC070001FC0EFC070001FE1EFC0F00 00FE1C7E0E0000FE1C7E0E0000FF383F1E00007F383F1C00007F783F3C00003FF01FB800 003FF01FB800003FF01FF800001FE00FF000001FE00FF000000FC007E000000FC007E000 000FC007E00000078003C00000078003C0002F1B7F9A32>III<3FFFFF803FFFFF803F007F003C00FE003801FE007803FC007803F8 007007F800700FF000700FE000001FC000003FC000007F8000007F000000FF000001FE03 8001FC038003F8038007F803800FF007800FE007801FE007003FC00F003F801F007F007F 00FFFFFF00FFFFFF00191B7E9A1F>I E /Fn 21 121 df<00001E000000003E00000000 FE00000003FE0000003FFE0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00007FFFFFFF C07FFFFFFFC07FFFFFFFC07FFFFFFFC0223879B731>49 D<0007FE0000007FFFE00001FF FFF80003FFFFFE0007F01FFF000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC000 7FF0FFE0007FF8FFF0003FF8FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE0003F FC3FC0003FFC1F80003FFC0000003FFC0000003FF80000007FF80000007FF00000007FF0 000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00000007F80000 000FF00000001FC00000003F800000007F00000000FC00000001F800000001F0003C0003 E0003C0007C0003C000F000078001E000078003C00007800780000F800F00000F801FFFF FFF803FFFFFFF007FFFFFFF00FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFFFF F0FFFFFFFFE0FFFFFFFFE0FFFFFFFFE026387BB731>I<0003FF8000001FFFF000007FFF FE0000FE03FF0001F000FF8003C000FFC00780007FE00FF0007FF00FF8007FF01FFC007F F81FFE007FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF80FFC007FF007F8007FF0 03F0007FF0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC0000 001FF000000FFFC000000FFF8000000FFFF800000003FE00000000FF800000007FE00000 003FF00000003FF80000003FFC0000001FFC0000001FFE0000001FFE0200001FFF1FC000 1FFF3FE0001FFF7FF0001FFF7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF8001F FEFFF0001FFE7FF0003FFC7FE0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FFC0 03FFFFFF0001FFFFFE00003FFFF0000007FF800028397CB731>I<00000007C000000000 0FC0000000000FC0000000001FC0000000003FC0000000007FC000000000FFC000000000 FFC000000001FFC000000003FFC000000007FFC00000000FFFC00000000FFFC00000001E FFC00000003CFFC00000007CFFC0000000F8FFC0000000F0FFC0000001E0FFC0000003C0 FFC0000007C0FFC000000F80FFC000000F00FFC000001E00FFC000003C00FFC000007C00 FFC00000F800FFC00000F000FFC00001E000FFC00003C000FFC00007C000FFC0000F8000 FFC0000F0000FFC0001E0000FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFFFF FFFFC0FFFFFFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0000001FFC000000001FFC000000001 FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001 FFC000000001FFC0000007FFFFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A377D B631>I<04000000C00F800007C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000FFF FFFC000FFFFFF8000FFFFFF0000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F8000 00000F800000000F800000000F800000000F800000000F800000000F800000000F800000 000F81FF00000F8FFFE0000FBFFFF8000FFE03FE000FF000FF000FC000FF800F80007FC0 0F00007FE00700007FF00000003FF00000003FF80000003FF80000003FF80000003FFC00 00003FFC0600003FFC1F80003FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFFF0 003FFCFFF0003FF8FFE0003FF8FFE0003FF87FC0007FF07F00007FF03C00007FE03E0000 FFC01F0000FF800FC003FF0007F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC00 0026397BB731>I<00000FF80000007FFF000003FFFF80000FFC07C0001FE001E0003FC0 01F0007F0007F000FF000FF001FE001FF803FC003FF807FC003FF80FFC003FF80FF8003F F81FF8001FF01FF8000FE03FF80007C03FF00000003FF00000007FF00000007FF0000000 7FF00000007FF07FF000FFF0FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0FF FC001FF0FFFC000FF8FFF8000FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFFF0 000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF000 0FFF3FF0000FFF3FF0000FFE3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF8001F F807FC001FF003FC001FF001FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF800 0000FFC00028397CB731>I<1E00000000001F00000000001FF0000000001FFFFFFFFFC0 1FFFFFFFFFC01FFFFFFFFFC03FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE00 3FFFFFFFFC003FFFFFFFF8003FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC000 7800001F80007800001F00007800003E0000F000007E0000F00000FC0000F00001F80000 000003F00000000003E00000000007E0000000000FC0000000000F80000000001F800000 00003F80000000003F00000000007F00000000007F0000000000FF0000000000FE000000 0001FE0000000001FE0000000003FE0000000003FE0000000003FE0000000007FC000000 0007FC0000000007FC000000000FFC000000000FFC000000000FFC000000000FFC000000 000FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000 001FFC000000001FFC000000001FFC000000001FFC000000000FF80000000007F0000000 0003E00000002A3B7BB931>I<0001FF8000000FFFF800003FFFFE00007F00FF0000F800 3F8001F0000FC003E0000FE007C00007F007C00007F00FC00003F80FC00003F81FC00003 F81FC00003F81FE00003F81FF00003F81FF80003F81FFC0007F01FFF0007F01FFFC007E0 0FFFF00FE00FFFFC1FC007FFFE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE0000 7FFFFF00003FFFFF80003FFFFFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1FC0 07FFFC3F8001FFFE3F8000FFFE7F00003FFF7F00000FFFFE000003FFFE000001FFFE0000 00FFFE000000FFFE0000007FFE0000007FFE0000007EFE0000007E7F0000007E7F000000 FC3F800000FC3FC00001F81FE00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE00 003FFFF8000003FF800028397CB731>I<0001FF8000001FFFF000007FFFFC0000FF81FE 0001FE007F0007FC003F800FF8003FC00FF8001FE01FF8001FF03FF0001FF03FF0000FF8 7FF0000FF87FF0000FFCFFF0000FFCFFF0000FFCFFF0000FFEFFF0000FFEFFF0000FFEFF F0000FFEFFF0000FFEFFF0000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0001FFF7FF0 001FFF3FF0001FFF3FF0001FFF1FF0003FFF0FF8003FFF07F8007FFF03FC006FFF01FE01 CFFF00FFEF8FFF007FFF0FFF000FFE0FFE0000000FFE0000000FFE0000000FFE0000000F FC0000000FFC03E0001FFC07F0001FF80FF8001FF81FFC001FF01FFC003FF01FFC003FE0 1FFC003FC01FF8007FC00FF000FF800FE001FF0007C003FE0007F01FFC0003FFFFF00001 FFFFC000007FFF0000000FF8000028397CB731>I<00000001F80000000000000001F800 00000000000003FC0000000000000003FC0000000000000007FE0000000000000007FE00 00000000000007FE000000000000000FFF000000000000000FFF000000000000001FFF80 0000000000001FFF800000000000001FFF800000000000003FFFC00000000000003FFFC0 0000000000007FFFE00000000000007DFFE00000000000007DFFE0000000000000FDFFF0 000000000000F8FFF0000000000001F8FFF8000000000001F07FF8000000000001F07FF8 000000000003F07FFC000000000003E03FFC000000000007E03FFE000000000007C01FFE 00000000000FC01FFF00000000000F801FFF00000000000F800FFF00000000001F800FFF 80000000001F0007FF80000000003F0007FFC0000000003E0003FFC0000000003E0003FF C0000000007E0003FFE0000000007C0001FFE000000000FC0001FFF000000000F80000FF F000000000F80000FFF000000001FFFFFFFFF800000001FFFFFFFFF800000003FFFFFFFF FC00000003FFFFFFFFFC00000003E000003FFC00000007E000003FFE00000007C000001F FE0000000FC000001FFF0000000F8000000FFF0000001F8000000FFF8000001F0000000F FF8000001F00000007FF8000003F00000007FFC000003E00000003FFC000007E00000003 FFE00000FF00000001FFE000FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001FF FFFFF0FFFFFC0001FFFFFFF0443B7DBA4B>65 D<0000001FFF000030000001FFFFE000F0 00000FFFFFFC01F000007FFFFFFE03F00001FFFE007F87F00003FFE0000FCFF0000FFF00 0003FFF0001FFC000001FFF0003FF80000007FF0007FF00000003FF000FFC00000003FF0 01FFC00000001FF003FF800000000FF007FF000000000FF00FFF0000000007F00FFE0000 000007F01FFE0000000003F01FFE0000000003F03FFC0000000003F03FFC0000000001F0 3FFC0000000001F07FFC0000000001F07FF80000000001F07FF80000000000007FF80000 00000000FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000 FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF80000 00000000FFF8000000000000FFF80000000000007FF80000000000007FF8000000000000 7FF80000000000007FFC0000000000F03FFC0000000000F03FFC0000000000F03FFC0000 000000F01FFE0000000000F01FFE0000000001E00FFE0000000001E00FFF0000000001E0 07FF0000000003C003FF8000000003C001FFC0000000078000FFE00000000F00007FF000 00001F00003FF80000003E00001FFC0000007C00000FFF000001F8000003FFE00007F000 0001FFFE003FC00000007FFFFFFF000000000FFFFFFC0000000001FFFFF000000000001F FF0000003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007FE0000 0FF801FF00001FFC00FF80001FFC007FC0001FFC007FE0001FFC003FE0000FF8003FF000 0FF8003FF00007F0003FF00001C0003FF0000000003FF0000000003FF0000000003FF000 0000FFFFF000000FFFFFF000007FF83FF00001FF803FF00007FE003FF0000FF8003FF000 1FF0003FF0003FE0003FF0007FE0003FF0007FE0003FF000FFC0003FF000FFC0003FF000 FFC0003FF000FFC0003FF000FFC0007FF0007FE0007FF0007FE000DFF0003FF0039FF800 1FFC0F0FFFF007FFFE0FFFF001FFFC07FFF0003FE000FFF02C267DA530>97 D<000000003FC00000003FFFC00000003FFFC00000003FFFC00000003FFFC000000001FF C000000000FFC000000000FFC000000000FFC000000000FFC000000000FFC000000000FF C000000000FFC000000000FFC000000000FFC000000000FFC000000000FFC000000000FF C000000000FFC000000000FFC000000000FFC000000000FFC00000FFC0FFC0000FFFF8FF C0003FFFFEFFC000FFE03FFFC001FF000FFFC003FE0003FFC007FC0001FFC00FF80000FF C01FF00000FFC01FF00000FFC03FF00000FFC03FE00000FFC07FE00000FFC07FE00000FF C07FE00000FFC0FFE00000FFC0FFE00000FFC0FFE00000FFC0FFE00000FFC0FFE00000FF C0FFE00000FFC0FFE00000FFC0FFE00000FFC0FFE00000FFC07FE00000FFC07FE00000FF C07FE00000FFC03FF00000FFC03FF00000FFC01FF00000FFC00FF80001FFC007F80003FF C003FC0007FFC001FE000FFFE000FFC07EFFFF003FFFFCFFFF000FFFF0FFFF0001FF80FF FF303C7DBB37>100 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007 FC001FC00FF8000FE00FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE0 0003FC7FE00003FC7FE00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE000 0000FFE0000000FFE0000000FFE00000007FE00000007FE00000007FE00000003FE00000 003FF000003C1FF000003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E0 00FFC03FC0003FFFFF000007FFFC000000FFE00026267DA52D>I<00FF00000000FFFF00 000000FFFF00000000FFFF00000000FFFF0000000007FF0000000003FF0000000003FF00 00000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF00 00000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF00 00000003FF0000000003FF0000000003FF007FC00003FF01FFF80003FF07FFFC0003FF0F 03FE0003FF1C01FF0003FF3001FF8003FF6000FF8003FFE000FFC003FFC000FFC003FF80 00FFC003FF8000FFC003FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF00 00FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF00 00FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF00 00FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF00 00FFC0FFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF303C7CBB37>104 D<00F00003FC0007FE000FFE000FFF001FFF001FFF001FFF000FFF000FFE0007FE0003FC 0000F00000000000000000000000000000000000000000000000000000000000000000FF 00FFFF00FFFF00FFFF00FFFF0007FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF 0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF 0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF00FFFFF8FFFFF8FFFF F8FFFFF8153D7DBC1B>I<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FF FE1C01FF0007FE3001FF8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003 FF8000FFC003FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003 FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003 FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003 FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FF FFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<00FF01FF8000FFFF0FFFF000FFFF3FFFFC00FFFFFE03FF00FFFFF000FF8003FFC0007F C003FF80003FE003FF00003FF003FF00001FF803FF00001FFC03FF00000FFC03FF00000F FE03FF00000FFE03FF000007FE03FF000007FF03FF000007FF03FF000007FF03FF000007 FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007 FE03FF000007FE03FF00000FFE03FF00000FFC03FF00000FFC03FF00001FF803FF00001F F803FF00003FF003FF80003FE003FFC0007FC003FFF001FF8003FFFC07FF0003FF3FFFFC 0003FF0FFFF00003FF01FF000003FF0000000003FF0000000003FF0000000003FF000000 0003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF000000 0003FF0000000003FF0000000003FF00000000FFFFFC000000FFFFFC000000FFFFFC0000 00FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80 FFFE707FC007FE60FFE003FEE0FFE003FEC0FFE003FFC0FFE003FF807FC003FF807FC003 FF803F8003FF800E0003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF0000 0003FF000000FFFFFE0000FFFFFE0000FFFFFE0000FFFFFE000023267DA529>114 D<0007800000078000000780000007800000078000000F8000000F8000000F8000000F80 00001F8000001F8000003F8000003F8000007F800000FF800001FF800007FF80001FFFFF F0FFFFFFF0FFFFFFF0FFFFFFF001FF800001FF800001FF800001FF800001FF800001FF80 0001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF80 0001FF800001FF800001FF800001FF800001FF803C01FF803C01FF803C01FF803C01FF80 3C01FF803C01FF803C01FF803C00FF807800FFC078007FC070003FE0E0001FFFC00007FF 800001FF001E377EB626>116 D120 D E /Fo 8 107 df0 D<70F8F8F87005057C8D0D>I< 01800180018001800180C183F18F399C0FF003C003C00FF0399CF18FC183018001800180 0180018010147D9417>3 D<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF 7FFE7FFE3FFC1FF80FF003C010127D9317>15 D<000000C0000003C000000F0000003C00 0000F0000003C00000070000001C00000078000001E00000078000001E00000078000000 E0000000780000001E0000000780000001E0000000780000001C0000000700000003C000 0000F00000003C0000000F00000003C0000000C000000000000000000000000000000000 0000000000000000000000007FFFFF80FFFFFFC01A247C9C23>20 DI<00000004000000000200000000020000000001000000000080000000 00400000000020FFFFFFFFFCFFFFFFFFFC00000000200000000040000000008000000001 0000000002000000000200000000040026107D922D>33 D106 D E /Fp 1 98 df<001800003C00003C00003C00007E00007E00007E0000CF 0000CF0000CF000187800187800387C00303C00303C007FFE007FFE00601E00C00F00C00 F01E00F8FF03FFFF03FF18177F961C>97 D E /Fq 38 90 df<387C7C7E3A0404040808 10204080070E789F0D>39 D<1C3E7E7E3A0202040408081020C0070E7D840D>44 D<3078F8787005057C840D>46 D<007E0001C3000301800701C00E00C00E00E01C00E01C 00E03C01E03801E07801E07801E07801E07801E07801E0F003C0F003C0F003C0F003C0F0 03C0F00380F00780E00780E00700E00700E00E00600E00701C003038003870000FC00013 1F7C9D17>48 D<000C001C00FC0F38003800380038003800380070007000700070007000 7000E000E000E000E000E000E001C001C001C001C001C001C0038003C0FFFE0F1E7C9D17 >I<003F8000C1E00100F00200780400780400780F007C0F807C0F807C0F007806007800 00F80000F00001E00001C0000380000700000E00001C0000380000600000C00001800003 00200600200800401000403FFFC07FFF80FFFF80161E7E9D17>I<007F000183C00201E0 0400F00700F00F00F00F01F00F01F00001E00001E00003C0000380000700000E0000F800 000E000007000007800007C00003C00007C03007C07807C0F807C0F807C0F00780800F00 400E00201C0018780007E000141F7D9D17>I<0000600000600000E00001C00003C00005 C0000DC00009C00011C000238000438000C3800083800103800203800407000807001807 00100700200700400700FFFFF0000E00000E00000E00000E00000E00001C00001E0001FF E0141E7E9D17>I<01803001FFE003FFC003FF0003FC0002000002000002000004000004 0000040000047C000587000603800C01800801C00001C00001E00001E00001E00001E070 03C0F803C0F003C0E00380800780400700400E00201C0018700007C000141F7D9D17>I< 000F8000704000C0200180E00301E00701E00E00C01E00001C00003C000038000078F800 790E007A07007C0300F80380F80380F003C0F003C0F003C0F003C0F00780E00780E00780 E00700E00F00600E00701C0030180018700007C000131F7C9D17>I<2000003FFFE07FFF C07FFF80400100C002008002008004000008000010000020000040000040000080000180 000300000300000700000600000E00000E00001E00001C00001C00003C00003C00003C00 00780000780000780000300000131F799D17>I<003F0000C1C00100600200600400300C 00300C00300C00300C00600E00600F80C00FC18007F60003FC0001FC0001FF00063F800C 0F801007C03003C06001C06000C0C000C0C000C0C000C0C00080C0010060030030040018 180007E000141F7D9D17>I<007E0001C3000301800601C00E01C01C00C03C00E03C00E0 3C01E07801E07801E07801E07801E07803E07803E03803C03807C01C0BC00C13C003E380 000780000780000700000E00600E00F01C00F01800E0300080600041C0003F0000131F7C 9D17>I<07E01838201C401C701CF03CF03C603C0038007000E001C00180030002000600 0400040008000800080008000000000000000000000030007800F800780070000E20799F 15>63 D<0000100000001800000038000000380000007800000078000000FC000001BC00 00013C0000033C0000023C0000063C0000043E0000081E0000081E0000101E0000101E00 00201E0000200F0000400F0000400F0000FFFF0000800F0001000F800100078002000780 0200078004000780040007800C0007C03E0007C0FF807FFC1E207E9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800F8007800 F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007C001E003 C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003C003C007 C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E061800380138 007000F801E0007803C0007007800030078000300F0000301F0000301E0000303E000020 3C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000F8000000 F80000007800004078000080780000803C0000803C0001001C0002000E00020006000C00 0300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F0003C007800 78003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00F0001F00 F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01E0007C03 C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FFFFE00020 1F7E9E23>I<07FFFFF8007C0078003C0038003C00180078001800780008007800080078 0008007800080078080800F0100000F0100000F0100000F0300000FFF00000F0700001E0 200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C0002003C0 002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFFF8007C00 78003C0038003C001800780018007800080078000800780008007800080078000800F010 0000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E0200001E020 0001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C00000078000 0007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C00E0003C 01C0003803800018078000180F0000181F0000181E0000183E0000103C0000007C000000 7C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E0780001E0 780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B800380118001E06080 003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C007800007800F000 007800F000007800F000007800F000007800F000007800F00000F001E00000F001E00000 F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003C00001E0 03C00001E003C00001E003C00003C007800003C007800003C007800003C007800003C007 800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C0000 3C00003C0000780000780000780000780000780000780000F00000F00000F00000F00000 F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C00003C00003 C00003C00007800007C000FFFC00131F7F9E10>I<00FFFC0007C00003C00003C0000780 000780000780000780000780000780000F00000F00000F00000F00000F00000F00001E00 001E00001E00001E00001E00001E00003C00303C00783C00F83C00F83800F07800807000 40E00021C0001F000016207D9E17>I<07FFE07FE0007C001F00003C000C00003C001800 00780010000078004000007800800000780100000078020000007804000000F008000000 F010000000F060000000F0F0000000F1F0000000F278000001E478000001E878000001F0 3C000001E03C000001E01E000001E01E000003C00F000003C00F000003C00F000003C007 800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F7E9E23>I<07FF F000007E0000003C0000003C000000780000007800000078000000780000007800000078 000000F0000000F0000000F0000000F0000000F0000000F0000001E0000001E0000001E0 000001E0000001E0008001E0010003C0010003C0010003C0030003C0020003C0060003C0 060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC0007C0000F800003C00 017800003C00017800004E0002F000004E0002F000004E0004F000004E0004F000004E00 08F000004E0008F00000870011E00000870011E00000870021E00000870021E000008700 41E00000838041E00001038083C00001038083C00001038103C00001038203C0000101C2 03C0000101C403C0000201C40780000201C80780000201C80780000201D00780000200F0 0780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A1F7E9E2A>I<07FC 01FFC0003E003E00003E001800003E001800004F001000004F0010000047801000004780 10000043C010000043C010000083C020000081E020000081E020000080F020000080F020 000080782000010078400001007C400001003C400001003C400001001E400001001E4000 02000F800002000F800002000F800002000780000200078000060003800006000300000F 00010000FFE0010000221F7E9E22>I<0003F800001E0E000038070000E0038001C001C0 03C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F83C0000F87C0000F8 7C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0F80003E0780003E0 780003C0780007C07C0007803C000F003C001E001E001C000E0038000700F00003C3C000 00FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F0007800F0007800F800 7800F8007800F8007800F8007800F000F001F000F001E000F003C000F0078000F00F0000 FFF80001E0000001E0000001E0000001E0000001E0000001E0000003C0000003C0000003 C0000003C0000003C0000003C000000780000007C00000FFFC00001D1F7E9E1F>I<07FF FC00007C0700003C03C0003C01E0007801E0007801F0007801F0007801F0007801F00078 01E000F003E000F003C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E0 1C0001E01C0001E01C0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C0 3E0207803E0407C01F04FFFC0F18000003E01F207E9E21>82 D<003F040060CC01803C03 801C03001C0700180600080E00080E00080E00080E00000F00000F80000FE00007FE0003 FF8001FFC0007FE00007E00001E00000E00000F00000F04000E04000E04000E04000E060 00C0600180E00380F80300C60C0081F80016217D9F19>I<3FFFFFF03C0780F030078030 60078030400F0010400F0010C00F0010800F0010800F0010800F0010001E0000001E0000 001E0000001E0000001E0000001E0000003C0000003C0000003C0000003C0000003C0000 003C000000780000007800000078000000780000007800000078000000F0000001F80000 7FFFE0001C1F7A9E21>IIII<03FFC0FFC0007F007E00003E003800001E003000001E0020 00000F004000000F008000000F81000000078200000007C600000003C400000003E80000 0001F000000001F000000000F000000000F800000000F8000000017C000000023C000000 043C0000000C1E000000081E000000101F000000200F000000400F800000C00780000080 07C000010003C000070003E0001F8007E000FFE01FFE00221F7F9E22>II E /Fr 23 123 df<70F8F8F87005057C840D> 58 D<70F8FCFC74040404080810102040060E7C840D>I<000001C00000078000001E0000 0078000001E00000078000000E00000038000000F0000003C000000F0000003C000000F0 000000F00000003C0000000F00000003C0000000F0000000380000000E00000007800000 01E0000000780000001E0000000780000001C01A1A7C9723>I62 D<007C01C207010E0F1E0F1C0E3C04780078007800F000F000F000F000F0007001700230 0418380FC010147E9314>99 D<0000780003F80000700000700000700000700000E00000 E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C03803807007807 00780700780700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F07 8015207E9F18>I<007C01C207010E011C013C013802780C7BF07C00F000F000F000F000 7000700170023004183807C010147E9315>I<00007C0000CE00019E00039E00030C0007 00000700000700000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C 00001C00001C000038000038000038000038000038000070000070000070000070000070 0000E00000E00000E00000E00000C00001C000318000798000F300006200003C00001729 7E9F16>I<001E3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E 01C01E01C01E01C01E01C01E03801E03800E07800E0B8006170001E70000070000070000 0E00000E00300E00781C00F038006070003FC000151D809316>I<00E001E001E000C000 000000000000000000000000000E00130023804380438043808700070007000E000E001C 001C001C20384038403840388019000E000B1F7E9E10>105 D<0000C00001E00001E000 01C0000000000000000000000000000000000000000000001E0000630000438000838001 0380010380020700000700000700000700000E00000E00000E00000E00001C00001C0000 1C00001C0000380000380000380000380000700000700030700078E000F1C0006380003E 00001328819E13>I<01E0000FE00001C00001C00001C00001C000038000038000038000 0380000700000700000701E00706100E08700E10F00E20F00E40601C80001D00001E0000 1FC000387000383800383800381C20703840703840703840701880E01880600F0014207E 9F18>I<03C01FC0038003800380038007000700070007000E000E000E000E001C001C00 1C001C0038003800380038007000700070007100E200E200E200E200640038000A207E9F 0E>I<1E07C07C00231861860023A032030043C034030043803803804380380380870070 07000700700700070070070007007007000E00E00E000E00E00E000E00E00E000E00E01C 101C01C01C201C01C038201C01C038401C01C0184038038018801801800F0024147E9328 >I<1E07802318C023A06043C0704380704380708700E00700E00700E00700E00E01C00E 01C00E01C00E03821C03841C07041C07081C03083803101801E017147E931B>I<007C00 01C3000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0 F00780F00700700F00700E0030180018700007C00013147E9316>I<03C1E00462180474 1C08781C08701E08701E10E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380 780380700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00 001C0000FFC000171D819317>I<1E1E0023210023C38043C78043878043830087000007 00000700000700000E00000E00000E00000E00001C00001C00001C00001C000038000018 000011147E9315>114 D<007C018203010603060706060E00078007F803FC01FE001F00 077007F006F006E004400820301FC010147E9315>I<00C000E001C001C001C001C00380 0380FFF8038007000700070007000E000E000E000E001C001C001C001C10382038203820 384018800F000D1C7F9B10>I<03C1C00C62201034701038F02038F02038604070000070 0000700000700000E00000E00000E00000E02061C040F1C040F1C080E2C080446300383C 0014147E931A>120 D<0F00601180702180E021C0E041C0E04380E08381C00701C00701 C00701C00E03800E03800E03800E03800E07000C07000C07000E0F00061E0003EE00000E 00000E00001C0078180078380070700060600021C0001F0000141D7E9316>I<01E02003 F04007F8C00C1F8008010000020000040000080000100000600000C00001000002000004 00800801001003003F060061FC0040F80080700013147E9315>I E /Fs 86 128 df<001F83E000F06E3001C078780380F8780300F0300700700007007000 0700700007007000070070000700700007007000FFFFFF80070070000700700007007000 070070000700700007007000070070000700700007007000070070000700700007007000 0700700007007000070070000700700007007000070070007FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700000700000700 00FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000 E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F04 0001C07C06000380F80F000300F00F000700F00F00070070000007007000000700700000 070070000007007000000700700000FFFFFFFF0007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 07007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C07C0F000380F80F000300 F00700070070070007007007000700700700070070070007007007000700700700070070 0700FFFFFFFF000700700700070070070007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 070070070007007007000700700700070070070007007007007FE3FE3FF02420809F26> I<7038F87CFC7EFC7E743A0402040204020804080410081008201040200F0E7E9F17>34 D<70F8FCFC74040404080810102040060E7C9F0D>39 D<0020004000800100020006000C 000C00180018003000300030007000600060006000E000E000E000E000E000E000E000E0 00E000E000E000E0006000600060007000300030003000180018000C000C000600020001 000080004000200B2E7DA112>I<800040002000100008000C0006000600030003000180 0180018001C000C000C000C000E000E000E000E000E000E000E000E000E000E000E000E0 00C000C000C001C001800180018003000300060006000C00080010002000400080000B2E 7DA112>I<01800180018001800180C183F18F399C0FF003C003C00FF0399CF18FC18301 80018001800180018010147DA117>I<0006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 00060000FFFFFFF0FFFFFFF0000600000006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 1C207D9A23>I<70F8FCFC74040404080810102040060E7C840D>II<70F8F8F87005057C840D>I<000100030003000600060006000C000C000C0018001800 1800300030003000600060006000C000C000C00180018001800300030003000600060006 000C000C000C00180018001800300030003000600060006000C000C000C000102D7DA117 >I<03F0000E1C001C0E00180600380700700380700380700380700380F003C0F003C0F0 03C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C070038070 03807003807807803807001806001C0E000E1C0003F000121F7E9D17>I<018003800F80 F38003800380038003800380038003800380038003800380038003800380038003800380 038003800380038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C00100E002007 00400780800780F007C0F803C0F803C0F803C02007C00007C0000780000780000F00000E 00001C0000380000700000600000C0000180000300000600400C00401800401000803FFF 807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00780F8078078078078038 0F80000F80000F00000F00000E00001C0000380003F000003C00000E00000F0000078000 07800007C02007C0F807C0F807C0F807C0F00780400780400F00200E001C3C0003F00012 1F7E9D17>I<000600000600000E00000E00001E00002E00002E00004E00008E00008E00 010E00020E00020E00040E00080E00080E00100E00200E00200E00400E00C00E00FFFFF0 000E00000E00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I<1803001FFE 001FFC001FF8001FE00010000010000010000010000010000010000011F000161C00180E 001007001007800003800003800003C00003C00003C07003C0F003C0F003C0E003804003 80400700200600100E000C380003E000121F7E9D17>I<007C000182000701000E03800C 07801C0780380300380000780000700000700000F1F000F21C00F40600F80700F80380F8 0380F003C0F003C0F003C0F003C0F003C07003C07003C07003803803803807001807000C 0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF80400100800200800200 80040000080000080000100000200000200000400000400000C00000C00001C000018000 038000038000038000038000078000078000078000078000078000078000078000030000 121F7D9D17>I<03F0000C0C001006003003002001806001806001806001807001807803 003E03003F06001FC8000FF00003F80007FC000C7E00103F00300F806003804001C0C001 C0C000C0C000C0C000C0C000806001802001001002000C0C0003F000121F7E9D17>I<03 F0000E18001C0C00380600380700700700700380F00380F00380F003C0F003C0F003C0F0 03C0F003C07007C07007C03807C0180BC00E13C003E3C000038000038000038000070030 0700780600780E00700C002018001070000FC000121F7E9D17>I<70F8F8F87000000000 00000000000070F8F8F87005147C930D>I<70F8F8F8700000000000000000000070F0F8 F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF000000000000000000000 00000000000000000000000000000000000000000000FFFFFFF07FFFFFE01C0C7D9023> 61 D<0FC0307040384038E03CF03CF03C603C0038007000E000C0018001800100030002 00020002000200020002000000000000000000000007000F800F800F8007000E207D9F15 >63 D<000100000003800000038000000380000007C0000007C0000007C0000009E00000 09E0000009E0000010F0000010F0000010F00000207800002078000020780000403C0000 403C0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F000200078002 00078002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F204F00 07A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000001FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F00 001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004078004 0F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C>II<080410082010201040204020804080408040B85CFC7E FC7E7C3E381C0F0E7B9F17>II<1FE00030300078 1800781C00300E00000E00000E00000E0000FE00078E001E0E00380E00780E00F00E10F0 0E10F00E10F01E10781E103867200F83C014147E9317>97 D<0E0000FE00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E3E000EC3800F01C00F00 E00E00E00E00700E00700E00780E00780E00780E00780E00780E00780E00700E00700E00 E00F00E00D01C00CC300083E0015207F9F19>I<03F80E0C1C1E381E380C70007000F000 F000F000F000F000F00070007000380138011C020E0C03F010147E9314>I<000380003F 8000038000038000038000038000038000038000038000038000038000038003E380061B 801C0780380380380380700380700380F00380F00380F00380F00380F00380F003807003 807003803803803807801C07800E1B8003E3F815207E9F19>I<03F0000E1C001C0E0038 0700380700700700700380F00380F00380FFFF80F00000F00000F0000070000070000038 00801800800C010007060001F80011147F9314>I<007C00C6018F038F07060700070007 000700070007000700FFF007000700070007000700070007000700070007000700070007 00070007000700070007007FF01020809F0E>I<0000E003E3300E3C301C1C30380E0078 0F00780F00780F00780F00780F00380E001C1C001E380033E00020000020000030000030 00003FFE001FFF800FFFC03001E0600070C00030C00030C00030C000306000603000C01C 038003FC00141F7F9417>I<0E0000FE00000E00000E00000E00000E00000E00000E0000 0E00000E00000E00000E00000E3E000E43000E81800F01C00F01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC 16207F9F19>I<1C001E003E001E001C000000000000000000000000000E007E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A1F80 9E0C>I<00E001F001F001F000E0000000000000000000000000007007F000F000700070 007000700070007000700070007000700070007000700070007000700070007000700070 00706070F060F0C061803F000C28829E0E>I<0E0000FE00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E0FF00E03C00E03000E02000E04000E08 000E10000E30000E70000EF8000F38000E1C000E1E000E0E000E07000E07800E03800E03 C00E03E0FFCFF815207F9F18>I<0E00FE000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E00FFE00B20809F0C>I<0E1F01F000FE618618000E81C81C000F00F00E000F00F0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E00 FFE7FE7FE023147F9326>I<0E3E00FE43000E81800F01C00F01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC 16147F9319>I<01F800070E001C03803801C03801C07000E07000E0F000F0F000F0F000 F0F000F0F000F0F000F07000E07000E03801C03801C01C0380070E0001F80014147F9317 >I<0E3E00FEC3800F01C00F00E00E00E00E00F00E00700E00780E00780E00780E00780E 00780E00780E00700E00F00E00E00F01E00F01C00EC3000E3E000E00000E00000E00000E 00000E00000E00000E00000E0000FFE000151D7F9319>I<03E0800619801C05803C0780 380380780380700380F00380F00380F00380F00380F00380F00380700380780380380380 3807801C0B800E138003E380000380000380000380000380000380000380000380000380 003FF8151D7E9318>I<0E78FE8C0F1E0F1E0F0C0E000E000E000E000E000E000E000E00 0E000E000E000E000E000E00FFE00F147F9312>I<1F9030704030C010C010C010E00078 007F803FE00FF00070803880188018C018C018E030D0608F800D147E9312>I<02000200 0200060006000E000E003E00FFF80E000E000E000E000E000E000E000E000E000E000E00 0E080E080E080E080E080610031001E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 03C00603C0030DC001F1FC16147F9319>III<7FC3FC0F01E00701 C007018003810001C20000E40000EC00007800003800003C00007C00004E000087000107 000303800201C00601E01E01E0FF07FE1714809318>II<3FFF380E200E201C40384078407000E001E001C00380078007010E 011E011C0338027006700EFFFE10147F9314>II<30307878F87C78 7830300E057C9E17>127 D E /Ft 71 123 df<000FF07F00007FF9FF8000F83FC7C001 E07F8FC003E07F0FC007C07F0FC007C03F078007C01F000007C01F000007C01F000007C0 1F000007C01F0000FFFFFFF800FFFFFFF80007C01F000007C01F000007C01F000007C01F 000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F00 0007C01F000007C01F000007C01F000007C01F000007C01F00003FF8FFF0003FF8FFF000 2220809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C0007C07C0007C07C00 07C0380007C0000007C0000007C0000007C1FE00FFFFFE00FFFFFE0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>I<00 0FFE00007FFE0000F87E0001E0FE0003E0FE0007C07E0007C07E0007C03E0007C03E0007 C03E0007C03E0007C03E00FFFFFE00FFFFFE0007C03E0007C03E0007C03E0007C03E0007 C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007 C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>I<0018007000E001C00380 038007000E000E001E001C003C003C007800780078007800F800F000F000F000F000F000 F000F000F000F000F80078007800780078003C003C001C001E000E000E00070003800380 01C000E0007000180D2D7DA114>40 DI<00E00001F00000E00000E000E0E0E0F0E1E0FCE7E0FE4FE01F5F0003F80003 F8001F5F00FE4FE0FCE7E0F0E1E0E0E0E000E00000E00001F00000E00013147DA11A>I< 387CFEFFFF7F3B030306060E0C18702008107C860F>44 DI<387CFEFEFE7C3807077C860F>I<0000600000E00000E00001C00001C00003800003 80000380000700000700000700000E00000E00001C00001C00001C000038000038000038 0000700000700000E00000E00000E00001C00001C0000380000380000380000700000700 000700000E00000E00001C00001C00001C0000380000380000380000700000700000E000 00E00000C00000132D7DA11A>I<01FC0007FF001F07C01E03C03E03E07C01F07C01F07C 01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC 01F8FC01F87C01F07C01F07C01F03E03E01E03C01F8FC007FF0001FC00151D7E9C1A>I< 00E00001E0000FE000FFE000F3E00003E00003E00003E00003E00003E00003E00003E000 03E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E000 03E00003E00003E000FFFF80FFFF80111D7C9C1A>I<07F0001FFE00383F007C1F80FE0F C0FE0FC0FE0FE0FE07E07C07E03807E0000FE0000FC0000FC0001F80001F00003E000078 0000F00000E00001C0000380600700600E00601C00E01FFFC03FFFC07FFFC0FFFFC0FFFF C0131D7D9C1A>I<01FC0007FF000E0F801E0FC03F07E03F07E03F07E03F07E01E0FC000 0FC0000F80001F0001FC0001FC00000F800007C00003E00003F00003F83803F87C03F8FE 03F8FE03F8FE03F0FC03F07807E03C0FC01FFF8003FC00151D7E9C1A>I<0001C00003C0 0007C00007C0000FC0001FC0003BC00073C00063C000C3C00183C00383C00703C00E03C0 0C03C01803C03803C07003C0E003C0FFFFFEFFFFFE0007C00007C00007C00007C00007C0 0007C000FFFE00FFFE171D7F9C1A>I<3803803FFF803FFF003FFE003FFC003FF0003F80 0030000030000030000030000033F80037FE003C1F00380F801007C00007C00007E00007 E07807E0FC07E0FC07E0FC07E0FC07C0780FC0600F80381F001FFC0007F000131D7D9C1A >I<003F0001FFC007E0E00F81E01F03F01E03F03E03F07C03F07C01E07C0000FC1000FC FF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8FC01F8FC01F87C01F87C01F87C01F83C 01F03E01F01E03E00F07C007FF8001FE00151D7E9C1A>I<6000007FFFF87FFFF87FFFF0 7FFFE07FFFC0E00180C00300C00300C00600000C00001800003800003800007800007000 00F00000F00001F00001F00001F00001F00003F00003F00003F00003F00003F00003F000 01E00000C000151E7D9D1A>I<01FC0007FF000F07801E03C01C01E03C01E03C01E03E01 E03F01E03FC3C01FE3801FFF000FFE0007FF8007FFC01FFFE03C3FF0780FF07803F8F001 F8F000F8F00078F00078F000707800707C00E03E03C00FFF8003FC00151D7E9C1A>I<01 FC000FFF001F07803E03C07C03E07C01E0FC01F0FC01F0FC01F0FC01F8FC01F8FC01F8FC 01F87C03F87C03F83E05F81FFDF807F9F80041F80001F03C01F07E01F07E03E07E03E07E 07C03C0780381F001FFC0007F000151D7E9C1A>I<7FFFFFFF80FFFFFFFFC0FFFFFFFFC0 000000000000000000000000000000000000000000000000000000000000000000000000 000000000000000000FFFFFFFFC0FFFFFFFFC07FFFFFFF80220F7D9229>61 D<0000E000000000E000000001F000000001F000000001F000000003F800000003F80000 0006FC00000006FC0000000EFE0000000C7E0000000C7E000000183F000000183F000000 303F800000301F800000701FC00000600FC00000600FC00000C007E00000FFFFE00001FF FFF000018003F000018003F000030001F800030001F800060001FC00060000FC000E0000 FE00FFE00FFFE0FFE00FFFE0231F7E9E28>65 DI<0007FC02003FFF0E00FE03DE03F000FE07E0003E0FC0001E1F80001E3F 00000E3F00000E7F0000067E0000067E000006FE000000FE000000FE000000FE000000FE 000000FE000000FE0000007E0000007E0000067F0000063F0000063F00000C1F80000C0F C0001807E0003803F0007000FE01C0003FFF800007FC001F1F7D9E26>IIII<0007FC0200003FFF0E0000FE03DE 0003F000FE0007E0003E000FC0001E001F80001E003F00000E003F00000E007F00000600 7E000006007E00000600FE00000000FE00000000FE00000000FE00000000FE00000000FE 003FFFE0FE003FFFE07E00007E007E00007E007F00007E003F00007E003F00007E001F80 007E000FC0007E0007E0007E0003F000FE0000FE01FE00003FFF8E000007FC0600231F7D 9E29>III 75 DIII<001FF80000FFFF0001F81F8007E007 E00FC003F01F8001F81F0000F83F0000FC7F0000FE7E00007E7E00007EFE00007FFE0000 7FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007F7E00007E7F0000 FE7F0000FE3F0000FC3F8001FC1F8001F80FC003F007E007E001F81F8000FFFF00001FF8 00201F7D9E27>II<001FF80000 FFFF0001F81F8007E007E00FC003F01F8001F81F8001F83F0000FC7F0000FE7F0000FE7E 00007EFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE 00007F7E00007E7E00007E7F0000FE3F0000FC3F87C1FC1F8FE1F80FD833F007F81FE001 F81F8000FFFF00001FFE0300000E0300000F0700000FFF000007FF000007FE000007FE00 0003FC000001F8000000F020287D9E27>II< 03FC080FFF381E03F83800F8700078700038F00038F00018F00018F80000FC00007FC000 7FFE003FFF801FFFE00FFFF007FFF000FFF80007F80000FC00007C00003CC0003CC0003C C0003CE00038E00078F80070FE01E0E7FFC081FF00161F7D9E1D>I<7FFFFFFC7FFFFFFC 7C07E07C7007E01C6007E00C6007E00CE007E00EC007E006C007E006C007E006C007E006 0007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E000 0007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00003FFFFC0 03FFFFC01F1E7E9D24>IIII II<3FFFFF803FFFFF803F803F003E007F0038 007E003800FC007001FC007001F8006003F0006007F0006007E000000FC000001FC00000 1F8000003F0000007F0000007E000000FC000001FC018001F8018003F0018007F0018007 E003800FC003801FC003001F8007003F000F007F001F007E007F00FFFFFF00FFFFFF0019 1F7D9E20>I<07FC001FFF003F0F803F07C03F03E03F03E00C03E00003E0007FE007FBE0 1F03E03C03E07C03E0F803E0F803E0F803E0FC05E07E0DE03FF8FE0FE07E17147F9319> 97 DI<01FE 0007FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC0000FC0000FC0000FC0000FC00 007C00007E00003E00603F00C01F81C007FF0001FC0013147E9317>I<0007F80007F800 00F80000F80000F80000F80000F80000F80000F80000F80000F80000F801F8F80FFEF81F 83F83E01F87E00F87C00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87C 00F87E00F83E01F81F07F80FFEFF03F8FF18207E9F1D>I<01FE0007FF800F83C01E01E0 3E00F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C00003E0018 1E00180F807007FFE000FF8015147F9318>I<001F8000FFC001F3E003E7E003C7E007C7 E007C3C007C00007C00007C00007C00007C000FFFC00FFFC0007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 003FFC003FFC0013207F9F10>I<01FC3C07FFFE0F079E1E03DE3E03E03E03E03E03E03E 03E03E03E01E03C00F07800FFF0009FC001800001800001C00001FFF800FFFF007FFF81F FFFC3C007C70003EF0001EF0001EF0001E78003C78003C3F01F80FFFE001FF00171E7F93 1A>II<1C00 3E003F007F003F003E001C00000000000000000000000000FF00FF001F001F001F001F00 1F001F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B217EA00E>I107 DI< FE0FE03F80FE1FF07FC01E70F9C3E01E407D01F01E807E01F01F807E01F01F007C01F01F 007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F007C01F01F00 7C01F01F007C01F01F007C01F01F007C01F0FFE3FF8FFEFFE3FF8FFE27147D932C>II<01FF0007FFC0 1F83F03E00F83E00F87C007C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C007C 7C007C3E00F83E00F81F83F007FFC001FF0017147F931A>II<01F81807FE381F87783F01F83E01F87E00F87C00F8FC00F8FC 00F8FC00F8FC00F8FC00F8FC00F87C00F87E00F87E00F83F01F81F87F80FFEF803F8F800 00F80000F80000F80000F80000F80000F80000F80007FF0007FF181D7E931C>II<0FE63FFE701E600E E006E006F800FFC07FF83FFC1FFE03FE001FC007C007E007F006F81EFFFCC7F010147E93 15>I<01800180018003800380038007800F803F80FFFCFFFC0F800F800F800F800F800F 800F800F800F800F800F860F860F860F860F8607CC03F801F00F1D7F9C14>IIIIII<3F FFE03FFFE03C07C0380F80701F80603F00603E00607C0000F80001F80003F00003E06007 C0600F80601F80E03F00C03E01C07C03C0FFFFC0FFFFC013147F9317>I E /Fu 36 122 df<00000007FF800000000001FFFFF0000000000FFFFFFC000000003FFF FFFE00000000FFFC00FF00000001FFC0003F80000007FF00007FC000000FFE0001FFC000 001FFC0001FFE000001FF80003FFE000003FF00003FFE000003FF00003FFE000007FE000 03FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00001FFC000 007FE00000FF8000007FE000003E0000007FE00000000000007FE00000000000007FE000 00000000007FE00000000000007FE00000000000007FE00000000000007FE0003FFFF000 FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFF FFFFF000007FF00000FFF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000 007FF0003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE0 3FFFFFE03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC007FFE00FFFE00FFFF 00FFFF00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E780000780000780000F 00000F00000F00000F00001E00001E00003C00003C0000780000F80000F00001F00003E0 0007C0000F80001F00001E00000C00001124788F21>44 D<0000000003E0000000000000 00000007F000000000000000000007F00000000000000000000FF8000000000000000000 0FF80000000000000000000FF80000000000000000001FFC0000000000000000001FFC00 00000000000000003FFE0000000000000000003FFE0000000000000000007FFF00000000 00000000007FFF0000000000000000007FFF000000000000000000FFFF80000000000000 0000FFFF800000000000000001FFFFC00000000000000001FFFFC00000000000000001FF FFC00000000000000003FFFFE00000000000000003EFFFE00000000000000007EFFFF000 00000000000007CFFFF00000000000000007C7FFF0000000000000000FC7FFF800000000 0000000F83FFF8000000000000001F83FFFC000000000000001F03FFFC00000000000000 1F01FFFC000000000000003F01FFFE000000000000003E00FFFE000000000000007E00FF FF000000000000007C007FFF00000000000000FC007FFF80000000000000F8007FFF8000 0000000000F8003FFF80000000000001F8003FFFC0000000000001F0001FFFC000000000 0003F0001FFFE0000000000003E0001FFFE0000000000003E0000FFFE0000000000007E0 000FFFF0000000000007C00007FFF000000000000FC00007FFF800000000000F800007FF F800000000000F800003FFF800000000001F800003FFFC00000000001F000001FFFC0000 0000003FFFFFFFFFFE00000000003FFFFFFFFFFE00000000007FFFFFFFFFFF0000000000 7FFFFFFFFFFF00000000007FFFFFFFFFFF0000000000FC0000007FFF8000000000F80000 003FFF8000000001F80000003FFFC000000001F00000003FFFC000000001F00000001FFF C000000003F00000001FFFE000000003E00000000FFFE000000007E00000000FFFF00000 0007C00000000FFFF000000007C000000007FFF00000000FC000000007FFF80000000F80 00000003FFF80000001F8000000003FFFC0000001F0000000003FFFC0000007FC0000000 01FFFC0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFF FF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF8051487CC75A>65 DI<00 0000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E000003FFFFFFFE00FE00 000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001FFFC000003FFFE00 07FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003FFF000000001FFE00 7FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FFF00000000003FE03 FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFFC00000000000FE0F FFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF8000000000007E3F FF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF0000000000003E7F FF000000000000007FFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF000000000000007FFF000000000000007F FF000000000000007FFF000000000000007FFF8000000000003E3FFF8000000000003E3F FF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFFC000000000003E0F FFC000000000007C0FFFC000000000007C07FFE000000000007C07FFE00000000000F803 FFF00000000000F803FFF00000000001F801FFF80000000001F000FFFC0000000003E000 7FFE0000000007E0003FFF000000000FC0001FFF800000001F80000FFFC00000003F0000 07FFF0000000FE000001FFFC000001FC000000FFFF80000FF80000003FFFF8007FF00000 000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000000FFFFFE0000000 0000003FFE000000474979C756>I69 D<000000003FFE00000E000000000FFFFFC0001E000000007FFFFFF8003E0000 0003FFFFFFFE00FE0000000FFFFFFFFF81FE0000003FFFF800FFC3FE000000FFFF80000F F7FE000001FFFC000003FFFE000007FFF0000001FFFE00000FFFC00000007FFE00001FFF 800000003FFE00003FFF000000001FFE00007FFE000000000FFE0000FFFC0000000007FE 0001FFF80000000007FE0003FFF00000000003FE0003FFF00000000001FE0007FFE00000 000001FE0007FFE00000000000FE000FFFC00000000000FE000FFFC000000000007E001F FFC000000000007E001FFF8000000000007E003FFF8000000000007E003FFF8000000000 003E003FFF8000000000003E007FFF8000000000003E007FFF0000000000003E007FFF00 000000000000007FFF0000000000000000FFFF0000000000000000FFFF00000000000000 00FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF000000 0000000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FF FF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000007FFF FFFFFE7FFF0000007FFFFFFFFE7FFF0000007FFFFFFFFE7FFF8000007FFFFFFFFE7FFF80 00007FFFFFFFFE3FFF8000000000FFFE003FFF8000000000FFFE003FFF8000000000FFFE 001FFF8000000000FFFE001FFFC000000000FFFE000FFFC000000000FFFE000FFFC00000 0000FFFE0007FFE000000000FFFE0007FFE000000000FFFE0003FFF000000000FFFE0003 FFF800000000FFFE0001FFF800000000FFFE0000FFFC00000000FFFE00007FFE00000000 FFFE00003FFF00000000FFFE00001FFF80000000FFFE00000FFFE0000001FFFE000007FF F0000003FFFE000001FFFC000007FFFE000000FFFF80001FFFFE0000003FFFFC00FFC7FE 0000000FFFFFFFFF83FE00000003FFFFFFFE00FE000000007FFFFFF8003E000000000FFF FFE0000E00000000003FFE000000004F4979C75D>71 D73 D76 DI80 D82 D<3FFFFFFFFFFFFFFFFF 003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFF FFFFFFFF003FFF0003FFF8003FFF007FF80003FFF80007FF807FE00003FFF80001FF807F C00003FFF80000FF807F800003FFF800007F807F000003FFF800003F807F000003FFF800 003F807E000003FFF800001F807E000003FFF800001F807E000003FFF800000F807C0000 03FFF800000F807C000003FFF800000F807C000003FFF800000F807C000003FFF800000F 80FC000003FFF800000FC0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FF F8000007C0F8000003FFF8000007C0F8000003FFF8000007C000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF8000000000003FFFFFFFFFF F800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FF FFFFFFFFF800004A467CC553>84 D<0007FFFC000000007FFFFFC0000001FFFFFFF80000 03FFFFFFFE000007FE001FFF000007FF0003FFC0000FFF8001FFE0000FFF8000FFF0000F FF80007FF0000FFF80007FF8000FFF80007FF80007FF00003FFC0007FF00003FFC0003FE 00003FFC0000F800003FFC00000000003FFC00000000003FFC00000000003FFC00000000 003FFC00000007FFFFFC000000FFFFFFFC000007FFFFFFFC00003FFFE03FFC0000FFFE00 3FFC0003FFF0003FFC0007FFC0003FFC000FFF00003FFC001FFE00003FFC003FFC00003F FC007FF800003FFC007FF800003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC 00FFF000003FFC00FFF000003FFC00FFF000007FFC007FF80000FFFC007FF80001EFFC00 3FFC0003EFFC003FFF0007CFFF000FFFC03F8FFFF807FFFFFF07FFFC01FFFFFC03FFFC00 7FFFF001FFFC0003FF80007FF8362E7DAD3A>97 D<007FC00000000000FFFFC000000000 00FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC00FFC00000001FFC07FFFC0000001FFC3FFFFF000 0001FFCFFFFFFC000001FFDFF00FFF000001FFFF8003FF800001FFFE0001FFC00001FFF8 00007FE00001FFF000007FF00001FFE000003FF80001FFE000001FFC0001FFE000001FFC 0001FFE000001FFE0001FFE000001FFE0001FFE000000FFF0001FFE000000FFF0001FFE0 00000FFF0001FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF 8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE0 00000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000000FFF 0001FFE000000FFF0001FFE000001FFE0001FFE000001FFE0001FFE000001FFC0001FFE0 00001FFC0001FFE000003FF80001FFF000003FF80001FFF800007FF00001FFFC0000FFE0 0001FFFE0001FFC00001FFBF0007FF800001FF1FE01FFE000001FE0FFFFFFC000001FC03 FFFFF0000001F800FFFF80000001F0001FF800000039487CC742>I<00001FFFC0000000 FFFFF8000007FFFFFE00001FFFFFFF80007FFC00FFC000FFE001FFC001FFC003FFE003FF 8003FFE007FF0003FFE00FFE0003FFE00FFE0003FFE01FFC0001FFC01FFC0001FFC03FFC 0000FF803FFC00003E007FF8000000007FF8000000007FF800000000FFF800000000FFF8 00000000FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF8 00000000FFF800000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC 000000003FFC000000001FFC000000F81FFE000000F80FFE000000F80FFF000001F007FF 800003F003FFC00007E001FFE0000FC000FFF0001F80007FFE00FF00001FFFFFFE000007 FFFFF8000000FFFFE00000001FFE00002D2E7CAD35>I<00000000007FC00000000000FF FFC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000 00000003FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC00000000FFC01FFC0000000FFFF81FFC0000007FFFFE1 FFC000001FFFFFF9FFC000007FFC03FFFFC00000FFF0007FFFC00001FFC0001FFFC00003 FF80000FFFC00007FF000007FFC0000FFE000003FFC0000FFE000003FFC0001FFC000003 FFC0001FFC000003FFC0003FFC000003FFC0003FFC000003FFC0007FF8000003FFC0007F F8000003FFC0007FF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003 FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FF F8000003FFC000FFF8000003FFC000FFF8000003FFC0007FF8000003FFC0007FF8000003 FFC0007FF8000003FFC0003FF8000003FFC0003FFC000003FFC0003FFC000003FFC0001F FC000003FFC0001FFC000003FFC0000FFE000007FFC00007FF00000FFFC00003FF00001F FFC00001FFC0003FFFC00000FFE000FFFFE000007FF807FBFFFF80001FFFFFF3FFFF8000 07FFFFC3FFFF800001FFFF03FFFF8000001FF803FFFF8039487CC742>I<00001FFE0000 0001FFFFE0000007FFFFF800001FFFFFFE00007FFC07FF0000FFE001FF8001FFC0007FC0 03FF80003FE007FF00003FF00FFE00001FF01FFE00000FF81FFC00000FF83FFC00000FFC 3FFC000007FC7FFC000007FC7FF8000007FC7FF8000007FE7FF8000007FEFFF8000007FE FFF8000007FEFFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFCFFF800000000 FFF800000000FFF800000000FFF8000000007FF8000000007FF8000000007FFC00000000 3FFC000000003FFC000000003FFC0000001C1FFE0000003E0FFE0000003E07FF0000007E 07FF000000FC03FF800001F801FFC00003F0007FF0001FE0003FFE00FFC0001FFFFFFF80 0007FFFFFE000000FFFFF80000000FFF80002F2E7DAD36>I<00000000001F8000007FF0 00FFE00007FFFF03FFF0001FFFFFC7FFF0007FFFFFFFC7F800FFE03FFE0FF801FF800FFC 0FF803FF0007FE0FF807FE0003FF07F007FE0003FF07F00FFC0001FF81C00FFC0001FF80 000FFC0001FF80001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC000 1FFC0001FFC0001FFC0001FFC0000FFC0001FF80000FFC0001FF80000FFC0001FF800007 FE0003FF000007FE0003FF000003FF0007FE000001FF800FFC000000FFE03FF8000001FF FFFFF0000001DFFFFFC0000003C7FFFF00000003C07FF000000007C0000000000007C000 0000000007C0000000000007C0000000000007E0000000000007F0000000000007F80000 00000007FFFFFFF0000007FFFFFFFF000003FFFFFFFFE00003FFFFFFFFF80001FFFFFFFF FE0001FFFFFFFFFF0000FFFFFFFFFF80007FFFFFFFFF8003FFFFFFFFFFC00FFFFFFFFFFF C01FF800001FFFE03FE0000001FFE07FC00000007FF07FC00000003FF0FF800000001FF0 FF800000001FF0FF800000001FF0FF800000001FF0FF800000001FF07FC00000003FE07F C00000003FE03FE00000007FC03FF0000000FFC01FFC000003FF800FFF00000FFF0003FF F000FFFC0000FFFFFFFFF000003FFFFFFFC0000007FFFFFE000000003FFFC0000035447D AE3B>103 D<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC0000000 0000FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001FF C0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000 000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FF C0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000 000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FF C001FFC0000001FFC00FFFF8000001FFC03FFFFE000001FFC0FFFFFF000001FFC1FC07FF 800001FFC3E003FFC00001FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FF DC0000FFE00001FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FF F00001FFF00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFF FFC07FFFFFE03B487CC742>I<00FC0001FF0003FF8007FFC00FFFC01FFFE01FFFE01FFF E01FFFE01FFFE01FFFE00FFFC007FFC003FF8001FF0000FC000000000000000000000000 00000000000000000000000000000000000000000000007FC0FFFFC0FFFFC0FFFFC0FFFF C0FFFFC003FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF18497CC820>I<007FC000000000FFFFC0000000 00FFFFC000000000FFFFC000000000FFFFC000000000FFFFC00000000003FFC000000000 01FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001 FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FF C00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC0 0000000001FFC00000000001FFC00000000001FFC00000000001FFC0007FFFF801FFC000 7FFFF801FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0000FFE0001FFC00007 F00001FFC0000FE00001FFC0003FC00001FFC0007F800001FFC000FE000001FFC001FC00 0001FFC007F8000001FFC00FF0000001FFC01FC0000001FFC03F80000001FFC0FF000000 01FFC1FE00000001FFC3FF00000001FFCFFF80000001FFDFFFC0000001FFFFFFC0000001 FFFFFFE0000001FFFFFFF0000001FFFCFFF8000001FFF87FFC000001FFE03FFC000001FF C01FFE000001FFC01FFF000001FFC00FFF800001FFC007FFC00001FFC003FFC00001FFC0 01FFE00001FFC001FFF00001FFC000FFF80001FFC0007FFC0001FFC0003FFC0001FFC000 1FFE0001FFC0000FFF0001FFC0000FFF8001FFC0000FFFC0FFFFFF807FFFFFFFFFFF807F FFFFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FFFFC000FFFFC000FFFFC00003FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC000FFFFFF80FFFFFF80FFFFFF80FFFFFF80FFFFFF 8019487CC720>I<007FC001FFC00000FFE00000FFFFC00FFFF80007FFFC0000FFFFC03F FFFE001FFFFF0000FFFFC0FFFFFF007FFFFF8000FFFFC1FC07FF80FE03FFC000FFFFC3E0 03FFC1F001FFE00003FFC7C001FFC3E000FFE00001FFCF0001FFE78000FFF00001FFDE00 00FFEF00007FF00001FFDC0000FFEE00007FF00001FFFC0000FFFE00007FF80001FFF800 00FFFC00007FF80001FFF00000FFF800007FF80001FFF00000FFF800007FF80001FFF000 00FFF800007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800FFFFFFC0 7FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC0 7FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF05C2E7CAD63>I<007FC001FFC00000FF FFC00FFFF80000FFFFC03FFFFE0000FFFFC0FFFFFF0000FFFFC1FC07FF8000FFFFC3E003 FFC00003FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001 FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF00001FFF00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFFC07F FFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B 2E7CAD42>I<00000FFF0000000000FFFFF000000007FFFFFE0000001FFFFFFF8000003F FC03FFC00000FFE0007FF00001FF80001FF80003FF00000FFC0007FE000007FE000FFE00 0007FF000FFC000003FF001FFC000003FF803FFC000003FFC03FF8000001FFC03FF80000 01FFC07FF8000001FFE07FF8000001FFE07FF8000001FFE0FFF8000001FFF0FFF8000001 FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FF F0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF07FF8000001FFE07FF8000001FFE0 7FF8000001FFE07FF8000001FFE03FFC000003FFC03FFC000003FFC01FFC000003FF801F FE000007FF800FFE000007FF0007FF00000FFE0003FF80001FFC0001FFC0003FF80000FF E0007FF000007FFC03FFE000001FFFFFFF80000007FFFFFE00000000FFFFF0000000000F FF000000342E7DAD3B>I<007FC00FFC000000FFFFC07FFFC00000FFFFC3FFFFF00000FF FFCFFFFFFC0000FFFFDFF01FFF0000FFFFFF8007FF800003FFFE0001FFC00001FFF80000 FFE00001FFF00000FFF00001FFE000007FF80001FFE000003FFC0001FFE000003FFC0001 FFE000003FFE0001FFE000001FFE0001FFE000001FFF0001FFE000001FFF0001FFE00000 1FFF0001FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000001FFF0001 FFE000001FFF0001FFE000001FFE0001FFE000001FFE0001FFE000003FFC0001FFE00000 3FFC0001FFE000007FF80001FFF000007FF80001FFF80000FFF00001FFFC0001FFE00001 FFFE0003FFC00001FFFF0007FF800001FFFFE03FFE000001FFEFFFFFFC000001FFE3FFFF F0000001FFE0FFFF80000001FFE01FF800000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE00000 00000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE00000000000FFFFFFC000 000000FFFFFFC000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC00000000039 427CAD42>I<00000FFC0003C0000000FFFF0007C0000007FFFFC00FC000001FFFFFF01F C000003FFE03F81FC00000FFF000FC3FC00001FFE0003E7FC00003FFC0001F7FC00007FF 80001FFFC0000FFF00000FFFC0000FFF000007FFC0001FFE000007FFC0001FFE000003FF C0003FFC000003FFC0003FFC000001FFC0007FFC000001FFC0007FFC000001FFC0007FF8 000001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FF C000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF8 000001FFC000FFF8000001FFC0007FF8000001FFC0007FFC000001FFC0007FFC000001FF C0003FFC000001FFC0003FFC000001FFC0003FFE000003FFC0001FFE000003FFC0001FFE 000007FFC0000FFF00000FFFC00007FF80001FFFC00003FF80003FFFC00001FFC0007FFF C00000FFF000FFFFC000007FFC07FBFFC000001FFFFFE3FFC0000007FFFFC3FFC0000001 FFFF03FFC00000001FF803FFC0000000000003FFC0000000000003FFC0000000000003FF C0000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC0000000 000003FFC0000000000003FFC0000000000003FFC0000000000003FFC0000000000003FF C0000000000003FFC0000000000003FFC0000000000003FFC00000000001FFFFFF800000 0001FFFFFF8000000001FFFFFF8000000001FFFFFF8000000001FFFFFF8039427CAD3F> I<00FF803F8000FFFF80FFF000FFFF83FFFC00FFFF87FFFE00FFFF8FC3FF00FFFF8F07FF 0003FF9E0FFF8001FFBC0FFF8001FFB80FFF8001FFF80FFF8001FFF00FFF8001FFF007FF 0001FFF007FF0001FFE003FE0001FFE000F80001FFE000000001FFE000000001FFC00000 0001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC00000 0001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC00000 0001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC00000 0001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000FFFFFFE000 00FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000292E7CAD31>I<000FFF00 E0007FFFF3E001FFFFFFE007FFFFFFE00FF800FFE01FC0001FE03F80000FE03F000007E0 7F000003E07F000003E0FF000003E0FF000003E0FF800003E0FFC0000000FFF0000000FF FE000000FFFFF800007FFFFFC0007FFFFFF0003FFFFFFC001FFFFFFF000FFFFFFF8007FF FFFFC003FFFFFFE000FFFFFFF0003FFFFFF00003FFFFF800001FFFF8000000FFFC000000 1FFC7800000FFCF8000007FCF8000003FCFC000003FCFC000003FCFE000003F8FE000003 F8FF000003F8FF800007F0FFC0000FF0FFF0001FE0FFFC00FFC0FFFFFFFF80FC7FFFFE00 F81FFFF800E003FF8000262E7CAD2F>I<0001F000000001F000000001F000000001F000 000001F000000001F000000003F000000003F000000003F000000007F000000007F00000 0007F00000000FF00000000FF00000001FF00000003FF00000003FF00000007FF0000001 FFF0000003FFF000000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0 000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF000 0000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF00000 00FFF0000000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000 FFF001F000FFF001F000FFF001F0007FF001E0007FF803E0003FF803E0003FFC07C0001F FE0F80000FFFFF800007FFFE000001FFFC0000001FF00024427EC12E>I<007FE000003F F000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFF E0007FFFF00003FFE00001FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FF F00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FF E00000FFF00001FFE00001FFF00001FFE00001FFF00001FFE00001FFF00001FFE00003FF F00000FFE00007FFF00000FFE0000F7FF000007FE0001F7FF000007FF0003E7FF800003F FC00FC7FFFE0001FFFFFF87FFFE00007FFFFE07FFFE00001FFFF807FFFE000003FFE007F FFE03B2E7CAD42>III<7FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF 007FFFFF801FFFFF007FFFFF801FFFFF00007FF80001FF0000007FFC0001FC0000003FFE 0003F80000001FFF0007F00000000FFF0007E000000007FF800FC000000003FFC01F8000 000003FFE03F8000000001FFF07F0000000000FFF8FE00000000007FF9FC00000000003F FFF800000000003FFFF000000000001FFFE000000000000FFFC0000000000007FFC00000 00000003FFC0000000000001FFE0000000000001FFF0000000000001FFF8000000000003 FFFC000000000003FFFE000000000007FFFE00000000000FEFFF00000000001FCFFF8000 0000003F87FFC0000000007F03FFE000000000FE01FFE000000001FC00FFF000000001F8 007FF800000003F0007FFC00000007F0003FFE0000000FE0001FFF0000001FC0000FFF00 00003F800007FF800000FF800007FFC000FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8 003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC03A2E7EAD3F>I<7FFFFFC000FFFF80 7FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF8000FFF000 000FE00000FFF800000FC00000FFF800000FC000007FFC00000F8000007FFC00001F8000 003FFC00001F0000003FFE00003F0000001FFE00003E0000001FFF00007E0000000FFF00 007C0000000FFF8000FC00000007FF8000F800000007FFC001F800000003FFC001F00000 0003FFE003F000000003FFE003F000000001FFF003E000000001FFF007E000000000FFF0 07C000000000FFF80FC0000000007FF80F80000000007FFC1F80000000003FFC1F000000 00003FFE3F00000000001FFE3E00000000001FFF7E00000000000FFF7C00000000000FFF FC00000000000FFFFC000000000007FFF8000000000007FFF8000000000003FFF0000000 000003FFF0000000000001FFE0000000000001FFE0000000000000FFC0000000000000FF C00000000000007F800000000000007F800000000000003F000000000000003F00000000 0000003F000000000000003E000000000000007E000000000000007C00000000000000FC 000000001F8000F8000000003FC001F8000000007FE001F000000000FFF003F000000000 FFF003E000000000FFF007E000000000FFF00FC000000000FFF01F8000000000FFF03F80 000000007FE07F00000000007F43FE00000000003FFFF800000000001FFFF00000000000 07FFC0000000000001FE00000000000039427EAD3F>I E /Fv 47 122 df<0000C018000000C018000000C018000001803000000180300000018030000001 80300000030060000003006000000300600000030060000003006000000600C000000600 C000000600C000000600C000000C018000FFFFFFFFC0FFFFFFFFC0001803000000180300 00001803000000180300000030060000003006000000300600000030060000FFFFFFFFC0 FFFFFFFFC000600C000000C018000000C018000000C018000000C0180000018030000001 803000000180300000018030000003006000000300600000030060000003006000000600 C000000600C000000600C00000222D7DA229>35 D<70F8FCFC7404040404080810102040 060F7C840E>44 DI<70F8F8F87005057C840E>I<01F000071C00 0C06001803003803803803807001C07001C07001C07001C0F001E0F001E0F001E0F001E0 F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001C07001C0 7001C07803C03803803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038003800380038003800380038007C0FFFE0F217CA018 >I<03F0000C1C001007002007804003C04003C08003E0F003E0F801E0F801E0F801E020 03E00003E00003C00003C0000780000700000E00001C0000180000300000600000C00001 80000100000200200400200800201800603000403FFFC07FFFC0FFFFC013217EA018>I< 03F8000C1E001007002007804007C07807C07803C07807C03807C0000780000780000700 000F00000E0000380003F000001C00000F000007800007800003C00003C00003E02003E0 7003E0F803E0F803E0F003C04003C0400780200780100F000C1C0003F00013227EA018> I<000200000600000E00000E00001E00001E00002E00004E00004E00008E00008E00010E 00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E00800E00FFFF F8000E00000E00000E00000E00000E00000E00000E00001F0001FFF015217FA018>I<10 00801E07001FFF001FFE001FF80013E00010000010000010000010000010000010000010 F800130E001407001803801003800001C00001C00001E00001E00001E00001E07001E0F0 01E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013227EA018>I< 007E0001C1000300800601C00E03C01C03C0180180380000380000780000700000700000 F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0F001E0F001E0 7001E07001E07001E03801C03801C01803801C03000C0600070C0001F00013227EA018> I<01F800060E000803001001802001802000C06000C06000C06000C07000C07801803E01 003F02001FC4000FF80003F80003FC00067F00083F80100F803007C06001C06000E0C000 E0C00060C00060C00060C000606000406000C03000801803000E0E0003F00013227EA018 >56 D<01F000060C000C0600180700380380700380700380F001C0F001C0F001C0F001E0 F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E003E1E00001C0 0001C00001C0000380000380300300780700780600700C002018001030000FC00013227E A018>I<0001800000018000000180000003C0000003C0000003C0000005E0000005E000 000DF0000008F0000008F0000010F800001078000010780000203C0000203C0000203C00 00401E0000401E0000401E0000800F0000800F0000FFFF000100078001000780030007C0 020003C0020003C0040003E0040001E0040001E00C0000F00C0000F03E0001F8FF800FFF 20237EA225>65 D<0007E0100038183000E0063001C00170038000F0070000F00E000070 1E0000701C0000303C0000303C0000307C0000107800001078000010F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F800000078000000780000107C000010 3C0000103C0000101C0000201E0000200E000040070000400380008001C0010000E00200 00381C000007E0001C247DA223>67 D69 DI73 D77 DI80 D82 D<03F0200C0C601802603001E07000E06000 60E00060E00060E00020E00020E00020F00000F000007800007F00003FF0001FFE000FFF 0003FF80003FC00007E00001E00000F00000F0000070800070800070800070800070C000 60C00060E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF8780780786007 8018400780084007800840078008C007800C800780048007800480078004800780040007 800000078000000780000007800000078000000780000007800000078000000780000007 800000078000000780000007800000078000000780000007800000078000000780000007 80000007800000078000000FC00003FFFF001E227EA123>I<0FE0001838003C0C003C0E 0018070000070000070000070000FF0007C7001E07003C0700780700700700F00708F007 08F00708F00F087817083C23900FC1E015157E9418>97 D<0E0000FE00001E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F000E61C0 0E80600F00300E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E 0E001C0E003C0E00380F00700C80600C41C0083F0017237FA21B>I<01FE000703000C07 801C0780380300780000700000F00000F00000F00000F00000F00000F00000F000007000 007800403800401C00800C010007060001F80012157E9416>I<0000E0000FE00001E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E001F8E007 04E00C02E01C01E03800E07800E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F0 00E07000E07800E03800E01801E00C02E0070CF001F0FE17237EA21B>I<01FC00070700 0C03801C01C03801C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000F00000 7000007800203800201C00400E008007030000FC0013157F9416>I<003C00C6018F038F 030F070007000700070007000700070007000700FFF80700070007000700070007000700 0700070007000700070007000700070007000700070007807FF8102380A20F>I<000070 01F198071E180E0E181C07001C07003C07803C07803C07803C07801C07001C07000E0E00 0F1C0019F0001000001000001800001800001FFE000FFFC00FFFE03800F0600030400018 C00018C00018C000186000306000303800E00E038003FE0015217F9518>I<0E0000FE00 001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E1F800E60C00E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C 001E003E001E001C00000000000000000000000000000000000E00FE001E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A227FA10E >I<01C003E003E003E001C00000000000000000000000000000000001E00FE001E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F18061803E000B2C82A10F>I<0E0000FE00001E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E03FC0E01 F00E01C00E01800E02000E04000E08000E10000E38000EF8000F1C000E1E000E0E000E07 000E07800E03C00E01C00E01E00E00F00E00F8FFE3FE17237FA21A>I<0E00FE001E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B237FA20E>I<0E1F C07F00FE60E183801E807201C00F003C00E00F003C00E00E003800E00E003800E00E0038 00E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800 E00E003800E00E003800E00E003800E00E003800E00E003800E0FFE3FF8FFE27157F942A >I<0E1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F941B>I< 01FC000707000C01801800C03800E0700070700070F00078F00078F00078F00078F00078 F00078F000787000707800F03800E01C01C00E038007070001FC0015157F9418>I<0E1F 00FE61C00E80600F00700E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E00 1E0E001E0E003C0E003C0E00380F00700E80E00E41C00E3F000E00000E00000E00000E00 000E00000E00000E00000E00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F06 0F000E000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF010157F94 13>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C80 0CC00CC008E018D0308FC00E157E9413>I<02000200020002000600060006000E001E00 3E00FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E04 0E040708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E0070 0E00700E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F0060170 03827800FC7F18157F941B>III121 D E /Fw 20 118 df45 D68 D73 D77 D80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E0780000E0 700000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F8000000 7C0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC0000FFFF00 000FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F8000000F8 8000007880000078800000788000007880000078C0000078C0000070E00000F0E00000E0 F00000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E003E001E 0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E001F001E 003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E0478003E 047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700 040F00041E001E1C003E3C003E7C003E7C0008780000F80000F80000F80000F80000F800 00F80000F80000F80000F800007800007C00007C00003C00011E00011E00020F00020700 0403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E00000 01E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E00000 01E0000001E0000001E0000001E0000001E0000001E0001F81E000F061E001C019E00780 05E00F0003E00E0003E01E0001E03C0001E03C0001E07C0001E0780001E0F80001E0F800 01E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0780001E07800 01E03C0001E03C0001E01C0001E01E0003E00E0005E0070009E0038011F000E061FF003F 81FF20327DB125>I<003F800000E0E0000380380007003C000E001E001E001E001C000F 003C000F007C000F0078000F8078000780F8000780F8000780FFFFFF80F8000000F80000 00F8000000F8000000F8000000F8000000780000007C0000003C0000003C0000801E0000 800E0001000F0002000780020001C00C0000F03000001FC000191F7E9E1D>I<0007E000 1C1000383800707C00E07C01E07C01C03803C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C000FFFFC0FFFFC003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00007E0007FFF007F FF0016327FB114>I<000000F0007F030801C1C41C0380E81C070070080F0078001E003C 001E003C003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E003C 000F007800070070000780E00009C1C000087F0000180000001800000018000000180000 00180000001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E0300000 F070000070E0000038E0000038E0000038E0000038E00000387000007070000070380000 E01C0001C00700070001C01C00003FE0001E2F7E9F21>I<07000F801F801F800F800700 000000000000000000000000000000000000000000000780FF80FF800F80078007800780 078007800780078007800780078007800780078007800780078007800780078007800780 0780078007800FC0FFF8FFF80D307EAF12>105 D<0780FE001FC000FF83078060F000FF 8C03C18078000F9001E2003C0007A001E4003C0007A000F4001E0007C000F8001E0007C0 00F8001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000 F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0 001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F000 1E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E 000FC001F8003F00FFFC1FFF83FFF0FFFC1FFF83FFF0341F7E9E38>109 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C000F000 07C000F000078000F000078000F000078000F000078000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F0000FC001 F800FFFC1FFF80FFFC1FFF80211F7E9E25>I<001FC00000F0780001C01C00070007000F 0007801E0003C01C0001C03C0001E03C0001E0780000F0780000F0780000F0F80000F8F8 0000F8F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8780000F07C0001F03C 0001E03C0001E01E0003C01E0003C00F00078007800F0001C01C0000F07800001FC0001D 1F7E9E21>I<0783E0FF8C18FF907C0F907C07A07C07C03807C00007C00007C000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0780000780000780000780000780000780000FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E00010E00010F00010F800007E00 003FF0001FFF000FFFC003FFE0003FF00001F80000F880003C80003C80001CC0001CC000 1CE0001CE00018F00038F00030CC0060C301C080FE00161F7E9E1A>I<00400000400000 400000400000400000C00000C00000C00001C00001C00003C00007C0000FC0001FFFE0FF FFE003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C01003C01003C01003C01003C01003C01003C01003 C01001C02001E02000E0400078C0001F00142C7FAB19>I<078000F000FF801FF000FF80 1FF0000F8001F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078001F000 078001F000078001F000038002F00003C004F00001C008F800007030FF80001FC0FF8021 1F7E9E25>I E /Fx 5 85 df<00000000600000000000700000000000F00000000001F0 0000000001F00000000003F00000000003F00000000007F00000000007F0000000000FF0 000000000FF0000000001BF00000000033F00000000033F00000000063F00000000063F8 00000000C1F800000000C1F80000000181F80000000381F80000000301F80000000601F8 0000000601F80000000C01F80000000C01F80000001801F80000001801F80000003001F8 0000006001F80000006001F8000000C001F8000000C001FC000001FFFFFC000001FFFFFC 0000030000FC0000070000FC0000060000FC00000C0000FC00000C0000FC0000180000FC 0000180000FC0000300000FC0000700000FC0000600000FC0000E00000FC0001E00000FC 0003E00000FE000FF00001FE00FFFE003FFFF0FFFE003FFFF02C327CB135>65 D<000FFFFFFF0000000FFFFFFFC00000003F8007F00000003F8001F80000003F00007C00 00003F00007E0000007F00003E0000007F00001F0000007E00001F0000007E00001F8000 00FE00000F800000FE00000F800000FC00000FC00000FC00000FC00001FC00000FC00001 FC00000FC00001F800000FC00001F800000FC00003F800000FC00003F800001FC00003F0 00001FC00003F000001FC00007F000001FC00007F000001F800007E000003F800007E000 003F80000FE000003F80000FE000003F00000FC000007F00000FC000007F00001FC00000 7E00001FC00000FE00001F800000FC00001F800000FC00003F800001F800003F800001F0 00003F000003F000003F000007E000007F000007C000007F00000FC000007E00001F8000 007E00003F000000FE00007E000000FE0000F8000000FC0001F0000000FC0007E0000001 FC003F800000FFFFFFFE000000FFFFFFF000000032317CB036>68 D<000FFFFFFFFE000FFFFFFFFE00003F8000FE00003F80003E00003F00001E00003F0000 1E00007F00000C00007F00000C00007E00000C00007E00000C0000FE00000C0000FE0000 0C0000FC00000C0000FC00000C0001FC00001C0001FC00C0180001F800C0000001F800C0 000003F801C0000003F801C0000003F00180000003F00380000007F00F80000007FFFF80 000007FFFF00000007E00F0000000FE0070000000FE0070000000FC0060000000FC00600 00001FC00E0000001FC00E0000001F800C0000001F80000000003F80000000003F800000 00003F00000000003F00000000007F00000000007F00000000007E00000000007E000000 0000FE0000000000FE0000000000FC0000000000FC0000000001FC00000000FFFFFC0000 00FFFFFC0000002F317CB02F>70 D<000FFFFFF800000FFFFFFF0000003F801FC000003F 8007E000003F0003F000003F0001F800007F0000FC00007F0000FC00007E0000FC00007E 0000FC0000FE0000FC0000FE0001FC0000FC0001FC0000FC0001FC0001FC0001F80001FC 0003F80001F80003F00001F80007E00003F80007E00003F8000F800003F0003F000003F0 007E000007F003F8000007FFFFE0000007FFFF80000007E007C000000FE003F000000FE0 01F000000FC000F800000FC000F800001FC000FC00001FC000FC00001F8000FC00001F80 00FC00003F8001FC00003F8001FC00003F0001FC00003F0001FC00007F0003F800007F00 03F800007E0003F800007E0003F80600FE0003F80E00FE0003F80C00FC0003F80C00FC00 03F81C01FC0001F838FFFFF000FC70FFFFF0007FE0000000001F802F327CB034>82 D<07FFFFFFFFF807FFFFFFFFF80FE007F001F80F8007F000F80E0007E000701E0007E000 701C000FE0007018000FE0007038000FC0007038000FC0007030001FC0006070001FC000 6060001F80006060001F80006060003F8000E0E0003F8000C000003F00000000003F0000 0000007F00000000007F00000000007E00000000007E0000000000FE0000000000FE0000 000000FC0000000000FC0000000001FC0000000001FC0000000001F80000000001F80000 000003F80000000003F80000000003F00000000003F00000000007F00000000007F00000 000007E00000000007E0000000000FE0000000000FE0000000000FC0000000000FC00000 00001FC0000000001FC0000000001F80000000003F80000000007FC00000007FFFFFC000 007FFFFFC000002D3174B033>84 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 795 908 a Fx(D)26 b(R)g(A)f(F)h(T)225 999 y Fw(Do)r(cumen)n(t) 20 b(for)i(a)f(Standard)g(Message-P)n(assing)f(In)n(terface)621 1194 y Fv(Message)c(P)o(assing)h(In)o(terface)e(F)l(orum)832 1320 y(Ma)o(y)h(28,)g(1993)87 1378 y(This)g(w)o(ork)g(w)o(as)h(supp)q (orted)g(b)o(y)f(ARP)l(A)g(and)g(NSF)g(under)g(con)o(tract)g(n)o(um)o (b)q(er)f(###,)g(b)o(y)g(the)192 1436 y(National)h(Science)f(F)l (oundation)i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q (erativ)o(e)76 1494 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o (y)e(the)h(Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1552 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 75 378 a Fu(Con)m(ten)m(ts)75 648 y Ft(Ac)o(kno)o(wledgmen)o (ts)1362 b(1)75 752 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(2)143 810 y Fs(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(2)143 868 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(3)143 926 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(3)143 983 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(3)143 1041 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b Fr(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b Fs(4)75 1145 y Ft(2)42 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)g(Comm)o(unication)1001 b(5)143 1203 y Fs(2.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(5)143 1260 y(2.2)46 b(Data)14 b(T)o(yp)q(es)30 b Fr(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(5)248 1318 y(2.2.1)50 b(Handle)14 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(5)248 1376 y(2.2.2)50 b(Arra)o(y)14 b(of)h(handles)35 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(6)248 1434 y(2.2.3)50 b(State)14 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(7)248 1491 y(2.2.4)50 b(Named)15 b(constan)o(ts)23 b Fr(:)f(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)91 b Fs(7)248 1549 y(2.2.5)50 b(Choice)20 b Fr(:)i(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)91 b Fs(7)143 1607 y(2.3)46 b(Pro)q(cesses)40 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b Fs(7)248 1665 y(2.3.1)50 b(Error)14 b(Handling)29 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(8)143 1722 y(2.4)46 b(Messages)f Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(9)248 1780 y(2.4.1)50 b(Data)19 b Fr(:)j(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(9)248 1838 y(2.4.2)50 b(En)o(v)o(elop)q(e)45 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(9)143 1895 y(2.5)46 b(Comm)o(unication)15 b(Bu\013ers)41 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(10)248 1953 y(2.5.1)50 b(Comm)o(unication)16 b(bu\013er)f(comp)q(onen)o(ts)24 b Fr(:)e(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fs(10)248 2011 y(2.5.2)50 b(Bu\013er)15 b(op)q(erations)22 b Fr(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(13)248 2069 y(2.5.3)50 b(T)o(yp)q(e)16 b(Matc)o(hing)f(and)g(Data) f(Con)o(v)o(ersion)g Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(17)248 2126 y(2.5.4)50 b(Data)14 b(con)o(v)o(ersion)46 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b Fs(18)143 2184 y(2.6)46 b(Receiv)o(e)17 b(Criteria)d Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b Fs(20)143 2242 y(2.7)46 b(Comm)o(unication)15 b(Mo)q(de)35 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(21)143 2300 y(2.8)46 b(Comm)o(unication)15 b(Ob)s(jects)28 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(22)248 2357 y(2.8.1)50 b(Comm)o(unication)16 b(Ob)s(ject)f (Creation)31 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(23)248 2415 y(2.8.2)50 b(Comm)o(unication)16 b(Start)39 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)69 b Fs(24)248 2473 y(2.8.3)50 b(Comm)o(unication)16 b(Completion)i Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(24)248 2530 y(2.8.4)50 b(Multiple)17 b(Completions)41 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(26)143 2588 y(2.9)46 b(Blo)q(c)o(king)16 b(Comm)o(unication)44 b Fr(:)23 b(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b Fs(28)143 2646 y(2.10)23 b(Non)o(blo)q(c)o (king)16 b(Comm)o(unication)41 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(29)143 2704 y(2.11)23 b(Con)o(tiguous)15 b(Blo)q(c)o(k)h(Comm)o (unication)f(Op)q(erations)21 b Fr(:)i(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(30)969 2828 y(i)p eop %%Page: 2 3 2 2 bop 75 -100 a Fs(ii)1518 b Fq(CONTENTS)143 45 y Fs(2.12)23 b(Prob)q(e)15 b(and)g(Cancel)g Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(31)143 102 y(2.13)23 b(Correctness)32 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(33)248 159 y(2.13.1)27 b(Order)37 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)69 b Fs(34)248 216 y(2.13.2)27 b(Progress)14 b(and)i(F)l(airness)39 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(34)143 273 y(2.14)23 b(Missing)42 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(36)75 376 y Ft(3)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(37)143 433 y Fs(3.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(37)143 490 y(3.2)46 b(Group)15 b(F)l(unctions)35 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(38)143 547 y(3.3)46 b(Comm)o(unication)15 b(F)l(unctions)24 b Fr(:)f(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(39)143 604 y(3.4)46 b(Sync)o(hronization)16 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(39)143 661 y(3.5)46 b(Data)14 b(mo)o(v)o(e)g(functions)29 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(39)143 718 y(3.6)46 b(Global)16 b(Compute)e(Op)q(erations)20 b Fr(:)i(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(47)143 775 y(3.7)46 b(Correctness)32 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(51)75 878 y Ft(4)42 b(Pro)q(cess)17 b(T)l(op)q(ologies)1250 b(53)143 935 y Fs(4.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(53)143 992 y(4.2)46 b(Virtual)16 b(T)l(op)q(ologies)37 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(54)143 1049 y(4.3)46 b(Em)o(b)q(edding)16 b(in)g(MPI)22 b Fr(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(55)143 1106 y(4.4)46 b(Ov)o(erview)16 b(of)e(the)i(prop)q(osed)f (MPI)g(functions)20 b Fr(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(55)248 1163 y(4.4.1)50 b(Lo)o(w{lev)o(el)16 b(top)q(ology)f(functions)37 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b Fs(56)248 1220 y(4.4.2)50 b(High{lev)o(el)17 b(top)q(ology)e(functions)25 b Fr(:)d(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(57)248 1277 y(4.4.3)50 b(T)l(op)q(ology)15 b(inquiry)i(functions) 42 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(59)248 1334 y(4.4.4)50 b(P)o(artitioning)16 b(of)e(cartesian)i(structures)22 b Fr(:)g(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fs(60)143 1391 y(4.5)46 b(A)15 b(simple)i(example)f Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(62)75 1494 y Ft(5)42 b(Language)19 b(Binding)1268 b(64)75 1597 y(6)42 b(Correctness)1410 b(65)248 1654 y Fs(6.0.1)50 b(Order)37 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(65)248 1711 y(6.0.2)50 b(Progress)14 b(and)i(F)l(airness)39 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)69 b Fs(65)75 1814 y Ft(7)42 b(En)o(vironmen)o(tal)18 b(Inquiry)1156 b(68)75 1917 y(8)42 b(Pro\014ling)1484 b(69)143 1974 y Fs(8.1)46 b(Requiremen)o(ts)28 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(69)143 2031 y(8.2)46 b(Discussion)21 b Fr(:)i(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(69)143 2088 y(8.3)46 b(Logic)16 b(of)e(the)i(design)29 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(70)143 2145 y(8.4)46 b(Examples)36 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(70)248 2202 y(8.4.1)50 b(Pro\014ler)16 b(implemen)o(tation)35 b Fr(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(70)248 2259 y(8.4.2)50 b(MPI)15 b(library)h(implemen)o(tation)h Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)69 b Fs(70)143 2316 y(8.5)46 b(Multiple)17 b(lev)o(els)f(of)f(in)o(terception)25 b Fr(:)d(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fs(72)75 2419 y Ft(9)42 b(Maps,)17 b(Groups)g(and)h(Con)o (texts)1032 b(73)143 2476 y Fs(9.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fs(73)143 2533 y(9.2)46 b(Maps)13 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)69 b Fs(74)248 2590 y(9.2.1)50 b(Op)q(erations)16 b(on)f(maps)35 b Fr(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(75)248 2647 y(9.2.2)50 b(Map)15 b(constructors)h Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(76)143 2704 y(9.3)46 b(Con)o(text)p 411 2704 14 2 v 15 w(id)15 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(77)p eop %%Page: 3 4 3 3 bop 75 -100 a Fq(CONTENTS)1504 b Fs(iii)248 45 y(9.3.1)50 b(Op)q(erations)16 b(on)f(con)o(text)p 840 45 14 2 v 16 w(id's)f Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(77)143 102 y(9.4)46 b(Comm)o(unicators)20 b Fr(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(78)248 158 y(9.4.1)50 b(Op)q(erations)16 b(on)f(comm)o(unicators)h Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b Fs(78)248 214 y(9.4.2)50 b(Comm)o(unicator)15 b(constructors)23 b Fr(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(79)143 271 y(9.5)46 b(W)l(orking)15 b(without)g(con)o(text)p 750 271 V 16 w(id's)33 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(80)143 327 y(9.6)46 b(W)l(orking)15 b(without)g(maps)g(and)g(con)o (text)p 957 327 V 16 w(id's)38 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(81)143 384 y(9.7)46 b(Examples)36 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(82)248 440 y(9.7.1)50 b(Nonreen)o(tran)o(t)14 b(parallel)j(pro)q(cedures)29 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b Fs(82)248 497 y(9.7.2)50 b(P)o(arallel)16 b(pro)q(cedures)g(that)f(are)g(nonreen)o(tran)o(t)f(within)j(eac)o(h)e (executing)h(group)36 b(83)248 553 y(9.7.3)50 b(W)l(ell)17 b(nested)e(parallel)i(pro)q(cedures)28 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(83)143 610 y(9.8)46 b(Left)41 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(84)75 712 y Ft(A)28 b(Pro)q(cess)17 b(T)l(op)q(ology)h(Routines) 1061 b(87)p eop %%Page: 0 5 0 4 bop 875 722 a Ft(Abstract)75 828 y Fs(The)18 b(Message)f(P)o (assing)h(In)o(terface)f(F)l(orum)h(\(MPIF\),)e(with)i(participation)h (from)e(o)o(v)o(er)g(40)g(organiza-)75 885 y(tions,)f(has)h(b)q(een)g (meeting)g(since)h(Jan)o(uary)e(1993)f(to)h(discuss)h(and)g(de\014ne)g (a)f(set)h(of)f(library)h(in)o(terface)75 941 y(standards)f(for)h (message)f(passing.)25 b(MPIF)16 b(is)i(not)e(sanctioned)i(or)e(supp)q (orted)h(b)o(y)g(an)o(y)g(o\016cial)g(stan-)75 998 y(dards)e (organization.)166 1054 y(This)22 b(is)f(a)g(draft)f(of)h(what)g(will)h (b)q(ecome)g(the)f(Final)h(Rep)q(ort,)h(V)l(ersion)f(1.0,)f(of)f(the)i (Message)75 1111 y(P)o(assing)c(In)o(terface)h(F)l(orum.)29 b(This)19 b(do)q(cumen)o(t)g(con)o(tains)g(all)g(the)g(tec)o(hnical)h (features)e(prop)q(osed)h(for)75 1167 y(the)c(in)o(terface.)20 b(This)c(cop)o(y)f(of)g(the)g(draft)g(w)o(as)f(pro)q(cessed)i(b)o(y)f (L)1174 1161 y Fp(a)1195 1167 y Fs(T)1220 1181 y(E)1246 1167 y(X)g(on)g(Ma)o(y)g(28,)f(1993.)166 1224 y(MPIF)k(in)o(vites)h (commen)o(ts)f(on)h(the)f(tec)o(hnical)i(con)o(ten)o(t)e(of)g(MPI,)g (as)g(w)o(ell)i(as)e(on)g(the)h(editorial)75 1280 y(presen)o(tation)14 b(in)g(the)g(do)q(cumen)o(t.)19 b(Commen)o(ts)13 b(receiv)o(ed)i(b)q (efore)e(July)i(1,)e(1993)g(will)i(b)q(e)f(considered)h(in)75 1336 y(pro)q(ducing)i(the)e(\014nal)h(draft)e(of)h(V)l(ersion)h(1.0)e (of)h(the)g(Message)g(P)o(assing)g(In)o(terface)g(Sp)q(eci\014cation.) 166 1393 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o (terface,)g(simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g (used)75 1449 y(standard)g(for)f(writing)i(message-passing)f(programs.) 24 b(As)17 b(suc)o(h)h(the)f(in)o(terface)g(should)h(establishing)75 1506 y(a)d(practical,)h(p)q(ortable,)f(e\016cien)o(t,)g(and)h (\015exible)h(standard)e(for)f(message)h(passing.)p eop %%Page: 1 6 1 5 bop 75 -100 a Fq(CONTENTS)1519 b Fs(1)75 75 y Fu(Ac)m(kno)m (wledgmen)m(ts)166 282 y Fs(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f (w)o(as)e(carried)i(out)f(b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o (as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d)h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P) o(assing)g(In)o(terface)75 395 y(\(MPI\),)f(man)o(y)h(p)q(eople)i(serv) o(ed)e(in)h(p)q(ositions)g(of)f(resp)q(onsibilit)o(y:)143 501 y Fo(\017)23 b Fs(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o (er,)g(Con)o(v)o(eners)g(and)g(Meeting)h(Chairs)143 595 y Fo(\017)23 b Fs(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes) 143 689 y Fo(\017)23 b Fs(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i (Lusk,)f(P)o(oin)o(t)g(to)g(P)o(oin)o(t)f(Comm)o(unications)143 782 y Fo(\017)23 b Fs(Al)16 b(Geist,)e(Marc)h(Snir,)h(Stev)o(e)f(Otto,) f(Collectiv)o(e)j(Comm)o(unications)143 876 y Fo(\017)23 b Fs(Rolf)15 b(Hemp)q(el,)i(Pro)q(cess)e(T)l(op)q(ologies)143 970 y Fo(\017)23 b Fs(Scott)14 b(Berryman,)h(Language)g(Binding)143 1064 y Fo(\017)23 b Fs(Stev)o(e)15 b(Zenith,)g(F)l(ormal)g(Seman)o (tics)143 1158 y Fo(\017)23 b Fs(William)17 b(Gropp,)d(En)o(vironmen)o (tal)i(Inquiry)143 1251 y Fo(\017)23 b Fs(James)15 b(Co)o(wnie,)g (Pro\014ling)143 1345 y Fo(\017)23 b Fs(T)l(on)o(y)16 b(Skjellum,)h(Lyndon)g(Clark)o(e,)f(Marc)g(Snir,)h(Ric)o(hard)g (Little\014eld,)i(Marc)d(Sears,)g(Comm)o(u-)189 1402 y(nication)g(Con)o(texts)143 1496 y Fo(\017)23 b Fs(Stev)o(e)15 b(Otto,)f(Editor.)166 1602 y(Do)h(w)o(e)f(w)o(an)o(t)h(an)g (organization/compan)o(y)f(list)i(here?)166 1658 y(The)22 b(follo)o(wing)h(list)f(includes)i(some)e(of)f(the)h(activ)o(e)g (participan)o(ts)h(in)f(the)g(MPI)g(pro)q(cess)g(not)75 1715 y(men)o(tioned)16 b(ab)q(o)o(v)o(e:)100 1758 y(Ed)f(Anderson)114 b(Jo)q(e)15 b(Baron)193 b(Eric)15 b(Barszcz)100 1815 y(Rob)g(Bjornson)99 b(Anne)16 b(Elster)159 b(Jim)16 b(F)l(eeney)100 1871 y(Sam)f(Fineb)q(erg)98 b(Jon)15 b(Flo)o(w)o(er)176 b(Daniel)16 b(F)l(ry)o(e)100 1928 y(Ian)f(Glendinning)54 b(Adam)15 b(Green)o(b)q(erg)56 b(Rob)q(ert)15 b(Harrison)100 1984 y(Leslie)i(Hart)153 b(T)l(om)15 b(Haupt)166 b(Don)15 b(Heller)100 2041 y(T)l(om)g(Henderson)60 b(C.)14 b(T.)h(Ho)o(w)o(ard)f (Ho)50 b(Stev)o(en)15 b(Huss-Lederman)100 2097 y(John)h(Kap)q(enga)86 b(Bob)15 b(Leary)188 b(Arth)o(ur)14 b(Maccab)q(e)100 2154 y(P)o(eter)h(Madams)83 b(Oliv)o(er)16 b(McBry)o(an)77 b(Phil)16 b(McKinley)100 2210 y(Ch)o(uc)o(k)f(Mosher)87 b(Dan)15 b(Nessett)152 b(P)o(eter)14 b(P)o(ac)o(heco)100 2266 y(P)o(aul)h(Pierce)147 b(Sanja)o(y)14 b(Rank)m(a)124 b(P)o(eter)14 b(Rigsb)q(ee)100 2323 y(Arc)o(h)h(Robison)103 b(Am)o(buj)15 b(Singh)136 b(Alan)16 b(Sussman)100 2379 y(Rob)q(ert)f(G.)g(V)l(oigt)50 b(Dennis)16 b(W)l(eeks)120 b(Stephen)16 b(Wheat)166 2435 y(The)c(follo)o(wing)h(organizations)f (made)g(the)h(draft)e(a)o(v)m(ailable)j(b)o(y)e(anon)o(ymous)g(FTP)f (access)i(and/or)75 2491 y(mail)h(serv)o(ers:)k(Oak)c(Ridge)g(National) g(Lab)q(oratory)l(,)e(These)i(outlets)f(w)o(ere)g(instrumen)o(tal)h(in) g(distribut-)75 2547 y(ing)i(the)f(do)q(cumen)o(t.)166 2604 y(T)l(alk)g(ab)q(out)g(\014nancial)i(supp)q(ort)f(here.)p eop %%Page: 2 7 2 6 bop 75 356 a Fn(Chapter)34 b(1)75 564 y Fu(In)m(tro)s(duction)41 b(to)g(MPI)75 805 y Fm(1.1)70 b(Ov)n(erview)21 b(and)j(Goals)75 906 y Fs(Message)11 b(passing)i(is)f(a)g(paradigm)f(used)i(widely)g(on) f(certain)h(classes)f(of)f(parallel)j(mac)o(hines;)f(esp)q(ecially)75 963 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1019 y(pro)q(cesses)12 b(comm)o(unicating)h(through)e(messages)g(is) i(w)o(ell)f(understo)q(o)q(d.)20 b(Ov)o(er)12 b(the)f(last)h(ten)g(y)o (ears,)f(sub-)75 1076 y(stan)o(tial)16 b(progress)g(has)g(b)q(een)h (made)f(in)h(casting)g(signi\014can)o(t)g(applications)h(in)f(this)f (paradigm.)23 b(Eac)o(h)75 1132 y(v)o(endor)c(has)h(implemen)o(ted)h (their)f(o)o(wn)f(v)m(arian)o(t.)33 b(More)19 b(recen)o(tly)l(,)j(sev)o (eral)d(systems)g(ha)o(v)o(e)h(demon-)75 1188 y(strated)12 b(that)h(a)g(message)f(passing)i(system)f(can)g(b)q(e)h(e\016cien)o (tly)h(and)e(b)q(e)h(p)q(ortably)f(implemen)o(ted.)21 b(It)13 b(is)75 1245 y(th)o(us)h(an)f(appropriate)h(time)g(to)f(try)g (to)g(de\014ne)i(b)q(oth)f(the)g(syn)o(tax)f(and)h(seman)o(tics)g(of)f (a)g(core)h(of)f(library)75 1301 y(routines)h(that)e(will)j(b)q(e)f (useful)h(to)e(a)g(wide)h(range)f(of)g(users)g(and)h(e\016cien)o(tly)g (implemen)o(table)i(on)d(a)g(wide)75 1358 y(range)i(of)g(computers.)166 1414 y(In)21 b(designing)i(MPI)e(w)o(e)f(ha)o(v)o(e)h(sough)o(t)f(to)g (mak)o(e)g(use)h(of)g(the)g(most)f(attractiv)o(e)f(features)i(of)f(a)75 1471 y(n)o(um)o(b)q(er)12 b(of)g(existing)h(message)e(passing)h (systems,)g(rather)f(than)h(selecting)h(one)f(of)g(them)g(and)g (adopting)75 1527 y(it)f(as)f(the)h(standard.)18 b(Th)o(us,)11 b(MPI)g(has)f(b)q(een)i(strongly)f(in\015uenced)i(b)o(y)d(w)o(ork)g(at) g(the)h(IBM)g(T.)f(J.)h(W)l(atson)75 1584 y(Researc)o(h)18 b(Cen)o(ter)g([1)o(,)g(2)o(],)g(In)o(tel's)g(NX/2)g([16)o(],)f(Express) h([15)o(],)g(nCUBE's)g(V)l(ertex)g([13)o(],)f(P4)h([3)o(],)g(and)75 1640 y(P)l(ARMA)o(CS)11 b([11)o(,)g(12)o(].)18 b(Other)11 b(imp)q(ortan)o(t)g(con)o(tributions)g(ha)o(v)o(e)g(come)g(from)f(Zip)q (co)q(de)i([17)o(,)f(18)o(],)g(Chimp)75 1697 y([7)o(,)k(8],)f(PVM)h([9) o(,)g(19],)f(and)h(PICL)h([10)o(].)166 1753 y(One)j(of)f(the)g(ob)s (jectiv)o(es)g(of)g(this)g(pap)q(er)h(is)g(to)e(promote)h(a)f (discussion)j(within)g(the)e(concurren)o(t)75 1810 y(computing)d (researc)o(h)f(comm)o(unit)o(y)f(of)h(the)g(issues)h(that)f(m)o(ust)f (b)q(e)i(addressed)f(in)h(establishing)h(a)e(prac-)75 1866 y(tical,)j(p)q(ortable,)g(and)f(\015exible)j(standard)d(for)g (message)f(passing.)24 b(This)17 b(co)q(op)q(erativ)o(e)g(pro)q(cess)g (b)q(egan)75 1922 y(with)f(a)e(w)o(orkshop)h(on)g(standards)g(for)f (message)h(passing)h(held)g(in)g(April)h(1992)d([20)o(].)166 1979 y(The)k(main)h(adv)m(an)o(tages)e(of)h(establishing)i(a)e(message) g(passing)g(standard)g(are)g(p)q(ortabilit)o(y)h(and)75 2035 y(ease-of-use.)h(In)14 b(a)g(distributed)h(memory)e(comm)o (unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f(higher)h(lev)o (el)75 2092 y(routines)c(and/or)g(abstractions)f(are)h(build)i(up)q(on) e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i(routines)f(the)g(b)q (ene\014ts)75 2148 y(of)22 b(standardization)h(are)f(particularly)h (apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g (message)75 2205 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop) q(osed)g(here,)h(pro)o(vides)f(v)o(endors)h(with)f(a)g(clearly)h (de\014ned)h(base)75 2261 y(set)10 b(of)g(routines)g(that)g(they)g(can) h(implemen)o(t)g(e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o (vide)f(hardw)o(are)g(supp)q(ort)75 2318 y(for,)k(thereb)o(y)h (enhancing)i(scalabilit)o(y)l(.)166 2374 y(The)h(goal)g(of)g(the)g (Message)g(P)o(assing)g(In)o(terface)g(simply)i(stated)d(is)i(to)f(dev) o(elop)h(a)f(widely)h(used)75 2431 y(standard)e(for)f(writing)i (message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface) g(should)h(establishing)75 2487 y(a)d(practical,)h(p)q(ortable,)f (e\016cien)o(t,)g(and)h(\015exible)h(standard)e(for)f(message)h (passing.)166 2543 y(A)g(complete)h(list)g(of)f(goals)g(follo)o(w.)143 2647 y Fo(\017)23 b Fs(Design)11 b(an)f(application)i(programming)e(in) o(terface)h(\(not)f(necessarily)i(for)d(compilers)j(or)e(a)g(system)189 2704 y(implemen)o(tation)16 b(library\).)964 2828 y(2)p eop %%Page: 3 8 3 7 bop 75 -100 a Fq(1.2.)34 b(WHO)15 b(SHOULD)h(USE)g(THIS)g(ST)l(AND) o(ARD?)824 b Fs(3)143 45 y Fo(\017)23 b Fs(Allo)o(w)15 b(e\016cien)o(t)h(comm)o(unication:)21 b(Av)o(oid)16 b(memory)e(to)h(memory)g(cop)o(ying)g(and)h(allo)o(w)f(o)o(v)o(erlap) 189 102 y(of)e(computation)g(and)h(comm)o(unication)g(and)g(o\017oad)f (to)g(comm)o(unication)h(copro)q(cessor,)g(where)189 158 y(a)o(v)m(ailable.)143 255 y Fo(\017)23 b Fs(Allo)o(w)15 b(\(but)g(no)h(mandate\))e(extensions)i(for)f(use)g(in)h(heterogeneous) f(en)o(vironmen)o(t.)143 352 y Fo(\017)23 b Fs(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C,)f(F)l(ortran)f(77,)g(F)l(ortran)g(90,)g(and) i(C++)f(bindings)i(for)e(in)o(terface.)143 449 y Fo(\017)23 b Fs(Assume)15 b(a)g(reliable)i(comm)o(unication)f(in)o(terface:)k (User)15 b(need)h(not)f(cop)q(e)h(with)f(comm)o(unication)189 506 y(failures.)21 b(Suc)o(h)15 b(failures)i(are)d(dealt)i(b)o(y)f(the) h(underlying)h(comm)o(unication)f(subsystem.)143 603 y Fo(\017)23 b Fs(F)l(o)q(cus)15 b(on)g(a)g(prop)q(osal)h(that)e(can)h (b)q(e)h(agreed)f(up)q(on)h(in)g(6)f(mon)o(ths.)143 700 y Fo(\017)23 b Fs(De\014ne)f(an)g(in)o(terface)g(that)f(is)h(not)f(to)q (o)g(di\013eren)o(t)h(from)f(curren)o(t)h(practice,)h(suc)o(h)f(as)g (PVM,)189 756 y(Express,)14 b(P4,)h(etc.)143 853 y Fo(\017)23 b Fs(De\014ne)12 b(an)f(in)o(terface)h(that)f(can)h(b)q(e)g(quic)o(kly) h(implemen)o(ted)g(on)f(man)o(y)f(v)o(endor's)g(platforms,)h(with)189 909 y(no)j(signi\014can)o(t)h(c)o(hanges)f(in)h(the)f(underlying)j (comm)o(unication)d(and)h(system)f(soft)o(w)o(are.)143 1007 y Fo(\017)23 b Fs(The)15 b(in)o(terface)h(should)g(not)f(con)o (tain)g(more)g(functions)h(than)f(are)g(really)h(necessary)l(.)143 1104 y Fo(\017)23 b Fs(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h (should)g(b)q(e)g(language)f(indep)q(enden)o(t.)75 1251 y Fm(1.2)70 b(Who)23 b(Should)g(Use)g(This)f(Standard?)75 1354 y Fs(This)16 b(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h (all)g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 1411 y(programs)21 b(in)j(F)l(ortran)d(77,)i(C,)f (F)l(ortran)g(90,)h(or)f(C++.)43 b(This)23 b(includes)i(individual)g (application)75 1467 y(programmers,)14 b(dev)o(elop)q(ers)j(of)e(soft)o (w)o(are)f(designed)j(to)e(run)h(on)g(parallel)h(mac)o(hines,)f(and)g (creators)e(of)75 1524 y(en)o(vironmen)o(ts,)g(and)f(to)q(ols.)19 b(In)14 b(order)f(to)g(b)q(e)h(attractiv)o(e)e(to)h(this)h(wide)g (audience,)h(the)f(standard)f(m)o(ust)75 1580 y(pro)o(vide)j(a)f (simple,)i(easy-to-use)e(in)o(terface)h(for)e(the)i(basic)g(user)f (while)i(not)e(seman)o(tically)i(precluding)75 1637 y(the)e(high-p)q (erformance)i(message-passing)e(op)q(erations)g(a)o(v)m(ailable)i(on)e (adv)m(anced)i(mac)o(hines.)75 1784 y Fm(1.3)70 b(What)23 b(Platforms)f(Are)h(T)-6 b(argets)23 b(F)-6 b(or)24 b(Implemen)n (tation?)75 1887 y Fs(The)17 b(attractiv)o(eness)f(of)g(the)h (message-passing)g(paradigm)f(at)g(least)h(partially)h(stems)e(from)g (its)h(wide)75 1944 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o(y)f(ma)o(y)g(run)h(on)g (distributed-memory)h(m)o(ultipro)q(cessors,)75 2000 y(net)o(w)o(orks)i(of)i(w)o(orkstations,)e(and)i(com)o(binations)g(of)g (all)g(of)f(these.)34 b(In)20 b(addition,)h(shared-memory)75 2057 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 2113 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 2170 y(th)o(us)d(should)i(b)q(e)f(b)q(oth) g(p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f (on)h(a)f(great)g(v)m(ariet)o(y)g(of)75 2226 y(mac)o(hines,)i (including)h(those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h (of)e(other)h(mac)o(hines,)g(parallel)75 2283 y(or)d(not,)f(connected)i (b)o(y)f(a)g(comm)o(unication)h(net)o(w)o(ork.)166 2340 y(The)21 b(in)o(terface)f(is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i (general)f(MIMD)f(programs,)g(as)g(w)o(ell)h(as)f(those)75 2396 y(written)d(in)g(the)g(more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g (Although)h(no)f(explicit)j(supp)q(ort)e(for)f(threads)g(is)75 2453 y(pro)o(vided,)g(the)f(in)o(terface)g(has)g(b)q(een)i(designed)f (so)f(as)g(not)g(to)f(prejudice)j(their)f(use.)75 2601 y Fm(1.4)70 b(What)23 b(Is)g(Included)f(In)h(The)f(Standard?)75 2704 y Fs(The)15 b(standard)g(includes)j(\(this)d(is)h(temp)q(orarily)f (as)g(inclusiv)o(e)j(as)d(p)q(ossible\):)p eop %%Page: 4 9 4 8 bop 75 -100 a Fs(4)903 b Fq(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)143 45 y Fo(\017)23 b Fs(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 139 y Fo(\017)23 b Fs(Collectiv)o(e)16 b(op)q(erations)143 233 y Fo(\017)23 b Fs(Pro)q(cess)15 b(groups)143 327 y Fo(\017)23 b Fs(Comm)o(unication)15 b(con)o(texts)143 420 y Fo(\017)23 b Fs(A)15 b(simple)i(w)o(a)o(y)d(to) g(create)h(pro)q(cesses)h(for)e(the)i(SPMD)e(mo)q(del)143 514 y Fo(\017)23 b Fs(Bindings)17 b(for)d(F)l(ortran)g(77,)g(F)l (ortran)h(90,)f(C)h(and)g(C++)143 608 y Fo(\017)23 b Fs(A)15 b(mo)q(del)h(implemen)o(tation)143 702 y Fo(\017)23 b Fs(A)15 b(formal)g(sp)q(eci\014cation.)143 796 y Fo(\017)23 b Fs(Pro)q(cess)15 b(top)q(ology)143 889 y Fo(\017)23 b Fs(A)15 b(v)m(alidation)i(suite)75 1033 y Fm(1.5)70 b(What)23 b(Is)g(Not)g(Included)f(In)h(The)f(Standard?)75 1134 y Fs(The)15 b(standard)g(do)q(es)h(not)f(sp)q(ecify:)143 1240 y Fo(\017)23 b Fs(Explicit)17 b(shared-memory)e(op)q(erations)143 1334 y Fo(\017)23 b Fs(Op)q(erations)c(that)e(require)i(more)f(op)q (erating)h(system)f(supp)q(ort)g(than)g(is)h(curren)o(tly)g(standard;) 189 1391 y(for)14 b(example,)i(in)o(terrupt-driv)o(en)g(receiv)o(es,)g (remote)f(execution,)h(or)e(activ)o(e)i(messages)143 1484 y Fo(\017)23 b Fs(Program)13 b(construction)j(to)q(ols)143 1578 y Fo(\017)23 b Fs(Debugging)15 b(facilities)143 1672 y Fo(\017)23 b Fs(Auxiliary)17 b(functions)e(suc)o(h)h(as)f (timers)143 1766 y Fo(\017)23 b Fs(Explicit)17 b(supp)q(ort)e(for)g (threads)166 1872 y(There)g(are)f(man)o(y)h(features)f(that)g(ha)o(v)o (e)g(b)q(een)i(considered)g(and)f(not)g(included)i(in)f(this)f (standard.)75 1929 y(This)j(happ)q(ened)h(for)e(a)g(n)o(um)o(b)q(er)h (of)f(reasons,)g(one)g(of)g(whic)o(h)i(is)f(the)f(time)h(constrain)o(t) f(that)g(w)o(as)f(self)75 1985 y(imp)q(osed)g(in)h(\014nishing)g(the)e (standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g(included)j(can)e (alw)o(a)o(ys)e(b)q(e)i(o\013ered)f(as)75 2042 y(extensions)h(b)o(y)f (sp)q(eci\014c)i(implemen)o(tations.)p eop %%Page: 5 10 5 9 bop 75 357 a Fn(Chapter)34 b(2)75 565 y Fu(P)m(oin)m(t)41 b(to)f(P)m(oin)m(t)h(Comm)m(unication)876 788 y Fs(Marc)14 b(Snir)656 845 y(William)j(Gropp)e(and)g(Ewing)h(Lusk)75 992 y Fm(2.1)70 b(In)n(tro)r(duction)75 1095 y Fs(This)19 b(section)f(is)h(a)f(draft)f(of)h(the)g(curren)o(t)g(prop)q(osal)g(for) g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unication.)30 b(It)18 b(do)q(es)75 1151 y(not)d(y)o(et)g(include)i(a)e(description)i(of)d (the)i(F)l(ortran)e(77)g(and)i(C)f(bindings.)166 1208 y(I)g(ha)o(v)o(e)f(tried)i(to)e(indicate,)i(wherev)o(er)e(appropriate,) h(gaps)f(and)h(unresolv)o(ed)h(issues,)f(using)g(small)75 1265 y(t)o(yp)q(e.)166 1398 y Fl(Discussion:)45 b Fk(The)17 b(follo)o(wing)d(sections)k(of)e(the)h(in)o(tro)q(duction)g(con)o(tain) f(general)h(information)d(on)i(the)75 1448 y(design)g(of)f(MPI)h(pro)q (cedures.)27 b(The)16 b(material)e(should)h(b)q(e)i(mo)o(v)o(ed)d(to)i (a)g(general)g(in)o(tro)q(duction)f(for)h(the)g(en)o(tire)75 1497 y(do)q(cumen)o(t.)21 b(The)15 b(actual)g(binding)f(of)g(these)i (ob)r(jects)g(in)f(F)m(ortran)g(and)f(C)h(will)f(b)q(e)h(discussed)i (in)d(the)i(language)75 1547 y(binding)d(sub)q(committee.)75 1777 y Fm(2.2)70 b(Data)23 b(T)n(yp)r(es)75 1881 y Fj(2.2.1)55 b(Handle)75 1968 y Fs(MPI)16 b(pro)q(cedures)h(use)f(at)f(v)m(arious)h (places)h Fi(hand)r(les)p Fs(.)22 b(Handles)17 b(are)e(used)i(to)e (access)h(opaque)g(ob)s(jects.)75 2025 y(Suc)o(h)j(ob)s(ject)e(can)h(b) q(e)h(created,)g(up)q(dated)f(and)h(destro)o(y)o(ed)e(only)i(b)o(y)f(b) o(y)g(calling)i(suitable)f(MPI)f(pro-)75 2081 y(cedures,)i(and)f(pro)o (viding)g(the)g(handle)h(as)e(parameter.)29 b(Opaque)20 b(ob)s(jects)e(hide)i(from)d(the)i(user)g(the)75 2138 y(in)o(ternal)e(represen)o(tation)e(used)i(for)e(v)m(arious)h(MPI)f(ob) s(jects,)g(th)o(us)h(allo)o(wing)g(to)f(ha)o(v)o(e)h(similar)g(calls)h (in)75 2194 y(C)i(and)g(F)l(ortran,)g(allo)o(wing)h(to)f(o)o(v)o (ercome)f(problems)i(with)f(the)g(t)o(yping)h(rules)g(in)g(these)f (languages,)75 2251 y(and)c(allo)o(wing)g(for)g(future)f(extension)i (of)e(their)h(functionalit)o(y)l(.)21 b(The)15 b(mec)o(hanism)h(for)e (opaque)h(ob)s(jects)75 2307 y(used)h(here)f(lo)q(osely)i(follo)o(ws)e (the)g(POSIX)h(F)l(ortran)e(binding)k(standard.)166 2364 y(An)h(opaque)h(ob)s(ject)e(can)i(b)q(e)f Fi(p)n(ersistent)f Fs(or)h Fi(ephemer)n(al)p Fs(.)32 b(A)19 b(p)q(ersisten)o(t)h(ob)s (ject)e(p)q(ersists)i(un)o(til)75 2421 y(destro)o(y)o(ed)c(b)o(y)h(an)g (explicit)i(op)q(eration.)25 b(An)17 b(ephemeral)h(ob)s(ject)f(is)g(go) q(o)q(d)g(for)f(a)g(single)j(use;)e(th)o(us)g(an)75 2477 y(ephemeral)c(ob)s(ject)d(asso)q(ciated)i(with)g(a)f(comm)o(unication)h (op)q(eration)g(disapp)q(ears)g(once)g(this)g(op)q(eration)75 2534 y(is)g(completed)g(\(or)f(once)g(this)h(ob)s(ject)f(is)h(not)f (needed)i(an)o(ymore)d(for)h(the)h(completion)g(of)f(the)g(op)q (eration\).)166 2591 y(Opaque)21 b(ob)s(ject)e(are)h(created)g(b)o(y)g (calls)h(that)f(are)f(sp)q(eci\014c)j(to)e(eac)o(h)g(ob)s(ject)f(t)o (yp)q(e.)35 b(They)20 b(are)75 2647 y(destro)o(y)o(ed)i(b)o(y)g(a)g (call)h(to)f Fh(MPI)p 629 2647 15 2 v 17 w(FREE)p Fs(.)f(Additional)j (MPI)f(functions)g(are)f(a)o(v)m(ailable)i(to)d(access)i(and)75 2704 y(up)q(date)16 b(sp)q(eci\014c)h(opaque)e(ob)s(jects.)964 2828 y(5)p eop %%Page: 6 11 6 10 bop 75 -100 a Fs(6)631 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Ft(MPI)p 275 45 16 2 v 18 w(FREE\(handle\))75 164 y(IN)h(handle)24 b Fs(handle)16 b(to)f(ob)s(ject)166 248 y(Mark)e(the)g(handle)i(for)e (destruction.)20 b(The)14 b(ob)s(ject)f(can)h(destro)o(y)o(ed)f(as)g (so)q(on)g(as)h(there)f(there)h(is)g(no)75 304 y(p)q(ending)k(op)q (eration)e(that)f(is)h(using)h(this)f(ob)s(ject.)22 b(This)16 b(is)g(equiv)m(alen)o(t)i(to)d(c)o(hanging)i(the)e(p)q(ersistence)75 361 y(of)h(the)h(handle)h(from)e Fh(persistent)f Fs(to)h Fh(ephemeral)p Fs(.)23 b(F)l(or)16 b(example,)i(if)f(a)f(comm)o (unication)h(ob)s(ject)f(is)75 417 y(freed,)e(after)e(the)i(comm)o (unication)g(started)f(but)g(b)q(efore)h(it)g(completed,)g(then)g(the)g (p)q(ending)h(op)q(eration)75 474 y(is)i(not)e(a\013ected;)h(one)g(can) h(use)f(the)g(handle)i(in)f(calls)g(to)e Fh(MPI)p 1147 474 15 2 v 17 w(WAIT)g Fs(or)h Fh(MPI)p 1404 474 V 17 w(STATUS)p Fs(.)f(Ho)o(w)o(ev)o(er,)g(once)75 530 y(the)e(op)q(eration) h(completes,)g(it)g(is)g(erroneous)f(to)g(reuse)g(the)h(freed)g(handle) g(for)f(a)g(new)h(comm)o(unication.)166 663 y Fl(Discussion:)166 713 y Fk(An)f(alternativ)o(e)f(design)g(p)q(oin)o(t)g(w)o(as)h (discussed,)h(whereb)o(y)f(it)f(is)h(the)g(user)g(resp)q(onsibilit)o(y) f(to)h(free)g(a)f(handle)75 762 y(only)e(if)h(there)h(are)g(no)f(p)q (ending)g(op)q(erations)h(on)f(this)g(handle.)17 b(The)12 b(curren)o(t)g(design)g(w)o(as)f(deemed)g(more)f(elegan)o(t.)75 812 y(Note)17 b(that)g(it)f(is)h(not)f(necessary)j(to)d(destro)o(y)i (the)f(handle)f(as)h(so)q(on)g(as)f(it)h(is)f(not)h(needed)h(an)o (ymore.)25 b(A)17 b(v)n(alid)75 862 y(implemen)o(tatio)o(n)11 b(ma)o(y)h(just)i(mark)f(freed)h(handles)h(and)e(garbage)h(collect)g (them)f(p)q(erio)q(dically)m(.)166 912 y(On)18 b(the)h(other)g(hand,)g (some)e(ob)r(jects,)j(suc)o(h)f(as)f(bu\013er)i(descriptor)f(ob)r (jects,)h(can)f(b)q(e)f(used)h(b)o(y)f(sev)o(eral)75 962 y(distinct)c(concurren)o(t)i(comm)o(unicati)o(on)11 b(op)q(erations.)18 b(Th)o(us,)c(to)g(\014nd)g(out)g(whether)h(suc)o(h) f(ob)r(ject)h(can)f(b)q(e)h(freed,)75 1011 y(one)e(w)o(ould)f(need)i(a) e(reference)k(coun)o(t)d(of)f(p)q(ending)h(op)q(erations.)18 b(This)12 b(complication)f(w)o(as)i(not)f(observ)o(ed)i(at)f(our)75 1061 y(last)h(meeting,)e(and)i(ma)o(y)e(b)q(e)i(a)g(go)q(o)q(d)f (argumen)o(t)g(to)h(reconsider)h(the)g(curren)o(t)g(de\014nition)f(of)f Fg(MPI)p 1635 1061 14 2 v 15 w(FREE)p Fk(.)166 1111 y(W)m(e)f(do)g(not) h(ha)o(v)o(e)f(y)o(et)h(a)f(mec)o(hanism)e(to)j(con)o(trol)f(where)i (from)c(storage)j(is)g(allo)q(cated)f(for)g(opaque)g(ob)r(jects.)75 1161 y(Some)h(b)q(eliev)o(e)h(it)f(is)h(desirable)g(to)g(allo)o(w)e (the)j(user)g(to)f(con)o(trol)f(this)h(allo)q(cation.)166 1211 y(I)i(no)o(w)h(ha)o(v)o(e)f(a)g(generic)i Fg(MPI)p 635 1211 V 15 w(FREE)e Fk(function)g(for)g(all)g(kinds)g(of)g(opaque)h (ob)r(jects.)27 b(An)o(y)16 b(reason)i(to)e(ha)o(v)o(e)75 1261 y(sp)q(ecialized)f(functions)f(for)f(eac)o(h)i(kind)e(of)g(ob)r (ject,)h(as)g(for)g Fg(CREATE)p Fk(?)166 1435 y Ft(MPI)p 275 1435 16 2 v 18 w(ASSOCIA)l(TED\(handle,)k(t)o(yp)q(e\))75 1554 y(IN)f(handle)24 b Fs(handle)16 b(to)f(ob)s(ject)75 1644 y Ft(OUT)j(t)o(yp)q(e)23 b Fs(state)166 1728 y(Returns)17 b(the)g(t)o(yp)q(e)g(of)f(the)h(ob)s(ject)g(the)g(handle)h(is)f(curren) o(tly)g(asso)q(ciated)g(with,)h(if)f(suc)o(h)g(exists.)75 1784 y(Returns)d(the)f(sp)q(ecial)i(t)o(yp)q(e)e Fh(MPI)p 643 1784 15 2 v 17 w(NULL)f Fs(if)i(the)f(handle)h(is)g(not)f(curren)o (tly)g(asso)q(ciated)h(with)f(an)o(y)g(ob)s(ject.)166 1841 y(MPI)19 b(ma)o(y)g(pro)o(vide)h(prede\014ned)h(opaque)f(ob)s (jects)f(and)g(prede\014ned,)j(static)d(handles)i(to)e(these)75 1897 y(ob)s(jects.)g(Suc)o(h)d(ob)s(jects)f(ma)o(y)f(not)h(b)q(e)h (destro)o(y)o(ed.)75 2017 y Fj(2.2.2)55 b(Arra)n(y)19 b(of)g(handles)75 2103 y Fs(An)j(MPI)f(call)i(ma)o(y)e(need)i(a)e (parameter)g(that)g(is)h(an)f Fi(arr)n(ay)i(of)f(hand)r(les)p Fs(.)39 b(The)22 b(arra)o(y-of-handles)75 2159 y(ob)s(ject)16 b(is)g(not)g(self-delimitin)q(g;)j(it)d(do)q(es)h(not)f(carry)f (information)i(on)f(the)g(n)o(um)o(b)q(er)g(of)g(en)o(tries)h(in)g(the) 75 2216 y(arra)o(y)l(.)29 b(This)19 b(information)g(has)f(to)g(b)q(e)h (pro)o(vided)h(b)o(y)e(an)h(additional)h Fh(len)e Fs(parameter,)g (whenev)o(er)h(a)75 2272 y(list-of-handles)e(parameter)e(is)g(used.)166 2405 y Fl(Discussion:)166 2455 y Fk(The)i(reason)h(for)f(not)g(ha)o (ving)f(self-delimiting)e(arra)o(ys)k(is)f(that)g(records)h(are)g(not)f (\014rst-class)h(citizens)g(in)75 2504 y(F)m(ortran)13 b(and)g(that)g(ha)o(ving)f(an)h(arra)o(y)g(where)h(one)f(en)o(try)h (enco)q(des)h(length,)e(whereas)h(the)g(other)f(en)o(tries)i(enco)q(de) 75 2554 y(handles)e(w)o(as)g(deemed)g(to)g(b)q(e)g(to)q(o)g (error-prone.)19 b(This)13 b(line)f(of)g(argumen)o(tation)f(really)h (implies)f(that)i(w)o(e)g(do)g(not)75 2604 y(view)e(arra)o (y-of-handles)g(as)h(an)f(opaque)h(ob)r(ject,)g(and)g(that)f(w)o(e)h (exp)q(ect)h(the)f(user)h(to)f(build)e(these)j(ob)r(jects)g(explic-)75 2654 y(itly)m(,)f(not)h(via)g(MPI)h(calls.)k(The)c(alternativ)o(e)f(is) h(for)f(arra)o(y-of-handles)g(to)h(b)q(e)g(an)f(opaque)h(ob)r(ject,)g (manipulated)75 2704 y(via)f(s)h(set)h(of)e(MPI)h(calls.)p eop %%Page: 7 12 7 11 bop 75 -100 a Fq(2.3.)34 b(PR)o(OCESSES)1403 b Fs(7)75 45 y Fj(2.2.3)55 b(State)75 133 y Fs(MPI)15 b(pro)q(cedures)h(use)g(at) e(v)m(arious)i(places)g(argumen)o(ts)f(with)g Fi(state)g Fs(t)o(yp)q(es.)20 b(The)c(v)m(alues)g(of)f(suc)o(h)h(data)75 190 y(t)o(yp)q(e)i(are)h(all)g(iden)o(ti\014ed)h(b)o(y)f(names,)f(and)h (no)f(op)q(eration)h(is)g(de\014ned)h(on)e(them.)29 b(F)l(or)18 b(example,)i(the)75 246 y Fh(MPI)p 150 246 15 2 v 17 w(APPEND)14 b Fs(routine)i(has)f(a)g(state)f(t)o(yp)q(e)h(parameter)g (with)g(v)m(alues)i Fh(MPI)p 1343 246 V 16 w(INT,)24 b(MPI)p 1551 246 V 17 w(REAL,)14 b Fs(etc.)75 376 y Fj(2.2.4)55 b(Named)17 b(constan)n(ts)75 464 y Fs(MPI)11 b(pro)q(cedures)h (sometimes)f(assign)g(a)g(sp)q(ecial)i(meaning)f(to)e(a)h(sp)q(ecial)i (v)m(alue)f(of)f(a)g(basic)g(t)o(yp)q(e)g(param-)75 520 y(eter;)17 b(e.g.)24 b Fh(tag)17 b Fs(is)g(an)g(in)o(teger)g(v)m(alued) h(parameter)e(of)g(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)f(op)q (erations,)75 577 y(with)f(a)f(sp)q(ecial)i Fh(DONTCARE)d Fs(v)m(alue.)22 b(Suc)o(h)16 b(parameters)e(will)j(ha)o(v)o(e)e(a)g (range)g(of)g(regular)h(v)m(alues,)g(whic)o(h)75 633 y(is)h(a)f(prop)q(er)h(subrange)f(of)g(the)h(range)f(of)g(v)m(alues)h (of)f(the)h(corresp)q(onding)g(basic)g(t)o(yp)q(e;)g(sp)q(ecial)h(v)m (alues)75 690 y(\(suc)o(h)e(as)g(DONTCARE\))g(will)i(b)q(e)f(outside)g (the)g(regular)f(range.)23 b(The)16 b(range)g(of)g(regular)g(v)m(alues) i(can)75 746 y(b)q(e)d(queried,)g(and)f(sometimes)g(set,)g(using)h(en)o (vironmen)o(t)f(inquiry)i(or)e(en)o(vironmen)o(t)g(setting)g(functions) 75 803 y(\(Section)i(7\).)166 937 y Fl(Discussion:)166 988 y Fk(Need)f(to)f(agree)g(on)g(a)f(language)g(mec)o(hanism)f(for)h (named)g(constan)o(ts.)166 1039 y(Implemen)o(ters)h(should)g(detect,)j (whenev)o(er)f(p)q(ossible,)f(illegal)e(uses)j(of)e(\\sp)q(ecial)h(v)n (alues".)21 b(Th)o(us,)15 b(the)g(use)75 1089 y(of)e(the)i Fg(DONTCARE)d Fk(v)n(alue)h(to)h(tag)f(a)h(message)g(sen)o(t)g(will)f (b)q(e)h(\015agged)g(as)g(an)g(error.)75 1301 y Fj(2.2.5)55 b(Choice)75 1389 y Fs(MPI)16 b(functions)g(sometimes)g(use)g (parameters)f(with)h(a)g Fi(choic)n(e)f Fs(\(or)g(union\))i(data)e(t)o (yp)q(e.)21 b(I.e.,)16 b(distinct)75 1445 y(calls)i(to)e(the)h(same)f (routine)h(ma)o(y)f(pass)h(b)o(y)f(reference)i(actual)e(parameters)g (of)g(di\013eren)o(t)h(t)o(yp)q(es.)25 b(The)75 1502 y(mec)o(hanism)16 b(for)e(pro)o(viding)j(suc)o(h)e(parameters)f(will)j (di\013er)f(from)e(language)i(to)e(language.)166 1636 y Fl(Discussion:)166 1687 y Fk(The)j(F)m(ortran)g(77)g(standard)g(sp)q (eci\014es)i(that)e(the)h(t)o(yp)q(e)f(of)f(actual)h(argumen)o(ts)f (need)i(to)f(agree)h(with)e(the)75 1737 y(t)o(yp)q(e)f(of)f(dumm)o(y)d (argumen)o(ts;)i(no)i(construct)h(equiv)n(alen)o(t)d(to)i(C)f(p)q(oin)o (ters)h(is)f(a)o(v)n(ailable.)j(Th)o(us,)e(it)f(w)o(ould)f(seem)75 1786 y(that)19 b(there)i(is)e(no)g(standard)h(conforming)d(mec)o (hanism)g(to)i(supp)q(ort)h(c)o(hoice)f(parameters.)35 b(Ho)o(w)o(ev)o(er,)20 b(most)75 1836 y(F)m(ortran)g(compiler)e(either) j(don't)e(c)o(hec)o(k)i(t)o(yp)q(e)f(consistency)h(of)e(calls)h(to)f (external)i(routines,)g(or)f(supp)q(ort)g(a)75 1886 y(sp)q(ecial)d(mec) o(hanism)d(to)j(link)e(foreign)h(\(e.g.,)g(C\))h(routines.)27 b(I)17 b(suggest)g(that)g(w)o(e)g(accept)g(this)g(nonconformit)o(y)75 1936 y(with)12 b(F)m(ortran)h(77)f(standard.)19 b(I.e.,)12 b(w)o(e)h(accept)h(that)f(the)g(same)f(routine)h(ma)o(y)e(b)q(e)i (passed)h(an)f(actual)f(parameter)75 1986 y(of)h(a)h(di\013eren)o(t)h (t)o(yp)q(e)f(at)g(distinct)g(calls.)166 2037 y(Generic)j(routines)g (can)f(b)q(e)h(used)h(in)d(F)m(ortran)i(90)e(to)h(pro)o(vide)h(a)f (standard)g(conforming)f(solution.)24 b(This)75 2087 y(solution)13 b(will)f(b)q(e)j(consisten)o(t)g(with)f(our)f (nonstandard)i(conforming)c(F)m(ortran)j(77)f(solution.)75 2320 y Fm(2.3)70 b(Pro)r(cesses)75 2507 y Fl(Discussion:)15 b Fk(This)f(material)e(b)q(elongs)i(to)g(an)f(in)o(tro)q(duction)h(or)g (en)o(vironmen)o(t)f(section)166 2647 y Fs(An)i(MPI)f(program)g(is)h (executed)h(b)o(y)e(sev)o(eral)h(autonomous)f(pro)q(cesses)h(that)e (eac)o(h)i(execute)g(their)75 2704 y(o)o(wn)k(co)q(de,)h(in)g(a)e(MIMD) h(st)o(yle.)31 b(The)19 b(co)q(des)h(executed)g(b)o(y)f(eac)o(h)g(pro)q (cess)g(need)h(not)f(b)q(e)g(iden)o(tical.)p eop %%Page: 8 13 8 12 bop 75 -100 a Fs(8)631 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(The)k(pro)q(cesses)g(comm)o (unicate)g(via)g(calls)g(to)f(MPI)h(comm)o(unication)g(primitiv)o(es.) 35 b(T)o(ypically)l(,)22 b(eac)o(h)75 102 y(pro)q(cessor)17 b(executes)i(in)f(its)g(o)o(wn)f(address)g(space,)h(although)g (shared-memory)g(implemen)o(tations)g(of)75 158 y(MPI)12 b(are)g(p)q(ossible.)21 b(This)13 b(do)q(cumen)o(t)f(sp)q(eci\014es)i (the)f(b)q(eha)o(vior)f(of)g(a)g(parallel)i(program)d(assuming)i(that) 75 214 y(only)h(MPI)g(calls)h(are)e(used)i(for)e(comm)o(unication.)20 b(The)14 b(in)o(teraction)g(of)f(an)h(MPI)g(program)e(with)i(other)75 271 y(p)q(ossible)i(means)e(of)g(comm)o(unication)g(\(e.g.,)f(shared)h (memory\))g(is)g(not)g(sp)q(eci\014ed.)22 b(In)14 b(particular,)h(it)f (is)75 327 y(assumed)h(that)g(message)g(bu\013ers)g(at)f(distinct)j (pro)q(cessors)e(are)f(disjoin)o(t.)166 386 y(MPI)d(do)q(es)g(not)g(sp) q(ecify)h(the)f(execution)h(mo)q(del)g(for)e(eac)o(h)h(pro)q(cess.)19 b(A)11 b(pro)q(cess)g(can)h(b)q(e)f(sequen)o(tial,)75 442 y(or)h(can)g(b)q(e)h(m)o(ultithreaded,)g(with)g(threads)f(p)q (ossibly)h(executing)h(concurren)o(tly)l(.)19 b(Care)12 b(has)g(b)q(een)h(tak)o(en)75 499 y(to)i(mak)o(e)f(MPI)h (\\thread-safe",)f(b)o(y)i(a)o(v)o(oiding)f(the)g(use)h(of)f(implicit)i (global)f(states.)166 557 y(The)d(initial)h(allo)q(cation)g(of)e(pro)q (cesses)g(to)g(an)g(MPI)h(computation)f(and)h(their)g(binding)h(to)e (ph)o(ysical)75 614 y(pro)q(cessors)17 b(is)i(not)e(sp)q(eci\014ed)j(b) o(y)d(the)h(program)f(itself.)28 b(It)18 b(is)g(exp)q(ected)h(that)e(v) o(endors)g(will)i(pro)o(vide)75 670 y(mec)o(hanisms)f(to)f(do)h(so)f (either)h(at)f(load)h(time)g(or)f(at)g(run)h(time.)27 b(Suc)o(h)19 b(mec)o(hanisms)f(will)h(allo)o(w)f(the)75 727 y(sp)q(eci\014cation)i(of)d(the)h(initial)i(n)o(um)o(b)q(er)e(of)f (required)i(pro)q(cesses,)f(the)g(co)q(de)g(to)f(b)q(e)i(executed)f(b)o (y)g(eac)o(h)75 783 y(initial)k(pro)q(cess,)e(and)g(the)g(allo)q (cation)h(of)e(pro)q(cesses)h(to)f(pro)q(cessors.)33 b(Also,)20 b(the)g(curren)o(t)g(prop)q(osal)75 839 y(do)q(es)f(not)f (pro)o(vide)h(for)e(dynamic)j(creation)e(or)g(deletion)i(of)e(pro)q (cesses)g(during)i(program)d(execution)75 896 y(\(the)f(total)f(n)o(um) o(b)q(er)h(of)f(pro)q(cesses)i(is)f(\014xed\),)g(although)g(it)g(is)g (in)o(tended)i(to)d(b)q(e)h(consisten)o(t)g(with)h(suc)o(h)75 952 y(extension.)j(Finally)l(,)13 b(the)f(curren)o(t)g(prop)q(osal)g (do)q(es)h(not)e(sp)q(ecify)i(a)f(naming)g(sc)o(heme)g(for)g(pro)q (cesses.)19 b(W)l(e)75 1009 y(prop)q(ose)e(to)f(alw)o(a)o(ys)g(iden)o (tify)i(pro)q(cesses)f(according)g(to)f(their)h(relativ)o(e)g(rank)f (in)i(a)e(con)o(text)g(\(group\),)75 1065 y(so)f(that,)f(e\013ectiv)o (ely)l(,)i(pro)q(cesses)f(are)g(iden)o(ti\014ed)j(b)o(y)d(consecutiv)o (e)h(in)o(tegers.)75 1198 y Fj(2.3.1)55 b(Error)18 b(Handling)75 1371 y Fl(Discussion:)51 b Fk(Material)18 b(of)f(this)i(section)g (should)f(b)q(e)g(mo)o(v)o(ed)f(to)h(a)g(general)g(in)o(tro)q(duction)g (section.)32 b(This)75 1427 y(material)12 b(has)i(not)g(y)o(et)g(b)q (een)h(appro)o(v)o(ed)f(b)o(y)f(MPIF)166 1568 y Fs(MPI)18 b(pro)o(vides)g(the)h(user)f(with)g(reliable)i(message)e(transmission:) 26 b(A)18 b(message)f(sen)o(t)h(is)h(alw)o(a)o(ys)75 1625 y(receiv)o(ed)f(correctly)l(,)f(and)h(the)f(user)g(do)q(es)g(not)f (need)i(to)e(c)o(hec)o(k)i(for)e(transmission)h(errors,)f(time-outs,)75 1681 y(or)i(other)g(error)f(conditions.)31 b(In)19 b(other)f(w)o(ords,) g(MPI)g(do)q(es)h(not)f(pro)o(vide)h(mec)o(hanisms)g(for)f(dealing)75 1738 y(with)i(failures)h(in)g(the)f(comm)o(unication)h(system.)34 b(Where)20 b(the)g(MPI)g(implemen)o(tation)h(is)g(built)g(on)75 1794 y(an)h(unreliable)i(underlying)g(mec)o(hanism,)f(then)g(it)f(is)g (the)g(job)f(of)h(the)g(implemen)o(ter)h(of)e(the)h(MPI)75 1851 y(subsystem)16 b(to)g(insulate)i(the)e(user)h(from)f(this)h (unreliabilit)o(y)l(,)i(or)d(to)g(re\015ect)h(unreco)o(v)o(erable)g (errors)e(as)75 1907 y(global)20 b(program)e(failures.)34 b(Similarly)21 b(MPI)f(itself)g(pro)o(vides)g(no)f(mec)o(hanisms)h(for) f(handling)i(no)q(de)75 1964 y(failures.)166 2022 y(Of)16 b(course,)h(MPI)f(programs)f(ma)o(y)h(still)i(b)q(e)f(erroneous.)23 b(A)16 b Ft(program)i(error)d Fs(can)i(o)q(ccur)f(when)75 2079 y(an)i(MPI)f(call)i(is)f(called)h(with)f(an)g(incorrect)g (parameter)f(\(non-existing)h(destination)h(in)f(a)g(send)g(op-)75 2135 y(eration,)i(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i(op) q(eration,)f(etc.\))33 b(This)20 b(t)o(yp)q(e)g(of)f(error)g(w)o(ould)h (o)q(ccur)g(in)75 2191 y(an)o(y)e(implemen)o(tation.)29 b(In)19 b(addition,)g(a)f Ft(resource)i(error)c Fs(ma)o(y)i(o)q(ccur)g (when)g(a)g(program)f(exceeds)75 2248 y(the)k(amoun)o(t)f(of)g(a)o(v)m (ailable)i(system)f(resources)f(\(n)o(um)o(b)q(er)h(of)f(p)q(ending)j (messages,)e(system)f(bu\013ers,)75 2304 y(etc.\).)29 b(The)19 b(o)q(ccurrence)g(of)f(this)h(t)o(yp)q(e)g(of)f(error)f(dep)q (ends)j(on)f(the)f(amoun)o(t)g(of)g(a)o(v)m(ailable)i(resources)75 2361 y(in)d(the)f(system)g(and)g(the)g(resource)g(allo)q(cation)h(mec)o (hanism)g(used;)g(this)f(ma)o(y)f(di\013er)i(from)e(system)h(to)75 2417 y(system.)j(The)13 b(recommended)i(implemen)o(tation)f(pro\014le)h (pro)o(vides)e(sev)o(eral)h(mec)o(hanisms)g(to)f(alleviate)75 2474 y(the)19 b(p)q(ortabilit)o(y)h(problem)g(this)f(represen)o(ts.)31 b(One)20 b(can)f(also)g(write)g Ft(safe)g Fs(programs,)f(that)h(are)f (not)75 2530 y(sub)s(ject)d(to)g(resource)g(errors.)166 2589 y(All)f(MPI)f(pro)q(cedure)h(calls)g(return)f(an)g(error)f (parameter)g(that)g(indicates)j(successful)f(completion)75 2645 y(of)h(the)g(op)q(eration,)g(or)g(the)g(error)g(condition)h(that)f (o)q(ccurred,)g(otherwise.)166 2704 y(The)21 b(recommended)h(implemen)o (tation)g(pro\014le)g(in)g(a)f(POSIX)h(en)o(vironmen)o(t)f(is)h(for)e (an)o(y)h(MPI)p eop %%Page: 9 14 9 13 bop 75 -100 a Fq(2.4.)34 b(MESSA)o(GES)1423 b Fs(9)75 45 y(routine)14 b(that)f(encoun)o(ters)h(a)g(reco)o(v)o(erable)f(error) g(to)h(generate)f(an)h Fi(MPI)g(err)n(or)h(signal)p Fs(,)d(using)j(a)e (sp)q(ecial)75 102 y(signal)20 b(v)m(alue.)31 b(The)19 b(default)h(handler)g(for)e(this)h(signal)h(terminates)e(the)h (execution)h(of)e(all)i(in)o(v)o(olv)o(ed)75 158 y(pro)q(cesses,)i (with)f(a)f(suitable)i(error)e(message)g(b)q(eing)i(returned)f(to)f (the)g(user.)37 b(Ho)o(w)o(ev)o(er,)20 b(the)h(user)75 214 y(can)d(pro)o(vide)h(his)g(or)f(her)h(o)o(wn)f(signal)h(handling)h (routine.)30 b(In)19 b(particular,)g(the)g(user)f(can)h(sp)q(ecify)g(a) 75 271 y(\\no)q(op")d(signal)i(handler,)g(th)o(us)f(relegating)g(all)h (error)e(handling)j(to)d(the)h(user)g(co)q(de,)g(using)h(the)f(error)75 327 y(parameters)d(returned)i(b)o(y)f(the)g(MPI)h(calls.)166 384 y(MPI)d(calls)h(ma)o(y)f(initiate)h(op)q(erations)g(that)e(con)o (tin)o(ue)i(async)o(hronously)g(after)e(the)i(call)g(returned.)75 440 y(Th)o(us,)22 b(the)g(op)q(eration)f(ma)o(y)g(return)g(with)h(a)f (co)q(de)h(indicating)h(successful)g(completion,)g(y)o(et)e(later)75 497 y(cause)15 b(an)g(error)g(exception)h(to)e(b)q(e)i(raised.)k(If)15 b(there)g(is)h(a)f(subsequen)o(t)g(call)h(that)f(relates)g(to)f(the)h (same)75 553 y(op)q(eration)f(\(e.g.,)f(a)g(call)i(that)e(v)o (eri\014es)i(that)e(an)h(async)o(hronous)f(op)q(eration)h(has)g (completed\))h(then)f(the)75 610 y(error)f(parameter)g(asso)q(ciated)h (with)g(this)g(call)h(will)g(b)q(e)g(used)f(to)f(indicate)i(the)f (nature)g(of)f(the)h(error.)k(In)75 666 y(a)c(few)h(cases,)f(the)g (error)g(ma)o(y)g(o)q(ccur)h(after)f(all)h(calls)h(that)d(relate)i(to)f (the)h(op)q(eration)f(ha)o(v)o(e)g(completed,)75 723 y(so)g(that)g(no)g(error)g(parameter)g(can)g(b)q(e)i(used)f(to)e (indicate)k(the)d(nature)g(of)g(the)h(error)f(\(e.g.,)f(an)h(error)g (in)75 779 y(a)e(send)i(with)f(the)g(ready)g(mo)q(de\).)19 b(In)13 b(suc)o(h)g(cases,)g(an)g(error)f(will)i(b)q(e)g(undetected,)g (if)f(the)g(user)g(disabled)75 835 y(the)i(MPI)g(error)g(signal.)166 968 y Fl(Discussion:)34 b Fk(The)14 b(alternativ)o(e)g(c)o(hoice)g(is)g (to)g(ha)o(v)o(e)f(fatal)g(and)h(non-fatal)e(signals.)166 1018 y(One)j(migh)o(t)c(w)o(an)o(t)j(di\013eren)o(t)h(signals)e(for)g (di\013eren)o(t)i(mo)q(dules.)166 1074 y(The)f(details)g(of)f(suc)o(h)i (prop)q(osal)e(need)i(b)q(e)g(elab)q(orated)f(in)f(an)h(appropriate)g (\\pro\014le")f(sub)q(committee.)75 1299 y Fm(2.4)70 b(Messages)75 1401 y Fs(A)15 b(message)g(consists)g(of)g(an)g Fi(envelop)n(e)f Fs(and)i Fi(data)p Fs(.)75 1521 y Fj(2.4.1)55 b(Data)75 1607 y Fs(The)19 b(data)f(part)g(of)g(a)h(message)f(consists) h(of)f(a)h(sequence)g(of)g(v)m(alues,)h(eac)o(h)f(of)f(a)h(basic)g (datat)o(yp)q(e)f(in)75 1664 y(the)d(host)f(language.)20 b(Th)o(us,)14 b(in)i(F)l(ortran)d(77,)h(a)h(message)f(consists)h(of)f (a)g(sequence)i(of)e(v)m(alues)i(that)e(are)75 1720 y(eac)o(h)20 b(of)g(t)o(yp)q(e)h Fh(INTEGER)p Fs(,)e Fh(REAL)p Fs(,)g Fh(DOUBLE)k(PRECISION)p Fs(,)c Fh(COMPLEX)p Fs(,)g Fh(DOUBLE)k (PRECISION)f(COMPLEX)p Fs(,)75 1776 y Fh(LOGICAL)p Fs(,)12 b(or)h(\(length)h(1\))f Fh(CHARACTER)p Fs(.)e(A)j(message)f(ma)o(y)f (also)i(con)o(tain)f(a)h(v)m(alue)g(of)f(t)o(yp)q(e)h Fh(BYTE)p Fs(,)e(whic)o(h)75 1833 y(consists)18 b(of)g(8)g(binary)h (digits.)29 b(A)19 b(b)o(yte)e(is)i(di\013eren)o(t)g(from)e(a)h(c)o (haracter.)28 b(Di\013eren)o(t)18 b(mac)o(hines)h(ma)o(y)75 1889 y(ha)o(v)o(e)f(di\013eren)o(t)h(represen)o(tations)g(for)f(the)h (same)f(c)o(haracter,)h(or)f(ma)o(y)g(use)h(more)g(than)f(one)h(b)o (yte)g(to)75 1946 y(represen)o(t)12 b(a)g(c)o(haracter.)18 b(On)13 b(the)f(other)g(hand,)h(a)e(b)o(yte)h(has)g(the)h(same)e (binary)i(v)m(alue)h(on)e(an)o(y)f(mac)o(hine.)75 2002 y(A)k(message)g(ma)o(y)f(mix)i(v)m(alues)g(of)f(di\013eren)o(t)h(t)o (yp)q(es.)166 2135 y Fl(Missing:)166 2185 y Fk(Need)d(to)f(agree)h(on)e (the)i(C)f(t)o(yp)q(es)h(\(including)e(handling)g(of)h (signed/unsigned\),)g(and)g(on)g(F)m(ortran)g(90)f(t)o(yp)q(es)75 2234 y(\(including)i(handling)g(of)g(kinds\).)75 2438 y Fj(2.4.2)55 b(En)n(v)n(elop)r(e)75 2524 y Fs(The)15 b(follo)o(wing)h(information)g(is)f(asso)q(ciated)h(with)f(eac)o(h)h (message:)75 2612 y Ft(source)22 b Fs(The)16 b(rank)f(of)f(the)i (sending)g(pro)q(cess)75 2704 y Ft(destination)25 b Fs(The)15 b(rank)g(of)g(the)g(receiving)i(pro)q(cess)p eop %%Page: 10 15 10 14 bop 75 -100 a Fs(10)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Ft(tag)24 b Fs(User)15 b(de\014ned)75 134 y Ft(con)o(text)23 b Fs(handle)166 214 y(The)14 b(range)g(of)g(v)m(alid)h(v)m(alues)h(for)d (the)h Ft(source)g Fs(and)g Ft(destination)i Fs(\014elds)g(is)e Fh(0)24 b(...)47 b(n-1)p Fs(,)14 b(where)75 271 y Fh(n)g Fs(is)h(the)f(n)o(um)o(b)q(er)h(of)f(pro)q(cesses)g(in)h(the)g(sp)q (eci\014ed)h(con)o(text.)j Fh(source)k(=)h(destination)13 b Fs(is)h(allo)o(w)o(ed:)20 b(a)75 327 y(pro)q(cess)15 b(can)h(send)g(a)e(message)h(to)g(itself.)166 384 y(The)h(ranges)f(of)h (v)m(alid)h(v)m(alues)g(for)e Fh(tag)h Fs(is)g(implemen)o(tation)h(dep) q(enden)o(t,)g(and)f(can)g(b)q(e)h(found)f(b)o(y)75 440 y(calling)h(a)e(suitable)h(query)g(function,)f(as)g(describ)q(ed)i(in)f (Section)g(7.)166 497 y(The)f Fh(tag)g Fs(\014eld)h(can)f(b)q(e)h (arbitrarily)g(set)f(b)o(y)g(the)g(application,)h(and)g(can)f(b)q(e)h (used)f(to)g(distinguish)75 553 y(di\013eren)o(t)g(messages.)166 609 y Fh(Context)f Fs(should)i(b)q(e)g(a)f(con)o(text)g(shared)g(b)o(y) g(b)q(oth)g(source)h(and)f(destination.)166 666 y(The)e(actual)h(mec)o (hanism)g(used)f(to)g(asso)q(ciate)g(an)g(en)o(v)o(elop)q(e)i(with)e(a) g(message)g(is)h(implemen)o(tation)75 722 y(dep)q(enden)o(t;)k(some)e (of)g(the)g(information)g(\(e.g.,)f Ft(sender)g Fs(or)h Ft(receiv)o(er)p Fs(\))f(ma)o(y)g(b)q(e)i(implicit,)i(and)d(need)75 779 y(not)f(b)q(e)h(explicitly)i(carried)d(b)o(y)h(a)e(message.)75 920 y Fm(2.5)70 b(Comm)n(unication)20 b(Bu\013ers)75 1021 y Fs(The)f(basic)h(p)q(oin)o(t)g(to)e(p)q(oin)o(t)i(comm)o (unication)g(op)q(erations)f(are)g Ft(send)g Fs(and)g Ft(receiv)o(e)p Fs(.)31 b(A)19 b Ft(send)g Fs(op-)75 1077 y(eration)g(creates)g(a)f(message;)i(the)g(message)e(data)g(is)i (tak)o(en)f(from)f(the)h Ft(send)i(bu\013er)p Fs(.)31 b(A)19 b Ft(receiv)o(e)75 1134 y Fs(op)q(eration)i(consumes)h(a)f (message;)i(the)e(message)g(data)f(is)i(put)f(in)o(to)g(the)g Ft(receiv)o(e)j(bu\013er)p Fs(.)37 b(The)75 1190 y(sp)q(eci\014cation) 17 b(of)e(send)h(or)e(receiv)o(e)i(bu\013ers)g(uses)f(the)g(same)g(syn) o(tax.)75 1310 y Fj(2.5.1)55 b(Comm)n(unication)17 b(bu\013er)h(comp)r (onen)n(ts)75 1395 y Fs(A)j(bu\013er)f(consists)h(of)f(a)g(sequence)i (of)e Ft(bu\013er)j(comp)q(onen)o(ts)p Fs(.)35 b(Eac)o(h)21 b(comp)q(onen)o(t)f(consists)h(of)f(a)75 1452 y(sequence)e(of)f(\(not)g (necessarily)i(distinct\))f(v)m(ariables)g(of)f(the)h(same)e(basic)i (datat)o(yp)q(e.)26 b(The)17 b(datat)o(yp)q(e)75 1508 y(of)i(suc)o(h)g(a)g(v)m(ariable)h(is)g(sp)q(eci\014ed)h(b)o(y)e(a)g (state)f(parameter.)31 b(The)19 b(p)q(ossible)i(v)m(alues)f(for)e(F)l (ortran)g(are)75 1565 y Fh(MPI)p 150 1565 15 2 v 17 w(INT)p Fs(,)13 b Fh(MPI)p 337 1565 V 17 w(REAL)p Fs(,)g Fh(MPI)p 548 1565 V 17 w(DOUBLE)p Fs(,)g Fh(MPI)p 807 1565 V 17 w(COMPLEX)p Fs(,)f Fh(MPI)p 1089 1565 V 17 w(DCOMPLEX)p Fs(,)h Fh(MPI)p 1396 1565 V 17 w(LOGICAL)p Fs(,)f Fh(MPI)p 1678 1565 V 17 w(CHAR)i Fs(and)75 1621 y Fh(MPI)p 150 1621 V 17 w(BYTE)p Fs(,)c(corresp)q(onding)i(to)e(datat)o(yp)q(es)g Fh(INTEGER)p Fs(,)g Fh(REAL)p Fs(,)g Fh(DOUBLE)23 b(PRECISION)p Fs(,)9 b Fh(COMPLEX)p Fs(,)h Fh(DOUBLE)75 1678 y(PRECISION)23 b(COMPLEX)p Fs(,)13 b Fh(LOGICAL)p Fs(,)g Fh(CHARACTER)h Fs(and)h(to)f(un)o(t)o(yp)q(ed)i Fh(BYTE)p Fs(.)d(The)i(p)q(ossible)i (v)m(alues)f(for)e(C)75 1734 y(are)h Fh(MPI)p 226 1734 V 17 w(CHAR)p Fs(,)f Fh(MPI)p 438 1734 V 17 w(SHORT)p Fs(,)g Fh(MPI)p 674 1734 V 17 w(INT)p Fs(,)g Fh(MPI)p 862 1734 V 17 w(LONG)p Fs(,)g Fh(MPI)p 1074 1734 V 17 w(FLOAT)p Fs(,)g Fh(MPI)p 1310 1734 V 17 w(DOUBLE)g Fs(and)h Fh(MPI)p 1645 1734 V 17 w(BYTE)p Fs(.)166 1867 y Fl(Discussion:)166 1916 y Fk(Ma)o(y)i(need)i(to)f(tak)o(e)g(care)h(of)e(kinds)h(in)g(F)m (ortran)f(90)h(and)g(of)f(signed/unsigned)h(in)g(C.)f(More)i(elab)q (orate)75 1966 y(de\014nitions)14 b(b)q(elong)f(to)h(the)h(language)e (binding.)166 2105 y Fs(There)i(are)g(\014v)o(e)h(kinds)g(of)f (bu\013er)g(comp)q(onen)o(ts:)75 2223 y Ft(Con)o(tiguous)j(comp)q(onen) o(t)75 2309 y Fs(A)d(sequence)i(of)d(con)o(tiguous)i(v)m(alues)g(of)f (the)g(same)g(basic)h(t)o(yp)q(e,)f(sp)q(eci\014ed)i(b)o(y)75 2389 y Ft(start)23 b Fs(Initial)17 b(elemen)o(t)75 2478 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fh(count)c Fo(\025)h Fh(0)p Fs(\))75 2567 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 2647 y(Th)o(us,)e(if)h Fh(A)e Fs(is)i(an)f(arra)o(y)f(of)h(in)o(tegers,)g(then)g Fh(A\(1\),)24 b(4,)f(MPI)p 1210 2647 V 17 w(INT)13 b Fs(is)g(the)g(bu\013er)g(comp)q(onen)o(t)h(with)75 2704 y(en)o(tries)i Fh(A\(1\),)23 b(A\(2\),)g(A\(3\),)g(A\(4\))p Fs(.)p eop %%Page: 11 16 11 15 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(11)75 45 y Ft(V)l(ector)17 b(comp)q(onen)o(t)75 137 y Fs(A)f(sequence)i(of)e(equally)i(spaced)f(and)f(equally)i(sized)g (blo)q(c)o(ks)f(of)f(elemen)o(ts)h(of)f(the)g(same)g(basic)h(t)o(yp)q (e,)75 194 y(sp)q(eci\014ed)g(b)o(y)75 300 y Ft(start)23 b Fs(Initial)17 b(elemen)o(t)75 407 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fh(count)c Fo(\025)h Fh(0)p Fs(\))75 513 y Ft(stride)23 b Fs(Num)o(b)q(er)15 b(of)g(elemen)o(ts)h(b)q(et)o(w)o(een)f(the)h(start)e(of)g(eac)o(h)i (blo)q(c)o(k)75 620 y Ft(len)o(blk)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(in)g(eac)o(h)g(blo)q(c)o(k)f(\()p Fh(lenblk)d Fr(>)h Fh(0)p Fs(\))75 727 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 833 y(Assume)c(that)f(an)h (elemen)o(t)h(of)e(t)o(yp)q(e)h Fh(datatype)f Fs(o)q(ccupies)i Fh(m)f Fs(b)o(ytes.)18 b(Then,)12 b(in)g(a)e(b)o(yte)h(addressable)75 890 y(mac)o(hine,)h(the)g(bu\013er)f(comp)q(onen)o(t)g(describ)q(ed)i (b)o(y)e(these)g(\014v)o(e)g(parameters)g(consists)g(of)g(the)g Fh(count)f Fs(v)m(ari-)75 946 y(ables)16 b(at)e(addresses)i Fh(start,)23 b(start+m,)g(...)47 b(,)24 b(start+m*\(lenblk-1\),)d (start+m*stride,)75 1003 y(start+m*\(stride+1\),)g(...,)j (start+m*\(stride+lenblk-)o(1\),)d(...)p Fs(.)h(F)l(or)16 b(example,)h(if)f Fh(A)g Fs(is)h(an)75 1059 y(arra)o(y)d(of)h(in)o (tegers,)g(then)143 1166 y Fo(\017)23 b Fh(A\(1\),)g(5,)g(3,)h(2,)g (MPI)p 622 1166 15 2 v 16 w(INT)17 b Fs(is)g(the)g(bu\013er)g(comp)q (onen)o(t)g(with)g(en)o(tries)g Fh(A\(1\),)24 b(A\(2\),)f(A\(4\),)189 1222 y(A\(5\),)g(A\(7\))p Fs(.)143 1329 y Fo(\017)g Fh(A\(1\),)g(5,)g (0,)h(2,)g(MPI)p 622 1329 V 16 w(INT)17 b Fs(is)g(the)g(bu\013er)g (comp)q(onen)o(t)g(with)g(en)o(tries)g Fh(A\(1\),)24 b(A\(2\),)f(A\(1\),)189 1385 y(A\(2\),)g(A\(1\))p Fs(.)143 1492 y Fo(\017)g Fh(A\(10\),)g(5,)g(-1,)h(2,)f(MPI)p 669 1492 V 17 w(INT)g Fs(is)h(the)g(bu\013er)g(comp)q(onen)o(t)g(with)g (en)o(tries)g Fh(A\(10\),)f(A\(11\),)189 1548 y(A\(9\),)g(A\(10\),)g (A\(8\))p Fs(.)166 1655 y Fh(Contiguous)18 b Fs(is)h(a)g(sp)q(ecial)i (case)e(of)g Fh(vector)p Fs(.)30 b(A)19 b(v)o(ector)g(bu\013er)g(comp)q (onen)o(t)g(can)g(b)q(e)h(used)g(to)75 1711 y(extract)14 b(an)i(arbitrary)e(submatrix)h(of)g(a)g(t)o(w)o(o-dimensional)h (matrix.)75 1850 y Ft(Indexed)h(comp)q(onen)o(t)75 1942 y Fs(A)e(sequence)i(of)d(elemen)o(ts)i(of)f(the)g(same)g(basic)h(t)o (yp)q(e,)f(sp)q(eci\014ed)i(b)o(y)75 2048 y Ft(start)23 b Fs(initial)17 b(elemen)o(t)75 2155 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fh(count)c Fo(\025)h Fh(0)p Fs(\))75 2261 y Ft(arra)o(y)p 199 2261 16 2 v 18 w(of)p 259 2261 V 19 w(indices)23 b Fs(Arra)o(y)11 b(of)g(displacemen)o(ts)i(of)e(the)h(elemen)o(ts)g(in)g(the)g(bu\013er) f(comp)q(onen)o(ts,)h(relativ)o(e)189 2318 y(to)i(the)h(initial)j (elemen)o(t.)75 2424 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 2531 y(F)l(or)c(example,)h(if)g Fh(A)f Fs(is)h(an)f(arra)o(y)f(of)h(in)o(tegers,)h(and)f Fh(I)h Fs(is)f(an)h(arra)o(y)e(with)h(v)m(alues)i Fh(2,)23 b(-1,)h(2,)f(1)12 b Fs(then)75 2587 y Fh(A\(3\),)23 b(4,)h(I,)f(MPI)p 436 2587 15 2 v 17 w(INT)15 b Fs(is)h(the)f(bu\013er)g(comp)q(onen)o(t) g(with)h(en)o(tries)f Fh(A\(5\),)24 b(A\(2\),)f(A\(5\),)g(A\(4\))p Fs(.)166 2647 y Fh(Vector)10 b Fs(can)h(b)q(e)h(though)o(t)e(of)h(as)f (a)h(sp)q(ecial)i(case)e(of)f Fh(indexed)g Fs(in)i(whic)o(h)g(the)f (indices)i(are)e(generated)75 2704 y(with)16 b(a)e(constan)o(t)h (stride.)p eop %%Page: 12 17 12 16 bop 75 -100 a Fs(12)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Ft(Heterogeneous)h (v)o(ector)g(\(h)o(v)o(ector\))g(comp)q(onen)o(t)75 131 y Fs(Same)e(as)g Fh(vector)p Fs(,)f(except)h(that)g Fh(stride)f Fs(is)h(giv)o(en)h(in)g(m)o(ultiple)h(of)e(b)o(ytes,)f(rather)h(than)g (m)o(ultiples)h(of)75 187 y(elemen)o(ts:)75 294 y Ft(start)23 b Fs(Initial)17 b(elemen)o(t)75 387 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fh(count)c Fo(\025)h Fh(0)p Fs(\))75 481 y Ft(stride)23 b Fs(Num)o(b)q(er)15 b(of)g(b)o(ytes)g(b)q(et)o(w)o(een)h(the)f(start)f(of)h(eac)o(h)g(blo)q (c)o(k)75 575 y Ft(len)o(blk)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(in)g(eac)o(h)g(blo)q(c)o(k)f(\()p Fh(lenblk)d Fr(>)h Fh(0)p Fs(\))75 669 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 775 y(Assume)c(that)f(an)h (elemen)o(t)h(of)e(t)o(yp)q(e)h Fh(datatype)f Fs(o)q(ccupies)i Fh(m)f Fs(b)o(ytes.)18 b(Then,)12 b(in)g(a)e(b)o(yte)h(addressable)75 832 y(mac)o(hine,)21 b(the)f(bu\013er)f(comp)q(onen)o(t)h(describ)q(ed) h(b)o(y)e(these)h(\014v)o(e)g(parameters)e(consists)i(of)f(the)h Fh(count)75 888 y Fs(v)m(ariables)j(at)e(addresses)h Fh(start,)h(start+m,)f(...)48 b(,)24 b(start+m*\(lenblk-1\),)d (start+stride,)75 944 y(start+stride+m,)h(...,)h (start+stride+m*\(lenblk-1\),)d(...)p Fs(.)166 1001 y(Consider)c(a)f(C) g(arra)o(y)e(declared)k(as)75 1107 y Fh(struct)23 b({)147 1164 y(char)g(info[3];)147 1220 y(short)g(number;)147 1277 y(double)g(val;)242 1333 y(})71 b(a[100])166 1439 y Fs(Assume)11 b(that)f(all)i(structure)e(comp)q(onen)o(ts)h(are)f(b)o (yte)h(aligned,)i(a)d Fh(short)g Fs(o)q(ccupies)i(t)o(w)o(o)e(b)o (ytes,)h(and)75 1496 y(a)h Fh(double)g Fs(o)q(ccupies)i(eigh)o(t)e(b)o (ytes.)19 b(Then)13 b(the)f(bu\013er)h(comp)q(onen)o(t)f Fh(&a,)24 b(100,)f(13,)g(2,)h(MPI)p 1694 1496 15 2 v 17 w(CHAR)12 b Fs(ex-)75 1552 y(tracts)f(the)h(\014rst)g(t)o(w)o(o)f(c) o(haracters)g(of)h(eac)o(h)g(structure.)19 b(The)12 b(bu\013er)g(comp)q (onen)o(t)g Fh(&\(a.number\),)23 b(100,)75 1609 y(13,)g(1,)h(MPI)p 317 1609 V 17 w(SHORT)19 b Fs(extracts)f(the)i Fh(short)f Fs(\014eld)h(of)f(eac)o(h)h(structure;)h(and)f(the)f(bu\013er)h(comp)q (onen)o(t)75 1665 y Fh(&\(a.val\),)j(100,)g(13,)g(1)18 b Fs(extracts)e(the)i Fh(double)f Fs(\014eld)i(of)e(eac)o(h)g (structure.)27 b(Th)o(us,)18 b(heterogeneous)75 1722 y(v)o(ectors)c(can)i(b)q(e)g(used)f(to)g(extract)f(v)o(ectors)h(of)g (homogeneous)g(v)m(alues)h(from)e(heterogeneous)i(data.)166 1778 y Fh(Vector)e Fs(is)i(a)f(sp)q(ecial)i(case)e(of)g Fh(heterogeneous)22 b(vector)p Fs(.)75 1898 y Ft(Heterogeneous)17 b(indexed)h(\(hindexed\))g(comp)q(onen)o(t)75 1984 y Fs(Same)e(as)f(indexed,)i(except)g(that)e(displacemen)o(ts)i(are)f(in)g (b)o(ytes,)f(rather)h(than)f(m)o(ultiples)j(of)d(elemen)o(t)75 2040 y(size.)75 2147 y Ft(start)23 b Fs(initial)17 b(elemen)o(t)75 2240 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fh(count)c Fo(\025)h Fh(0)p Fs(\))75 2334 y Ft(arra)o(y)p 199 2334 16 2 v 18 w(of)p 259 2334 V 19 w(indices)23 b Fs(Arra)o(y)17 b(of)g(b)o(yte)h(displacemen)o(ts)h(of)e(the)h(elemen) o(ts)g(in)h(the)f(bu\013er)f(comp)q(onen)o(ts,)189 2391 y(relativ)o(e)e(to)g(the)g(initial)j(elemen)o(t.)75 2484 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 2591 y(F)l(or)21 b(example,)k(if)d Fh(a)g Fs(is)h(the)f(C)g(arra)o(y)e (of)i(the)g(previous)h(example,)h(and)e Fh(I)g Fs(is)h(an)f(arra)o(y)f (with)75 2647 y(v)m(alues)14 b Fh(0,)24 b(13,)f(13,)h(27,)f(26)13 b Fs(then)h Fh(&a,)23 b(5,)h(I,)f(MPI)p 1042 2647 15 2 v 17 w(CHAR)13 b Fs(is)g(the)h(bu\013er)f(comp)q(onen)o(t)g(with)h (en)o(tries)75 2704 y Fh(a[0].info[0],)22 b(a[1].info[0],)g (a[1].info[0],)g(a[1].info[1],)g(a[1].info[0])p Fs(.)p eop %%Page: 13 18 13 17 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(13)166 45 y(Lik)o(e)20 b Fh(heterogeneous)i(vector) p Fs(,)e Fh(heterogeneous)i(indexed)d Fs(extends)h(the)f Fh(indexed)g Fs(facilit)o(y)75 102 y(so)c(as)g(to)f(allo)o(w)i(the)f (extraction)g(of)g(homogeneous)g(data)f(from)h(a)g(heterogeneous)g(la)o (y)o(out.)166 234 y Fl(Discussion:)166 284 y Fk(The)10 b(use)g(of)f(displacemen)o(ts)g(in)g(indexed)h(bu\013er)h(comp)q(onen)o (ts)e(has)h(the)g(same)f(e\013ect)i(as)e(indirect)h(addressing)75 334 y(in)o(to)j(an)g(arra)o(y)h(with)f(initial)e(index)j(zero.)19 b(I.e.,)12 b(if)h Fg(a)h Fk(is)f(an)g(arra)o(y)h(with)f(initial)f (index)h(zero,)h(then)g(a)g(bu\013er)g(com-)75 383 y(p)q(onen)o(t)j (de\014ned)h(b)o(y)e(parameters)g Fg(a[0],)21 b(N,)h(i,)f(type)16 b Fk(extract)h(the)g(elemen)o(ts)g Fg(a[i[0]],)j(a[i[1]],)g(...,)75 433 y(a[i[n]])p Fk(.)25 b(This)16 b(\014ts)h(w)o(ell)f(C)h(usage)g(but) f(ill-\014ts)g(F)m(ortran)g(usage)h(\(initial)e(index)i(one\).)26 b(W)m(e)16 b(to)q(ok)g(a)h(decision)75 483 y(of)c(principle)h(to)g (force)g(\\start)g(from)e(zero")j(index)f(arithmetic)f(on)g(F)m(ortran) h(users,)h(rather)f(than)g(ha)o(v)o(e)g(di\013eren)o(t)75 533 y(index)g(arithmetic)f(in)g(the)i(C)e(and)h(F)m(ortran)g(binding)f (of)g(MPI.)166 583 y(The)i(use)h(of)e(b)q(oth)h(bu\013er)h(comp)q(onen) o(ts)f(with)g(b)o(yte)g(and)g(elemen)o(t)f(size)i(displacemen)o(ts)e (is)h(motiv)n(ated)e(b)o(y)75 633 y(the)j(observ)n(ation)f(that)g(some) g(applications)f(require)i(the)g(more)f(general)g(b)o(yte)h (displacemen)o(t,)e(but)i(that)f(most)75 682 y(applications)d(w)o(ould) h(use)i(elemen)o(t)e(size)h(displacemen)o(t,)e(whic)o(h)i(is)f(m)o(uc)o (h)f(more)h(natural)g(and)g(con)o(v)o(enien)o(t)h(\(and)75 732 y(mac)o(hine)i(indep)q(enden)o(t\).)29 b(Th)o(us,)18 b(rather)g(than)g(forcing)e(all)g(users)j(to)e(do)g(b)o(yte)h (arithmetics,)e(w)o(e)i(decided)g(to)75 782 y(pro)o(vide)c(separately)g (the)h(more)e(adv)n(anced)h(function)f(to)h(\\exp)q(ert)h (programmers".)166 832 y(Leslie)f(Hart)g(prop)q(oses)h(to)f(replace)h (\\heterogeneous")g(with)f(\\general".)75 1035 y Fj(2.5.2)55 b(Bu\013er)18 b(op)r(erations)75 1121 y Fs(A)e(bu\013er)g(is)h(describ) q(ed)h(b)o(y)e(an)g(opaque)g Ft(bu\013er)i(descriptor)e Fs(ob)s(ject)g(accessed)g(via)h(a)f Ft(bu\013er)h(han-)75 1178 y(dle)p Fs(.)i(Suc)o(h)13 b(an)f(ob)s(ject)f(is)i(created)f(b)o(y) g Fh(MPI)p 809 1178 15 2 v 17 w(CREATE)p 970 1178 V 16 w(BUFFER)p Fs(.)f(It)h(is)g(asso)q(ciated)g(with)h(successiv)o(e)g (bu\013er)75 1234 y(comp)q(onen)o(ts)i(b)o(y)g(calling)i(in)f (succession)h(one)e(of)g(the)g(functions)h Fh(MPI)p 1285 1234 V 17 w(APPEND)p 1446 1234 V 16 w(xxx)p Fs(,)e(where)i Fh(xxx)f Fs(iden-)75 1291 y(ti\014es)20 b(the)e(t)o(yp)q(e)h(of)g (bu\013er)g(comp)q(onen)o(t)g(app)q(ended)h(to)f(the)f(bu\013er.)31 b(It)19 b(b)q(ecomes)h(a)o(v)m(ailable)g(for)f(use)75 1347 y(in)h(comm)o(unication)g(op)q(erations)g(once)f(committed)h(b)o (y)f(the)h(function)g Fh(MPI)p 1425 1347 V 17 w(COMMIT)p 1586 1347 V 16 w(BUFFER)p Fs(.)e(Once)75 1403 y(committed,)d(the)g (descriptor)h(cannot)f(b)q(e)h(mo)q(di\014ed)g(an)o(y)f(more.)166 1460 y(Note)g(that)f(the)h(commit)g(op)q(eration)g(do)q(es)h(not)f (create)f(y)o(et)h(a)g(message,)f(and)h(do)q(es)h(not)e(bind)j(the)75 1516 y(data)11 b(to)h(b)q(e)h(sen)o(t.)18 b(It)12 b(is)h(a)f(commit)g (for)f(the)h(bu\013er)g(descriptor,)h(not)f(a)f(commit)h(for)g(the)g (bu\013er)g(con)o(ten)o(t.)75 1573 y(A)k(bu\013er)g(descriptor)h(is)g (destro)o(y)o(ed)e(after)h(the)g(completion)h(of)f(the)g(\014rst)g (comm)o(unication)h(op)q(eration)75 1629 y(that)g(uses)h(it,)g(if)g(it) g(is)g(ephemeral,)h(or)e(after)f(it)i(is)g(freed)g(b)o(y)g(a)f(call)i (to)d Fh(MPI)p 1411 1629 V 17 w(FREE)h Fs(and)h(an)o(y)f(p)q(ending)75 1686 y(comm)o(unication)j(op)q(eration)f(that)f(uses)i(it)f(has)g (completed,)h(if)g(it)f(is)g(p)q(ersisten)o(t.)32 b(After)19 b(the)g(bu\013er)75 1742 y(descriptor)d(ob)s(ject)e(has)h(b)q(een)i (destro)o(y)o(ed,)d(the)h(handle)i(is)e(unde\014ned.)166 1834 y Ft(MPI)p 275 1834 16 2 v 18 w(CREA)l(TE)p 510 1834 V 19 w(BUFFER\()j(bu\013er,)e(p)q(ersistence)i(\))166 1926 y Fs(Create)d(a)f(new)i(bu\013er)f(descriptor)h(ob)s(ject.)j(The)c (parameters)g(are:)75 2016 y Ft(OUT)j(bu\013er)k Fs(bu\013er)15 b(handle)75 2108 y Ft(IN)i(p)q(ersistence)23 b Fs(bu\013er)g(p)q (ersistence)h(\(state)e(t)o(yp)q(e)h(with)g(p)q(ossible)h(v)m(alues)g Fh(MPI)p 1559 2108 15 2 v 17 w(PERSISTENT)d Fs(or)189 2164 y Fh(MPI)p 264 2164 V 16 w(EPHEMERAL)p Fs(\).)166 2330 y Fl(Discussion:)166 2380 y Fk(Ma)o(y)d(w)o(an)o(t)g(to)h(pro)o (vide)f(information)e(on)i(maxim)n(um)c(ob)r(ject)20 b(size,)g(and)e(p)q(erhaps)i(user)g(space)f(for)f(the)75 2429 y(ob)r(ject.)166 2479 y(Leslie)g(Hart)g(prop)q(oses)h Fg(MPI)p 631 2479 14 2 v 15 w(CREATE)p 778 2479 V 14 w(BUFFER)p 924 2479 V 15 w(DESC)p Fk(,)d(rather)j(than)f Fg(MPI)p 1351 2479 V 15 w(CREATE)p 1498 2479 V 14 w(BUFFER)p Fk(.)e(More)i(accu-)75 2529 y(rate,)c(but)g(longer.)k(P)o(erhaps)d Fg(MPI)p 617 2529 V 15 w(CREATE)p 764 2529 V 14 w(BUFDESC)p Fk(?)166 2704 y Ft(MPI)p 275 2704 16 2 v 18 w(APPEND)p 520 2704 V 18 w(CONTIGUOUS\()k(bu\013er,)d(start,)h(coun)o(t,)g(datat)o (yp)q(e\))p eop %%Page: 14 19 14 18 bop 75 -100 a Fs(14)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fs(App)q(end)h(a)e (blo)q(c)o(k)g(comp)q(onen)o(t)h(to)e(bu\013er.)20 b(The)c(parameters)e (are:)75 141 y Ft(INOUT)k(bu\013er)k Fs(bu\013er)15 b(handle)75 236 y Ft(IN)i(start)23 b Fs(bu\013er)15 b(comp)q(onen)o(t)h(initial)h (elemen)o(t)f(\(c)o(hoice\))75 332 y Ft(IN)h(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 428 y Ft(IN)i(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 559 y Ft(MPI)p 275 559 16 2 v 18 w(APPEND)p 520 559 V 18 w(VEC\()h(bu\013er,)f(coun)o(t,)i (stride,)f(len)o(blk,)g(datat)o(yp)q(e)h(\))166 651 y Fs(App)q(end)f(a)e(v)o(ector)f(bu\013er)h(comp)q(onen)o(t)g(to)g (bu\013er.)20 b(The)15 b(parameters)g(are:)75 747 y Ft(INOUT)j (bu\013er)k Fs(bu\013er)15 b(handle)75 843 y Ft(IN)i(start)23 b Fs(bu\013er)15 b(comp)q(onen)o(t)h(initial)h(elemen)o(t)f(\(c)o (hoice\))75 938 y Ft(IN)h(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 1034 y Ft(IN)i(stride)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(b)q(et)o(w)o(een)g(the)f (start)f(of)h(eac)o(h)g(blo)q(c)o(k)h(\(in)o(teger\))75 1130 y Ft(IN)h(len)o(blk)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i (in)g(eac)o(h)g(blo)q(c)o(k)g(\(in)o(teger\))75 1226 y Ft(IN)h(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h (\(status\))166 1357 y Ft(MPI)p 275 1357 V 18 w(APPEND)p 520 1357 V 18 w(INDEXED\()d(bu\013er,)g(start,)g(coun)o(t,)h(arra)o(y)p 1395 1357 V 17 w(of)p 1454 1357 V 19 w(indices,)h(datat)o(yp)q(e\))166 1505 y Fs(App)q(end)i(an)e(indexed)i(bu\013er)e(comp)q(onen)o(t)g(to)g (bu\013er.)20 b(The)15 b(parameters)f(are:)75 1601 y Ft(INOUT)k(bu\013er)k Fs(bu\013er)15 b(handle)75 1697 y Ft(IN)i(start)23 b Fs(initial)17 b(p)q(osition)g(for)d(indexing)j (\(c)o(hoice\))75 1793 y Ft(IN)g(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 1888 y Ft(IN)i(arra)o(y)p 277 1888 V 18 w(of)p 337 1888 V 19 w(indices)24 b Fs(arra)o(y)14 b(of)g(displacemen)o(ts)j(of)e(en)o(tries)g(relativ)o(e)h(to)f(start)f (\(arra)o(y)f(of)i(in)o(tegers\))75 1984 y Ft(IN)i(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 2080 y(The)j(index)h(v)m(alues)g(used)g(to)e(create)h(this)g(comp)q (onen)o(t)g(are)g(the)g(v)m(alues)h(pro)o(vided)g(at)e(the)h(call;)75 2136 y(subsequen)o(t)d(up)q(dates)g(of)e(the)i(arra)o(y)e(of)g(indices) k(will)e(not)f(a\013ect)g(the)g(bu\013er)g(descriptor)h(comp)q(onen)o (t.)166 2228 y Ft(MPI)p 275 2228 V 18 w(APPEND)p 520 2228 V 18 w(HVEC\()h(bu\013er,)f(start,)h(coun)o(t,)h(stride,)f(len)o (blk,)g(datat)o(yp)q(e)h(\))166 2321 y Fs(App)q(end)f(a)e (heterogeneous)g(v)o(ector)f(bu\013er)h(comp)q(onen)o(t)h(to)e (bu\013er.)20 b(The)c(parameters)e(are:)75 2417 y Ft(INOUT)k(bu\013er)k Fs(bu\013er)15 b(handle)75 2512 y Ft(IN)i(start)23 b Fs(bu\013er)15 b(comp)q(onen)o(t)h(initial)h(elemen)o(t)f(\(c)o (hoice\))75 2608 y Ft(IN)h(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 2704 y Ft(IN)i(stride)23 b Fs(Num)o(b)q(er)16 b(of)e(b)o(ytes)h(b)q(et)o(w)o(een)h(the)f(start)f (of)h(eac)o(h)g(blo)q(c)o(k)h(\(in)o(teger\))p eop %%Page: 15 20 15 19 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(15)75 45 y Ft(IN)17 b(len)o(blk)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(in)g(eac)o(h)g(blo)q(c)o(k)g (\(in)o(teger\))75 144 y Ft(IN)h(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 279 y Ft(MPI)p 275 279 16 2 v 18 w(APPEND)p 520 279 V 18 w(HINDEXED\()c(bu\013er,)f(start,)i(coun)o(t,)g(arra)o(y)p 1432 279 V 17 w(of)p 1491 279 V 19 w(indices,)g(datat)o(yp)q(e\))166 429 y Fs(App)q(end)k(an)e(heterogeneous)g(indexed)i(bu\013er)e(comp)q (onen)o(t)g(to)g(bu\013er.)20 b(The)15 b(parameters)g(are:)75 528 y Ft(INOUT)j(bu\013er)k Fs(bu\013er)15 b(handle)75 627 y Ft(IN)i(start)23 b Fs(initial)17 b(p)q(osition)g(for)d(indexing)j (\(c)o(hoice\))75 726 y Ft(IN)g(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 825 y Ft(IN)i(arra)o(y)p 277 825 V 18 w(of)p 337 825 V 19 w(indices)24 b Fs(arra)o(y)15 b(of)i(b)o(yte)f(displacemen)o(ts)i(of)f(en)o(tries)g(relativ)o(e)h(to) e(start)f(\(arra)o(y)h(of)g(in-)189 882 y(tegers\))75 981 y Ft(IN)h(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 1115 y Ft(MPI)p 275 1115 V 18 w(COMMIT)p 526 1115 V 18 w(BUFFER\()j(bu\013erin)e(\))166 1209 y Fs(Commit)i(a)g(bu\013er)g(descriptor)h(ob)s(ject.)31 b(The)19 b(ob)s(ject)g(cannot)g(b)q(e)h(c)o(hanged)g(an)o(ymore)e (after)h(it)75 1265 y(is)h(committed.)31 b(A)19 b(bu\013er)g (descriptor)g(ob)s(ject)g(can)g(b)q(e)h(used)f(for)g(comm)o(unication)g (only)h(after)e(it)h(is)75 1322 y(committed.)h(The)15 b(parameters)g(are:)75 1421 y Ft(INOUT)j(bu\013er)k Fs(bu\013er)15 b(descriptor)h(handle)166 1596 y Fl(Discussion:)166 1647 y Fk(The)j(commit)d(function)j(ma)o(y)e(do)h(prepro)q(cessing)j(of)d (the)i(bu\013er)g(descriptor)g(to)f(simplify)d(its)i(use)i(and)75 1697 y(commit)12 b(resources)17 b(in)d(the)i(comm)o(uni)o(cation)c (subsystem)j(for)f(its)h(use.)21 b(The)16 b(commit)11 b(function)k(ma)o(y)e(generate)75 1747 y(a)h(new)g(ob)r(ject)h(and,)e (hence,)i(mo)q(dify)c(the)k(ob)r(ject)f(handle.)166 1798 y(The)i(curren)o(t)i(prop)q(osal)e(do)q(es)g(not)g(allo)o(w)f(to)h (create)h(a)f(bu\013er)h(descriptor,)g(use)g(it)f(in)g(a)f(comm)o (unication)75 1848 y(after)k(committed,)f(then)i(mo)q(dify)d(it)i (partially)e(and)i(use)h(it)f(for)g(another)h(comm)o(unicatio)o(n)c({)j (a)g(new)h(bu\013er)75 1898 y(descriptor)e(has)f(to)g(b)q(e)h(created.) 29 b(Of)16 b(course,)j(the)f(same)e(bu\013er)i(descriptor)g(can)f(b)q (e)h(reused)g(man)o(y)d(times)h(in)75 1947 y(successiv)o(e)k(comm)o (unications)15 b(that)j(send)h(data)f(from)e(the)j(same)e(bu\013ers;)k (e.g.,)d(a)g(new)g(send)h(with)f(a)g(bu\013er)75 1997 y(descriptor)d(handle)f(can)g(b)q(e)h(initiated)e(once)h(the)h (previous)f(send)h(with)e(this)h(handle)g(has)g(completed.)166 2138 y Fs(Example)i(1:)75 2194 y(Consider)g(the)f(follo)o(wing)h (fragmen)o(t)e(of)h(F)l(ortran)f(co)q(de)75 2308 y Fh(DOUBLE)23 b(PRECISION)g(A\(10,20\))75 2365 y(INTEGER)g(B,)g(C\(5,10\))75 2421 y(INTEGER)g(BH)75 2478 y(...)75 2534 y(CALL)g (MPI_CREATE_BUFFER\(BH,)e(MPI_PERSISTENT\))75 2591 y(CALL)i (MPI_APPEND_BLOCK)f(\(BH,)h(B,)h(1,)f(MPI_INT\))75 2647 y(CALL)g(MPI_APPEND_VEC)f(\(BH,)i(A\(1,3\),)e(11,)i(4,)g(2,)f (MPI_DOUBLE\))75 2704 y(CALL)g(MPI_APPEND_INDEX\(BH,)e(C\(3,7\),)i (\(4,2,1\),)g(MPI_INT\))p eop %%Page: 16 21 16 20 bop 75 -100 a Fs(16)613 b Fq(CHAPTER)16 b(2.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fs(Then)g(the)f (bu\013er)g(asso)q(ciated)g(with)h(the)f(handle)i Fh(BH)e Fs(consists)g(of)g(the)g(sequence)h(of)f(v)m(ariables)166 102 y Fh(B,)23 b(A\(1,3\),)g(A\(2,3\),)g(A\(5,3\),)g(A\(6,3\),)g (A\(9,3\),)g(A\(10,3\),)g(A\(3,4\),)g(A\(4,4\),)g(A\(7,4\),)75 158 y(A\(8,4\),)g(A\(1,5\),)g(C\(2,8\),)g(C\(5,7\),)g(C\(4,7\))p Fs(.)166 214 y(A)17 b(message)g(created)g(from)f(this)i(bu\013er)f (will)i(consist)e(of)g(a)g(sequence)h(of)f(one)g(in)o(teger,)h(follo)o (w)o(ed)75 271 y(b)o(y)d(elev)o(en)h(double)h(precision)g(reals,)e (follo)o(w)o(ed)g(b)o(y)g(three)h(in)o(tegers.)166 327 y(Example)g(2:)75 384 y(Supp)q(ose)g(one)g(has)f(an)g(arra)o(y)f(of)h (data)f(declared)i(in)h(C)d(as)75 486 y Fh(struct)23 b({)147 543 y(int)g(type;)147 599 y(double)g(position[3];)147 656 y(double)g(velocity[3];)242 712 y(})h(particle[1000])166 815 y Fs(and)13 b(one)g(w)o(an)o(ts)e(to)i(transfer)f(in)h(one)g (message)g(the)g(n)o(um)o(b)q(er)g(of)f(particles)i(of)e(t)o(yp)q(e)h (1,)g(the)g(n)o(um)o(b)q(er)75 871 y(of)g(particles)h(of)e(t)o(yp)q(e)i (2,)e(and)i(the)f(p)q(ositions)h(of)f(all)h(the)f(particles)h(of)f (these)g(t)o(w)o(o)f(t)o(yp)q(es.)19 b(The)13 b(follo)o(wing)75 928 y(co)q(de)j(fragmen)o(t)e(creates)h(a)g(suitable)h(bu\013er)f (descriptor.)75 1030 y Fh(int)23 b(count1)g(=)h(0;)75 1087 y(int)f(count2)g(=)h(0;)75 1143 y(...)75 1200 y (mpi_create_buffer\()d(&handle,)i(MPI_PERSISTENT\);)75 1256 y(mpi_append_contiguous\()e(&handle,)i(&count1,)f(MPI_INT\);)75 1313 y(mpi_append_contiguous\()f(&handle,)i(&count2,)f(MPI_INT\);)75 1369 y(for)h(\(i=0;)h(i<1000;)f(i++\))g({)147 1426 y(if)g (\(particle[i].type)f(==)h(1\))h({)218 1482 y(count1++;)218 1539 y(mpi_append_contiguous\()d(&handle,)i(&particle[i].position,)e (3,)i(MPI_DOUBLE\);)218 1595 y(})147 1651 y(elseif)g (\(particle[i].type)e(==)j(2\))f({)218 1708 y(count2++;)218 1764 y(mpi_append_contiguous\()e(&handle,)i(&particle[i].position,)e (3,)i(MPI_DOUBLE\);)218 1821 y(})147 1877 y(})75 1934 y(mpi_commit_buffer\(handle\))75 1990 y(...)166 2093 y Fs(The)15 b(bu\013er)g(descriptor)g(handle)h(can)f(no)o(w)f(b)q(e)i (used)f(to)g(send)g(\(rep)q(eatedly\))g(the)g(particle)h(coun)o(ts)75 2149 y(for)i(t)o(yp)q(e)h(1)g(and)g(2,)g(and)g(the)g(p)q(ositions)h(of) f(the)g(particles)h(with)f(these)g(t)o(w)o(o)f(t)o(yp)q(es,)i(as)e (long)h(as)g(the)75 2206 y(t)o(yp)q(es)c(of)g(the)g(particles)h(is)g (not)f(mo)q(di\014ed.)166 2262 y(Supp)q(ose)h(the)f(incoming)i(data)d (is)i(to)e(b)q(e)i(stored)f(at)f(the)i(receiv)o(er)g(in)o(to)f(the)g(t) o(w)o(o)f(coun)o(t)h(v)m(ariables)75 2319 y Fh(count1)e Fs(and)i Fh(count2)e Fs(and)i(in)o(to)f(the)h Fh(position)e Fs(comp)q(onen)o(ts)h(of)g(the)h(initial)h(en)o(tries)f(of)f(the)g (arra)o(y)f(of)75 2375 y(particles.)20 b(The)13 b(follo)o(wing)h(co)q (de)g(generates)e(a)h(suitable)h(bu\013er)f(descriptor)g(for)g(the)g (receiv)o(e)h(op)q(eration)75 2432 y(\(w)o(e)h(assume)g(4)g(b)o(ytes)f (for)h(in)o(teger)g(and)h(8)f(b)o(ytes)g(for)f(double\).)75 2534 y Fh(...)75 2591 y(mpi_create_buffer\()21 b(&handle,)i (MPI_PERSISTENT\);)75 2647 y(mpi_append_contiguous\()e(&handle,)i (&count1,)f(MPI_INT\);)75 2704 y(mpi_append_contiguous\()f(&handle,)i (&count2,)f(MPI_INT\);)p eop %%Page: 17 22 17 21 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(17)75 45 y Fh(mpi_append_hvec\()22 b(&handle,)g(&particle[0].position,)f(1000,)i(52,)h(3,)g(MPI_DOUBLE\);) 75 102 y(mpi_commit_buffer\()d(&handle\))75 158 y(...)166 260 y Fs(A)c(bu\013er)g(handle)h(can)f(b)q(e)h(used)g(for)e(comm)o (unication,)i(ev)o(en)f(if)g(it)h(is)f(not)g(asso)q(ciated)g(with)g(an) o(y)75 317 y(v)m(ariables)12 b(\(i.e.,)f(ev)o(en)g(if)h(it)f(w)o(as)f (not)g(set)h(b)o(y)f(an)o(y)h Fh(MPI)p 978 317 15 2 v 17 w(APPEND)p 1139 317 V 16 w(xxx)f Fs(call\).)19 b(Suc)o(h)12 b(a)e(handle)i(is)g(asso)q(ciated)75 373 y(with)19 b(an)g(empt)o(y)f (bu\013er,)i(and)f(a)f(message)h(created)f(from)h(it)g(con)o(tains)g (no)f(data.)30 b(The)19 b(handle)i(still)75 430 y(need)16 b(b)q(e)g(committed)f(b)q(efore)h(it)f(is)h(used.)166 486 y(The)e(same)f(en)o(try)g(ma)o(y)g(app)q(ear)h(more)f(than)g(once)h (in)g(a)f(bu\013er)h(comp)q(onen)o(t,)g(and)f(distinct)i(bu\013er)75 543 y(comp)q(onen)o(ts)h(ma)o(y)f(o)o(v)o(erlap.)23 b(If)16 b(a)g(bu\013er)g(descriptor)g(with)h(replicated)g(en)o(tries)g(is)f (used)h(to)e(receiv)o(e)i(a)75 599 y(message,)12 b(then)h(the)g (\014nal)g(v)m(alue)h(of)e(an)g(en)o(try)g(that)g(o)q(ccurs)h(more)f (than)g(once)h(in)g(the)g(bu\013er)g(descriptor)75 656 y(is)j(unde\014ned.)166 788 y Fl(Discussion:)166 838 y Fk(It)g(is)g(not)g(clear)h(to)e(me)h(that)g(o)o(v)o(erlaps)g(ha)o(v)o (e)f(an)o(y)h(useful)g(purp)q(ose,)i(and)d(they)i(ma)o(y)d(complicate)h (imple-)75 888 y(men)o(tation.)h(But)f(w)o(e)f(had)g(an)f(o)o(v)o (erwhelming)f(v)o(ote)i(in)f(fa)o(v)o(or)g(of)h(this)g(feature.)75 1092 y Fj(2.5.3)55 b(T)n(yp)r(e)18 b(Matc)n(hing)h(and)g(Data)g(Con)n (v)n(ersion)75 1177 y Ft(T)o(yp)q(e)e(matc)o(hing)75 1263 y Fs(One)f(can)f(think)h(of)f(message)g(transmission)g(as)g (consisting)h(of)f(three)g(phases:)131 1354 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i(send)f(bu\013er)h(and)f(a)g(message)g (is)g(assem)o(bled)131 1447 y(2.)22 b(A)15 b(message)g(is)g (transferred)g(from)f(sender)i(to)f(receiv)o(er)131 1540 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h(incoming)i(message)e (and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o(e)h(bu\013er)166 1631 y(T)o(yp)q(e)k(matc)o(hing)f(has)h(to)e(b)q(e)j(observ)o(ed)e(at)g (eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f (eac)o(h)75 1687 y(v)m(ariable)d(in)f(the)f(sender)h(bu\013er)f(has)g (to)f(matc)o(h)h(the)g(t)o(yp)q(e)h(of)e(the)i(corresp)q(onding)g(en)o (try)f(in)h(the)f(bu\013er)75 1744 y(descriptor;)f(the)g(t)o(yp)q(e)f (of)g(an)g(en)o(try)g(in)h(the)g(bu\013er)f(descriptor)h(used)f(to)g (send)h(a)f(message)g(has)g(to)g(matc)o(h)75 1800 y(the)19 b(t)o(yp)q(e)f(of)g(the)h(corresp)q(onding)g(en)o(try)f(in)h(the)g (bu\013er)f(descriptor)h(used)g(to)f(receiv)o(e)h(the)g(message;)75 1857 y(and)g(the)f(t)o(yp)q(e)g(of)g(eac)o(h)h(v)m(ariable)h(in)f(the)f (receiv)o(e)h(bu\013er)g(has)f(to)g(matc)o(h)g(the)g(corresp)q(onding)h (en)o(try)75 1913 y(in)d(the)g(bu\013er)f(descriptor)h(for)f(this)h (bu\013er.)21 b(A)15 b(program)g(that)g(fails)h(to)f(observ)o(e)g (these)h(three)f(rules)h(is)75 1970 y(erroneous.)166 2026 y(T)l(o)f(de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l (,)j(w)o(e)d(need)i(to)e(deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 2083 y(t)o(yp)q(es)h(of)g(the)h(host)f (language)g(with)h(bu\013er)g(descriptor)f(t)o(yp)q(es;)h(and)g(matc)o (hing)f(of)g(bu\013er)h(descriptor)75 2139 y(t)o(yp)q(es)22 b(at)f(sender)h(and)g(receiv)o(er.)40 b(W)l(e)22 b(restrict)f(ourselv)o (es)i(to)e(the)g(case)h(where)g(the)g(sending)h(and)75 2195 y(receiving)17 b(programs)d(are)h(written)g(in)h(the)f(same)g (language.)166 2252 y(The)e(t)o(yp)q(e)g(of)f(an)h(en)o(try)f(in)h(the) g(sender)g(bu\013er)g(descriptor)g(matc)o(hes)g(the)f(t)o(yp)q(e)h(of)f (an)h(en)o(try)f(in)i(the)75 2308 y(receiv)o(er)g(bu\013er)f (descriptor)g(if)h(they)f(ha)o(v)o(e)g(iden)o(tical)i(names:)k Fh(MPI)p 1233 2308 V 16 w(INT)13 b Fs(matc)o(hes)g Fh(MPI)p 1580 2308 V 17 w(INT)p Fs(,)f Fh(MPI)p 1766 2308 V 17 w(REAL)75 2365 y Fs(matc)o(hes)j Fh(MPI)p 326 2365 V 17 w(REAL)p Fs(,)f(and)h(so)g(on.)166 2421 y(The)k(t)o(yp)q(e)h(of)f(a) g(v)m(ariable)i(in)f(a)f(host)g(program)f(matc)o(hes)h(the)g(t)o(yp)q (e)h(of)e(the)i(en)o(try)f(in)h(a)f(bu\013er)75 2478 y(descriptor)e(if)f(the)h(name)f(of)g(the)g(bu\013er)h(descriptor)f(en) o(try)g(t)o(yp)q(e)h(corresp)q(onds)f(to)g(the)g(basic)h(t)o(yp)q(e)f (of)75 2534 y(the)e(host)f(program)g(v)m(ariable:)21 b(an)14 b(en)o(try)f(with)h(t)o(yp)q(e)g(name)g Fh(MPI)p 1195 2534 V 17 w(INT)f Fs(matc)o(hes)h(a)f(F)l(ortran)g(v)m(ariable)i (of)75 2591 y(t)o(yp)q(e)f Fh(INTEGER)p Fs(,)f(an)h(en)o(try)f(with)i (t)o(yp)q(e)f(name)g Fh(MPI)p 948 2591 V 16 w(REAL)g Fs(matc)o(hes)f(a)h(F)l(ortran)f(v)m(ariable)i(of)f(t)o(yp)q(e)g Fh(REAL)p Fs(,)75 2647 y(and)g(so)g(on.)20 b(There)14 b(is)h(one)f(exception)h(to)f(this)h(last)f(rule:)20 b(An)15 b(en)o(try)e(with)i(t)o(yp)q(e)f(name)g Fh(MPI)p 1683 2647 V 17 w(BYTE)g Fs(can)75 2704 y(b)q(e)j(used)g(to)f(matc)o(h)g (an)o(y)g(b)o(yte)g(of)g(storage)f(\(on)i(a)f(b)o(yte-addressable)h (mac)o(hine\),)g(irresp)q(ectiv)o(e)h(of)e(the)p eop %%Page: 18 23 18 22 bop 75 -100 a Fs(18)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(datat)o(yp)q(e)d (of)h(the)g(v)m(ariable)h(that)e(con)o(tains)h(this)g(b)o(yte.)19 b(The)c(v)m(alue)g(of)e(the)h(message)f(en)o(try)h(will)h(b)q(e)g(the) 75 102 y(binary)h(v)m(alue)g(of)f(the)g(corresp)q(onding)h(b)o(yte)f (in)h(memory)l(.)166 158 y(W)l(e)f(th)o(us)g(ha)o(v)o(e)g(t)o(w)o(o)f (cases:)143 253 y Fo(\017)23 b Fs(Comm)o(unication)17 b(of)g(t)o(yp)q(ed)g(v)m(alues)h(\(e.g.,)e(with)i(datat)o(yp)q(e)e (di\013eren)o(t)i(from)e Fh(MPI)p 1620 253 15 2 v 17 w(BYTE)p Fs(,)g(where)189 310 y(the)d(datat)o(yp)q(es)g(of)g(the)g (corresp)q(onding)h(en)o(tries)g(in)g(the)f(sender)h(program,)f(in)h (the)f(sender)h(bu\013er)189 366 y(descriptor,)21 b(in)g(the)f(receiv)o (er)h(bu\013er)f(descriptor)g(and)g(in)h(the)f(receiv)o(er)h(program)e (should)i(all)189 423 y(matc)o(h.)143 518 y Fo(\017)i Fs(Comm)o(unication)17 b(of)g(un)o(t)o(yp)q(ed)h(v)m(alues)g(\(e.g.,)e (of)h(datat)o(yp)q(e)g Fh(MPI)p 1328 518 V 17 w(BYTE)p Fs(,)f(where)h(the)h(the)f(corre-)189 574 y(sp)q(onding)e(en)o(tries)g (in)h(the)e(sender)i(and)e(the)h(receiv)o(er)g(bu\013er)g(descriptors)g (ha)o(v)o(e)f(b)q(oth)g(datat)o(yp)q(e)189 631 y Fh(MPI)p 264 631 V 16 w(BYTE)p Fs(.)f(In)i(this)f(case,)g(there)g(are)f(no)h (requiremen)o(ts)g(on)g(the)g(datat)o(yp)q(es)f(of)h(the)f(corresp)q (ond-)189 687 y(ing)g(en)o(tries)h(in)g(the)g(sender)f(and)h(the)f (receiv)o(er)h(programs,)e(nor)h(is)h(it)g(required)g(that)f(they)g(b)q (e)h(the)189 744 y(same.)75 867 y Fj(2.5.4)55 b(Data)19 b(con)n(v)n(ersion)75 954 y Fs(W)l(e)c(use)h(the)f(follo)o(wing)h (terminology:)75 1049 y Ft(t)o(yp)q(e)h(con)o(v)o(ersion)22 b Fs(c)o(hanges)13 b(the)g(datat)o(yp)q(e)f(of)h(a)g(v)m(alue,)h(e.g.)k (b)o(y)13 b(rounding)h(a)f Fh(REAL)f Fs(to)g(an)h Fh(INTEGER)p Fs(.)75 1144 y Ft(represen)o(tation)k(con)o(v)o(ersion)22 b Fs(c)o(hanges)16 b(the)h(binary)g(represen)o(tation)f(of)g(a)g(v)m (alue,)h(e.g.)23 b(from)16 b(Hex)189 1200 y(\015oating)f(p)q(oin)o(t)g (to)g(IEEE)g(\015oating)h(p)q(oin)o(t.)166 1296 y(Represen)o(tation)23 b(con)o(v)o(ersion)f(is)g(needed)i(when)e(data)f(is)i(mo)o(v)o(ed)e (across)h(mac)o(hines)g(that)g(use)75 1352 y(di\013eren)o(t)17 b(binary)g(enco)q(dings)h(for)e(the)g(same)h(datat)o(yp)q(e.)23 b(Represen)o(tation)17 b(con)o(v)o(ersion)g(preserv)o(es,)g(to)75 1409 y(the)c(greatest)f(p)q(ossible)j(extend,)f(the)g Ft(v)m(alue)f Fs(represen)o(ted.)20 b(Ho)o(w)o(ev)o(er,)12 b(rounding)i(errors)f(and)g(o)o(v)o(er\015o)o(w)75 1465 y(and)e(under\015o)o(w)h(exceptions)g(ma)o(y)e(o)q(ccur)h(during)h (\015oating)f(p)q(oin)o(t)h(con)o(v)o(ersions;)g(con)o(v)o(ersion)f(of) g(in)o(tegers)75 1521 y(ma)o(y)18 b(also)h(lead)h(to)e(exceptions)i (when)g(a)f(v)m(alue)h(that)e(can)h(represen)o(ted)h(in)g(one)f(system) f(cannot)h(b)q(e)75 1578 y(represen)o(ted)d(in)g(the)f(other)g(system.) k(MPI)c(do)q(es)h(not)f(sp)q(ecify)h(rules)g(for)f(represen)o(tation)g (con)o(v)o(ersion.)166 1635 y(The)e(t)o(yp)q(e)g(matc)o(hing)g(rules)g (imply)h(that)e(MPI)h(comm)o(unication)h(nev)o(er)f(en)o(tails)g(t)o (yp)q(e)g(con)o(v)o(ersion.)75 1691 y(On)20 b(the)f(other)f(hand,)i (MPI)f(requires)h(that)e(a)h(represen)o(tation)g(con)o(v)o(ersion)g(b)q (e)h(p)q(erformed)f(when)g(a)75 1748 y(t)o(yp)q(ed)i(v)m(alue)h(is)f (transferred)f(across)g(en)o(vironmen)o(ts)h(that)f(use)h(di\013eren)o (t)f(represen)o(tations)h(for)f(the)75 1804 y(datat)o(yp)q(e)15 b(of)h(this)g(v)m(alue.)24 b(An)16 b(exception)h(o)q(ccurring)g(during) g(represen)o(tation)f(con)o(v)o(ersion)g(results)g(in)75 1860 y(a)f(failure)h(of)f(the)g(comm)o(unication;)g(an)h(error)e(o)q (ccurs)h(either)h(in)g(the)f(send)h(op)q(eration,)f(or)g(the)g(receiv)o (e)75 1917 y(op)q(eration,)g(or)g(b)q(oth.)166 1974 y(If)g(a)g(v)m (alue)h(sen)o(t)f(in)h(a)e(message)h(is)g(un)o(t)o(yp)q(ed)h(\(i.e.,)e (of)h(t)o(yp)q(e)g Fh(MPI)p 1280 1974 V 17 w(BYTE)p Fs(\),)f(then)h (the)g(binary)h(repre-)75 2030 y(sen)o(tation)g(of)g(the)h(b)o(yte)f (stored)g(at)f(the)i(receiv)o(er)g(is)g(iden)o(tical)h(to)e(the)g (binary)h(represen)o(tation)g(of)f(the)75 2087 y(b)o(yte)e(loaded)i(at) d(the)i(sender.)20 b(This)15 b(holds)h(true,)e(whether)h(sender)g(and)f (receiv)o(er)i(run)e(in)i(the)e(same)g(or)75 2143 y(in)i(distinct)g(en) o(vironmen)o(ts;)f(no)h(represen)o(tation)f(con)o(v)o(ersion)g(is)h (required.)166 2200 y(Note)d(that)g(no)g(con)o(v)o(ersions)h(ev)o(er)f (o)q(ccur)h(when)g(an)f(MPI)g(program)g(executes)h(in)g(a)f (homogeneous)75 2256 y(system,)i(where)h(all)g(pro)q(cesses)g(run)g(in) g(the)g(same)f(en)o(vironmen)o(t.)22 b(Also)16 b(note)f(the)h (di\013eren)o(t)f(b)q(eha)o(vior)75 2313 y(of)e Fh(MPI)p 200 2313 V 17 w(BYTE)f Fs(and)i(of)e Fh(MPI)p 533 2313 V 17 w(CHAR)p Fs(.)h(A)g(bu\013er)g(descriptor)h(en)o(try)f(with)g (datat)o(yp)q(e)g(of)g Fh(MPI)p 1587 2313 V 16 w(CHAR)g Fs(can)g(only)75 2369 y(matc)o(h)d(a)h(F)l(ortran)f(v)m(ariable)i(of)f (t)o(yp)q(e)g Fh(CHAR)p Fs(;)f(and)h(represen)o(tation)g(con)o(v)o (ersion)g(ma)o(y)f(o)q(ccur)h(when)h(v)m(alues)75 2426 y(of)j(t)o(yp)q(e)g Fh(MPI)p 304 2426 V 17 w(CHAR)f Fs(are)h (transferred.,)f(e.g.,)g(from)h(an)g(EBCDIC)g(enco)q(ding)i(to)d(an)h (ASCI)q(I)i(enco)q(ding.)166 2482 y(Consider)f(the)f(follo)o(wing)h (examples.)75 2591 y Fh(!)48 b(FIRST)23 b(EXAMPLE)75 2647 y(INTEGER)g(A\(100\))75 2704 y(...)p eop %%Page: 19 24 19 23 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(19)75 45 y Fh(CALL)23 b(MPI_CREATE_BUFFER\(BH,)e (MPI_PERSISTENT\))75 102 y(CALL)i(MPI_APPEND_CONTIGUOUS)e(\(BH,)j (A\(1\),)f(1,)g(MPI_CHAR\))166 205 y Fs(This)16 b(program)f(is)i (erroneous,)e(since)j(the)e(t)o(yp)q(e)g(of)f(the)h(bu\013er)g (descriptor)h(do)q(es)f(not)f(matc)o(h)h(the)75 262 y(t)o(yp)q(e)g(of)g (the)g(v)m(ariable)i(in)f(the)f(bu\013er.)23 b(The)16 b(b)q(eha)o(vior)h(or)e(this)i(program)e(is)i(unde\014ned.)24 b(A)16 b(desirable)75 318 y(b)q(eha)o(vior)d(is)h(for)e(that)g(error)g (to)h(b)q(e)g(detected)h(\(at)e(compile)i(time)f(or)g(run)g(time\))g (and)g(result)g(in)h(an)f(error)75 375 y(condition.)75 478 y Fh(!)48 b(SECOND)23 b(EXAMPLE)75 535 y(INTEGER)g(A\(100\))75 591 y(...)75 648 y(CALL)g(MPI_CREATE_BUFFER\(BH,)e(MPI_PERSISTENT\))75 704 y(CALL)i(MPI_APPEND_CONTIGUOUS)e(\(BH,)j(A\(1\),)f(1,)g(MPI_BYTE\)) 166 808 y Fs(This)16 b(program)f(is)h(correct.)k(The)c(b)o(yte)g(that)e (will)k(b)q(e)e(transmitted)f(in)i(a)e(message)g(created)h(with)75 864 y(this)i(bu\013er)g(descriptor)g(is)g(the)g(\014rst)g(b)o(yte)f(in) i(the)e(storage)g(o)q(ccupied)j(b)o(y)d Fh(A\(1\))p Fs(.)27 b(The)18 b(v)m(alue)h(of)f(this)75 921 y(b)o(yte)d(is)h(implemen)o (tation)h(dep)q(enden)o(t,)f(and)g(will)h(v)m(ary)e(according)h(to)f (the)g(n)o(um)o(b)q(er)h(of)f(b)o(ytes)g(used)h(to)75 977 y(store)e(F)l(ortran)h(v)m(ariables)h(of)f(t)o(yp)q(e)g Fh(INTEGER)f Fs(and)i(the)f(b)o(yte)g(ordering)g(on)h(the)f(host)g(mac) o(hine.)75 1081 y Fh(!)48 b(THIRD)23 b(EXAMPLE)75 1137 y(INTEGER)g(A\(100\))75 1194 y(...)75 1250 y(CALL)g (MPI_CREATE_BUFFER\(BH1,)e(MPI_PERSISTENT\))75 1307 y(CALL)i (MPI_APPEND_CONTIGUOUS)e(\(BH1,)i(A\(1\),)h(1,)f(MPI_INTEGER\))75 1363 y(...)75 1420 y(CALL)g(MPI_CREATE_BUFFER\(BH2,)e(MPI_PERSISTENT\)) 75 1476 y(CALL)i(MPI_APPEND_CONTIGUOUS)e(\(BH2,)i(A\(1\),)h(4,)f (MPI_BYTE\))166 1580 y Fs(Consider)16 b(the)f(follo)o(wing)h(three)f (cases:)131 1672 y(1.)22 b(A)d(message)g(is)h(sen)o(t)f(using)h (bu\013er)f(descriptor)h(handle)h Fh(BH1)e Fs(and)g(receiv)o(ed)i (using)f(bu\013er)f(de-)189 1728 y(scriptor)e(handle)h Fh(BH2)p Fs(.)25 b(The)18 b(program)e(is)h(erroneous,)h(since)g(the)f (sender)h(and)f(receiv)o(er)h(bu\013er)189 1785 y(descriptors)g(don't)g (matc)o(h.)28 b(The)19 b(b)q(eha)o(vior)f(of)g(the)h(program)e(is)h (unde\014ned,)j(and)d(a)g(p)q(ossible)189 1841 y(\(desirable)e(but)f (unlik)o(ely\))j(b)q(eha)o(vior)d(is)h(for)f(an)g(error)f(condition)j (to)d(o)q(ccur.)131 1934 y(2.)22 b(Both)17 b(sender)h(and)g(receiv)o (er)g(use)g(bu\013er)g(descriptor)g(handle)h Fh(BH1)p Fs(.)26 b(The)18 b(program)f(is)h(correct,)189 1991 y(and)h(an)f(in)o (teger)h(v)m(alue)h(is)f(transmitted)g(from)f(sender)h(to)f(receiv)o (er.)31 b(The)19 b(v)m(alue)h(assigned)g(to)189 2047 y Fh(A\(1\))e Fs(at)g(the)g(receiv)o(er)i(is)f(iden)o(tical)h(to)e(the) h(v)m(alue)h(of)e Fh(A\(1\))g Fs(at)g(the)g(sender,)i(but)f(the)f (binary)189 2104 y(represen)o(tation)g(of)g(this)g(v)m(alue)i(ma)o(y)d (b)q(e)i(di\013eren)o(t,)g(if)f(sender)h(and)f(receiv)o(er)h(run)g(in)g (di\013eren)o(t)189 2160 y(en)o(vironmen)o(ts.)131 2253 y(3.)j(Both)17 b(sender)h(and)g(receiv)o(er)g(use)g(bu\013er)g (descriptor)g(handle)h Fh(BH2)p Fs(.)26 b(The)18 b(program)f(is)h (correct.)189 2310 y(Assume)13 b(that)f(b)q(oth)h(sender)h(and)f (receiv)o(er)h(use)f(four)g(b)o(ytes)f(to)h(store)f(in)o(tegers.)19 b(Then)14 b(the)f(v)m(alue)189 2366 y(stored)h(in)i Fh(A\(1\))e Fs(at)h(the)g(receiv)o(er)h(has)e(the)i(same)e(binary)i(represen)o (tation)f(as)g(the)g(v)m(alue)h(of)e Fh(A\(1)189 2423 y Fs(at)i(the)i(sender.)26 b(If)18 b(the)f(same)g(b)o(yte)g(ordering)h (and)g(the)f(same)g(enco)q(ding)i(of)e(in)o(teger)g(v)m(alues)h(is)189 2479 y(used)d(at)g(b)q(oth)g(ends)h(then)f Fh(A\(1\))f Fs(at)h(the)g(receiv)o(er)h(is)f(set)g(to)g(the)g(v)m(alue)h(of)f Fh(A\(1\))f Fs(at)h(the)g(sender;)189 2536 y(otherwise)g(the)g(same)g (32)g(bits)g(ma)o(y)g(enco)q(de)h(di\013eren)o(t)g(v)m(alues.)166 2704 y Fl(Implemen)o(tati)o(on)c(note:)p eop %%Page: 20 25 20 24 bop 75 -100 a Fs(20)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fk(The)e(curren)o (t)h(de\014nition)e(do)q(es)h(not)g(require)g(messages)g(to)f(carry)i (t)o(yp)q(e)f(information.)h(A)e(message)h(can)f(b)q(e)75 95 y(comp)q(osed)h(and)h(sen)o(t)h(using)e(only)g(the)h(information)d (in)i(the)i(sender)g(bu\013er)g(descriptor,)g(and)e(can)h(b)q(e)h (receiv)o(ed)75 145 y(and)c(stored)h(using)f(only)g(the)h(information)c (in)j(the)h(receiv)o(er)h(bu\013er)f(descriptor.)19 b(If)11 b(messages)i(are)f(sen)o(t)i(b)q(et)o(w)o(een)75 195 y(di\013eren)o(t)g(mac)o(hines)f(then)g(one)h(can)f(either)h(use)g(a)f (\\univ)o(ersal")g(data)g(enco)q(ding)g(for)g(messages,)g(use)h(kno)o (wledge)75 244 y(of)g(the)i(receiv)o(er)h(en)o(vironmen)o(t)d(in)g (order)i(to)f(con)o(v)o(ert)h(data)e(at)h(the)h(sender,)h(or)e(use)g (kno)o(wledge)g(of)g(the)g(sender)75 294 y(en)o(vironmen)o(t)j(in)h (order)i(to)e(con)o(v)o(ert)h(data)f(at)h(the)g(receiv)o(er.)36 b(In)20 b(either)g(case)g(the)h(lo)q(cal)d(bu\013er)j(descriptor)75 344 y(can)d(b)q(e)h(used)g(to)f(deriv)o(e)h(the)f(t)o(yp)q(es)h(of)f (the)h(v)n(alues)e(transferred.)33 b(Ho)o(w)o(ev)o(er,)19 b(additional)d(t)o(yp)q(e)j(information)75 394 y(carried)c(b)o(y)e (messages)h(can)g(pro)o(vide)g(b)q(etter)h(error)g(detection.)k(It)14 b(is)g(p)q(ossible)g(to)g(add)f(alw)o(a)o(ys)g(suc)o(h)i(additional)75 444 y(t)o(yp)q(e)h(information,)c(for)j(b)q(etter)i(error)f(c)o(hec)o (king,)f(or)g(add)g(it)g(when)h(running)f(in)f(\\debug")h(mo)q(de,)f (where)j(some)75 493 y(p)q(erformance)d(is)g(sacri\014ced)h(for)e (added)h(error)h(c)o(hec)o(king,)f(etc.)166 709 y Fl(Discussion:)166 760 y Fk(There)j(is)e(no)h(agreemen)o(t)f(y)o(et)h(on)g(the)g(need)h (for)e(a)g(de\014nition)h(and,)f(if)g(needed,)i(on)f(the)g(righ)o(t)f (de\014nition)75 810 y(of)j(t)o(yp)q(e)i(matc)o(hing)d(and)i(con)o(v)o (ersion)g(b)q(et)o(w)o(een)i(programs)d(written)h(in)g(di\013eren)o(t)h (languages.)33 b(The)19 b(natural)75 859 y(in)o(terpretation)f(of)g (\\send)h(from)d(C)i(and)g(receiv)o(e)h(in)e(F)m(ortran")h(is)g(either) h(\\send)g(from)d(C,)h(receiv)o(e)j(in)d(C)h(and)75 909 y(con)o(v)o(ert)11 b(to)g(F)m(ortran",)f(or)h(\\con)o(v)o(ert)g(to)f(F) m(ortran,)h(send)h(from)c(F)m(ortran)j(and)f(receiv)o(e)i(in)f(F)m (ortran".)16 b(Ho)o(w)o(ev)o(er,)11 b(the)75 959 y(corresp)q(ondence)j (b)q(et)o(w)o(een)e(F)m(ortran)e(t)o(yp)q(es)i(and)e(C)h(t)o(yp)q(es)g (is)g(implemen)o(tatio)o(n)d(dep)q(enden)o(t.)19 b(Th)o(us,)11 b(the)g(diagram)75 1009 y(sho)o(wn)k(b)q(elo)o(w)f(is)h(not)f(comm)o (utativ)o(e,)e(and)j(the)g(t)o(w)o(o)f(in)o(terpretations)i(of)e(C)g (to)h(F)m(ortran)g(comm)o(unicati)o(on)d(ma)o(y)75 1059 y(lead)i(to)f(di\013eren)o(t)i(results.)251 1098 y(mac)o(hine)d(1)300 b(mac)o(hine)13 b(2)544 1148 y(send)244 1198 y(C)h(program)121 b Ff(!)i Fk(C)14 b(program)332 1298 y Ff(#)463 b(#)544 1348 y Fk(send)191 1397 y(F)m(ortran)14 b(program)67 b Ff(!)j Fk(F)m(ortran)13 b(program)166 1446 y(A)e(go)q(o)q(d)f (solution)f(w)o(ould)h(mak)o(e)f(this)i(diagram)d(comm)o(utativ)o(e.)14 b(Otherwise,)e(one)f(needs)h(to)e(pic)o(k)h(one)f(of)g(the)75 1496 y(t)o(w)o(o)15 b(paths)g(as)g(the)h(canonical)e(de\014nition)h(of) g(in)o(terlanguage)f(comm)o(unication.)k(If)d(the)h(upp)q(er)g(path)f (is)g(c)o(hosen)75 1546 y(\(message)f(carries)i(sender)g(datat)o(yp)q (es\),)f(then)g(the)h(receiv)o(er)g(of)e(a)g(message)g(needs)i(to)e (kno)o(w)h(the)g(language)e(the)75 1595 y(message)j(w)o(as)f(sen)o(t)i (from;)e(if)g(the)i(lo)o(w)o(er)e(path)h(is)g(c)o(hosen,)h(\(message)f (carries)h(receiv)o(er)g(datat)o(yp)q(es\),)g(then)f(the)75 1645 y(sender)f(needs)h(to)d(kno)o(w)h(the)g(language)f(the)i(message)e (is)h(receiv)o(ed)h(in)o(to.)166 1696 y(A)d(p)q(ossible)h(solution)f (\(prop)q(osed)h(b)o(y)g(Dann)o(y)e(Nessett\))k(is)d(to)h(lea)o(v)o(e)f (to)g(the)h(user)h(the)f(c)o(hoice)g(of)f(sp)q(ecifying)75 1746 y(whether)20 b(to)e(send)h(\\F)m(ortran)f(datat)o(yp)q(es")h(or)f (\\C)g(datat)o(yp)q(es",)h(in)f(the)h(ab)q(o)o(v)o(e)f(situation.)30 b(One)19 b(w)o(ould)f(still)75 1795 y(require)c(exact)f(matc)o(hing)e (of)h(send)h(bu\013er)h(descriptors)g(and)f(receiv)o(e)h(bu\013er)g (descriptors.)19 b(Ho)o(w)o(ev)o(er,)13 b(a)f(F)m(ortran)75 1845 y(program)k(could)i(use)h(a)f(bu\013er)h(descriptor)h(with)d(\\C)h (datat)o(yp)q(es",)h(and)f(vice)h(v)o(ersa.)31 b(The)18 b(standard)h(w)o(ould)75 1895 y(sp)q(ecify)d(whic)o(h)f(matc)o(hing)e (of)i(F)m(ortran)g(and)g(C)g(datat)o(yp)q(es)h(are)g(legal.)21 b(F)m(or)14 b(example,)g(it)h(w)o(ould)f(b)q(e)i(correct)h(to)75 1945 y(send)f(a)f(F)m(ortran)g(v)n(ariable)f(of)g(t)o(yp)q(e)i Fg(REAL)e Fk(using)h(a)g(bu\013er)h(descriptor)g(of)f(t)o(yp)q(e)h Fg(MPI)p 1435 1945 14 2 v 15 w(FLOAT)p Fk(.)d(Data)i(con)o(v)o(ersion) 75 1995 y(migh)o(t)d(b)q(e)i(required)h(if)e(C)h(\015oat)g(n)o(um)o(b)q (ers)f(and)h(F)m(ortran)f(real)h(n)o(um)o(b)q(ers)g(do)g(not)f(use)i (the)g(same)e(presen)o(tation.)166 2045 y(Another)e(\(non\)solution)f (is)h(to)g(ignore)f(the)h(issue)h(and)e(let)h(the)g(b)q(eha)o(vior)g (of)f(C)g(to)h(F)m(ortran)f(comm)o(unication)75 2095 y(b)q(e)h(unde\014ned.)19 b(The)11 b(user)h(can)f(alw)o(a)o(ys)e(matc)o (h)h(language)g(at)g(sending)h(and)g(receiving)g(end)g(if)f(he)h(or)g (she)g(is)g(willing)75 2145 y(to)j(struggle)g(with)f(the)i(issue)g(of)e (calling)f(MPI)i(C)g(functions)g(from)e(F)m(ortran)i(or)g(vice-v)o (ersa.)75 2375 y Fm(2.6)70 b(Receiv)n(e)20 b(Criteria)75 2478 y Fs(The)14 b(selection)h(of)e(a)g(message)g(b)o(y)g(a)h(receiv)o (e)g(op)q(eration)g(is)g(done)g(uniquely)h(according)f(to)f(the)h(v)m (alue)g(of)75 2534 y(the)e(message)g(en)o(v)o(elop)q(e.)20 b(The)13 b(receiv)o(e)g(op)q(eration)f(sp)q(eci\014es)j(an)d Ft(en)o(v)o(elop)q(e)i(pattern)p Fs(;)f(a)f(message)g(can)75 2591 y(b)q(e)17 b(receiv)o(ed)g(b)o(y)e(that)h(receiv)o(e)g(op)q (eration)g(only)h(if)f(its)g(en)o(v)o(elop)q(e)h(matc)o(hes)f(that)f (pattern.)21 b(A)16 b(pattern)75 2647 y(sp)q(eci\014es)e(v)m(alues)f (for)e(the)h Fh(source)p Fs(,)f Fh(tag)g Fs(and)h Fh(context)f Fs(\014elds)i(of)f(the)g(message)f(en)o(v)o(elop)q(e.)20 b(In)12 b(addition,)75 2704 y(the)17 b(v)m(alue)h(for)f(the)g Fh(dest)g Fs(\014eld)h(is)f(set,)g(implicitl)q(y)l(,)j(to)c(b)q(e)i (equal)g(to)e(the)h(receiving)i(pro)q(cess)e(id.)27 b(The)p eop %%Page: 21 26 21 25 bop 75 -100 a Fq(2.7.)34 b(COMMUNICA)l(TION)16 b(MODE)1073 b Fs(21)75 45 y(receiv)o(er)13 b(ma)o(y)e(sp)q(ecify)i(a)f Fh(DONTCARE)e Fs(v)m(alue)j(for)f Fh(source)p Fs(,)f(or)g Fh(tag)p Fs(,)h(indicating)i(that)d(an)o(y)h(source)g(and/or)75 102 y(tag)k(are)h(acceptable.)26 b(It)17 b(cannot)g(sp)q(ecify)h(a)f (DONTCARE)g(v)m(alue)h(for)e Fh(context)g Fs(or)h Fh(dest)p Fs(.)24 b(Th)o(us,)17 b(a)75 158 y(message)d(can)h(b)q(e)h(receiv)o(ed) f(b)o(y)g(a)f(receiv)o(e)i(op)q(eration)f(only)g(if)g(it)g(is)h (addressed)f(to)f(the)h(receiving)h(task,)75 214 y(has)h(a)g(matc)o (hing)g(con)o(text,)g(has)g(matc)o(hing)g(source)g(unless)i (source=DONTCARE)f(in)g(the)f(pattern,)75 271 y(and)g(has)f(a)g(matc)o (hing)g(tag)g(unless)h(tag=DONTCARE)f(in)h(the)g(pattern.)22 b(DONTCARE)17 b(v)m(alues)h(for)75 327 y(source)d(or)f(tag)f(are)h(sp)q (eci\014ed)j(via)e(a)f(named)h(constan)o(t)f(for)f(source)i(don)o (tcare)f(and)h(a)f(named)h(constan)o(t)75 384 y(for)g(tag)f(don)o (tcare.)166 442 y(The)g(length)h(of)e(the)h(receiv)o(ed)h(message)e(m)o (ust)h(b)q(e)g(less)h(then)f(or)f(equal)i(the)f(length)h(of)e(the)h (receiv)o(e)75 498 y(bu\013er.)31 b(I.e.,)19 b(all)g(incoming)h(data)e (m)o(ust)h(\014t,)g(without)f(truncation,)i(in)o(to)f(the)f(receiv)o(e) i(bu\013er.)31 b(It)18 b(is)75 555 y(erroneous)e(to)f(receiv)o(e)h(a)g (message)f(whic)o(h)i(length)f(exceed)h(the)f(receiv)o(e)h(bu\013er,)e (and)h(the)g(outcome)f(of)75 611 y(program)f(where)i(this)f(o)q(ccurs)h (is)f(undetermined.)75 762 y Fm(2.7)70 b(Comm)n(unication)20 b(Mo)r(de)75 866 y Fs(A)15 b(send)h(op)q(eration)f(can)h(o)q(ccur)f(in) h(one)g(of)f(three)g(mo)q(des:)75 966 y Ft(ST)l(AND)o(ARD)22 b Fs(The)15 b(send)h(ma)o(y)e(start)g(whether)i(or)e(not)h(a)g(matc)o (hing)g(receiv)o(e)h(has)f(b)q(een)i(p)q(osted.)75 1065 y Ft(READ)o(Y)22 b Fs(The)15 b(send)h(ma)o(y)f(start)f(only)h(if)h(a)f (matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)i(p)q(osted.)75 1165 y Ft(SECURE)24 b Fs(The)19 b(send)h(op)q(eration)f(will)i(not)e (complete)h(successfully)h(un)o(til)g(it)e(is)h(guaran)o(teed)f(that) 189 1221 y(the)c(message)g(sen)o(t)g(will)h(b)q(e)g(receiv)o(ed.)166 1321 y(The)21 b(start)f(and)h(completion)h(of)e(a)h Ft(standard)g Fs(send)h(do)q(es)f(not)f(dep)q(end)j(on)e(the)g(status)f(of)g(a)75 1377 y(matc)o(hing)15 b(receiv)o(e.)166 1435 y(A)e Ft(ready)i(send)e Fs(can)g(start)f(only)i(if)f(a)g(matc)o(hing)g(receiv)o(e)h(is)g (already)f(p)q(osted;)h(otherwise)g(the)f(op-)75 1491 y(eration)h(is)g(erroneous)g(and)g(its)g(outcome)g(is)g(unde\014ned.)21 b(In)15 b(some)e(systems,)h(this)g(allo)o(ws)g(the)g(remo)o(v)m(al)75 1548 y(of)g(a)g(hand-shak)o(e)h(op)q(eration)f(that)g(is)h(otherwise)f (required,)h(and)g(results)g(in)g(impro)o(v)o(ed)f(p)q(erformance.)75 1604 y(Its)h(completion)h(do)q(es)g(not)f(dep)q(end)i(on)e(the)g (status)f(of)h(a)g(matc)o(hing)g(receiv)o(e.)166 1662 y(A)e Ft(secure)h(send)f Fs(will)i(complete)e(if)h(a)f(matc)o(hing)g (receiv)o(e)h(is)f(p)q(osted,)g(and)h(the)f(receiv)o(e)h(op)q(eration) 75 1719 y(is)19 b(committed)g(to)f(receiv)o(e)i(the)f(message)g(sen)o (t)f(b)o(y)h(the)g(secure)g(send.)32 b(\(I.e.,)19 b(the)g(send)g(and)g (receiv)o(e)75 1775 y(op)q(erations)12 b(ha)o(v)o(e)g(already)g(b)q (een)h(matc)o(hed,)f(the)g(receiv)o(e)h(op)q(eration)f(has)g(already)g (started)f(and)h(cannot)75 1832 y(b)q(e)21 b(cancelled)i(an)o(ymore.\)) 36 b(The)21 b(start)e(of)i(a)f(secure)h(send)h(do)q(es)f(not)f(dep)q (end)j(on)d(the)h(status)f(of)g(a)75 1888 y(matc)o(hing)15 b(receiv)o(e.)166 1946 y(A)f(receiv)o(e)i(op)q(eration)e(can)h(o)q (ccur)g(in)g(one)f(of)g(t)o(w)o(o)f(mo)q(des:)20 b Ft(ST)l(AND)o(ARD)13 b Fs(and)i Ft(SECURE)p Fs(.)g(A)75 2002 y(secure)g(send)g(can)g(b)q(e)g (matc)o(hed)f(only)h(b)o(y)f(a)g Ft(secure)i(receiv)o(e)p Fs(;)e(a)g(program)f(where)i(a)f(secure)h(send)g(can)75 2059 y(b)q(e)j(matc)o(hed)g(b)o(y)g(a)f(standard)h(receiv)o(e)g(is)h (erroneous)e(and)h(its)g(b)q(eha)o(vior)h(is)f(unde\014ned.)29 b(The)18 b(secure)75 2115 y(receiv)o(e)e(can)f(b)q(e)h(p)q(osted)g (either)g(b)q(efore)f(or)g(after)f(the)i(matc)o(hing)f(send.)166 2173 y(A)f(standard)g(send)h(and)g(a)f(ready)g(send)h(can)f(b)q(oth)h (b)q(e)g(matc)o(hed)f(only)h(b)o(y)f(a)g Ft(standard)j(receiv)o(e)p Fs(;)75 2230 y(a)c(program)g(where)h(a)g(secure)g(receiv)o(e)h(can)f (matc)o(h)f(a)g(nonsecure)i(send)f(is)h(erroneous)e(and)h(its)g(b)q (eha)o(vior)75 2286 y(is)j(unde\014ned.)25 b(The)17 b(receiv)o(e)g(has) f(to)g(b)q(e)h(p)q(osted)f(ahead)h(of)f(the)g(send)h(for)f(a)g(ready)g (send)h(and)g(can)f(b)q(e)75 2342 y(p)q(osted)f(either)h(b)q(efore)g (or)f(after)f(the)h(send)h(is)g(p)q(osted)f(for)g(a)g(standard)f(send.) 166 2476 y Fl(Discussion:)166 2534 y Fk(I)f(got)g(criticisms)g(ab)q (out)g(the)h(previous)g(notation)e(\(REGULAR\))h(and)g(the)h(curren)o (t)h(one)e(\(ST)m(AND)o(ARD\).)75 2591 y(I)k(can)f(also)g(try)h (NORMAL,)g(DEF)-5 b(A)o(UL)m(T,)16 b(...)25 b(W)m(e)16 b(should)h(probably)f(v)o(ote,)h(at)f(some)g(p)q(oin)o(t.)26 b(Also,)16 b(Lyndon)75 2647 y(prop)q(oses)h(to)e(use)i(SYNCHR)o(ONOUS,) f(rather)g(than)g(secure,)h(since)g(the)f(other)g(mo)q(des)f(can)h (also)f(o\013er)h(secure)75 2704 y(comm)o(unication)10 b(with)k(the)g(righ)o(t)g(use.)p eop %%Page: 22 27 22 26 bop 75 -100 a Fs(22)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fl(Implemen)o(tati) o(on)c(note:)166 97 y Fk(The)18 b(curren)o(t)g(de\014nition)f(assumes)g (it)g(is)g(the)h(user)h(resp)q(onsibilit)o(y)e(to)g(prev)o(en)o(t)h (matc)o(hing)d(of)i(a)g(regular)75 147 y(send)12 b(with)f(a)f(secure)j (receiv)o(e)f(and)f(vice-v)o(ersa.)18 b(Otherwise,)13 b(messages)e(need)h(b)q(e)f(tagged)g(as)g(standard)g(or)g(secure.)166 199 y(Assume)18 b(an)f(implemen)o(tation)e(of)i(MPI)h(comm)o(unication) d(that)j(a)o(v)o(oids)f(retries.)31 b(What)18 b(proto)q(cols)g(can)75 249 y(b)q(e)f(used?)27 b(Since)17 b(a)g(receiv)o(e)h(ma)o(y)c(ha)o(v)o (e)j(a)f(wildcard)g(source)i(\014eld,)f(the)g(\014rst)g(step)h(in)e (the)h(proto)q(col)g(m)o(ust)e(b)q(e)75 298 y(executed)h(b)o(y)d(the)i (sender.)166 350 y(F)m(or)e(a)h(ready)g(send,)h(this)f(\014rst)g(step)h (ma)o(y)d(in)o(v)o(olv)o(e)h(sending)h(the)h(data,)e(since)i(the)f (receiv)o(er)i(is)d(guaran)o(teed)75 400 y(to)h(ha)o(v)o(e)f(space)i (to)f(store)h(it.)j(The)c(proto)q(col)g(is)f(send,)i(p)q(ossibly)e (follo)o(w)o(ed)g(b)o(y)g(an)h(ac)o(k-send.)166 452 y(The)h(proto)q (col)g(for)g(secure)i(send)f(is)f(lik)o(ely)f(to)h(b)q(e)h(the)f(follo) o(wing:)j(The)e(sender)h(sends)f(a)f(request-to-send)75 502 y(message.)j(The)13 b(receiv)o(er)i(stores)g(this)e(request)i (\(whic)o(h)f(con)o(tains)f(a)g(cop)o(y)g(of)g(the)h(message)f(en)o(v)o (elop)q(e\).)19 b(When)13 b(a)75 552 y(matc)o(hing)g(receiv)o(e)k(is)e (p)q(osted,)h(the)g(receiv)o(er)g(sends)h(bac)o(k)e(an)g(ac)o(kno)o (wledge,)g(and)g(the)g(sender)i(no)o(w)e(sends)h(the)75 601 y(message.)i(The)c(proto)q(col)g(is)g(\(req,)g(ac)o(k-req,)f (send\),)i(p)q(ossibly)e(follo)o(w)o(ed)g(b)o(y)g(an)h(ac)o(k-send.)166 653 y(What)f(proto)q(col)f(is)h(used)h(for)f(standard)h(sends?)19 b(Either)14 b(the)f(second)h(proto)q(col)f(\(req,)h(ac)o(k-req,)f (send\))h(or)f(a)75 703 y(mix)f(of)h(the)i(\014rst)f(and)g(the)h (second)g(\(e.g.)i(\014rst)e(used)g(for)e(short)i(messages,)e(second)i (for)f(long)f(messages\).)166 755 y(When)j(the)g(receiv)o(er)h(do)q(es) g(not)e(wildcard)h(the)g(source)h(\014eld)f(then)g(the)g(proto)q(col)g (can)g(b)q(e)g(rev)o(ersed:)24 b(The)75 805 y(receiv)o(er)15 b(sends)h(a)d(clear-to-send)i(message,)e(and)h(the)g(sender)i(can)e (pro)q(ceed)h(with)f(the)g(short)h(proto)q(col)e(if)g(it)h(has)75 855 y(a)g(clear-to-send)g(when)h(the)f(send)h(is)f(executed.)166 907 y(What)f(do)f(w)o(e)i(gain)e(from)f(the)j(\\secure)h(receiv)o(e"?)j (Some)12 b(optimizations)f(for)i(regular)g(receiv)o(es)i(\(e.g.,)d(in)g (a)75 957 y(regular)i(receiv)o(e)i(of)e(a)g(short)g(message)g(the)h (receiv)o(er)h(kno)o(ws)e(the)h(short)g(proto)q(col)f(will)f(b)q(e)i (used,)g(and)f(need)h(not)75 1006 y(c)o(hec)o(k)g(for)e(a)h (request-to-send\).)20 b(Q:)14 b(is)g(that)g(enough)g(justi\014cation)f (for)h(the)g(secure)i(receiv)o(e?)166 1058 y(A)i(coun)o(ter)h(argumen)o (t)e(\(Leslie)i(Hart\))f(is)g(that)g(one)g(could)g(conceiv)o(e)h(of)f (an)f(implemen)o(tation)e(where)k(a)75 1108 y(regular)d(receiv)o(e)i (reserv)o(es)g(bu\013er)g(space,)f(so)g(that)f(a)g(program)f(that)h (use)h(regular)f(receiv)o(es)i(is)e(not)h(safe,)f(ev)o(en)75 1158 y(if)f(only)h(secure)i(sends)g(are)f(used.)26 b(I)16 b(doubt)g(this)h(is)f(a)g(strong)h(argumen)o(t.)24 b(On)16 b(the)h(other)g(hand,)g(it)f(indicates)75 1208 y(that,)h(if)e(the)j (exp)q(ectation)f(is)f(that)h(programs)e(written)i(using)f(only)g (secure)i(sends)g(are)f(safe,)g(then)g(this)g(is)f(an)75 1257 y(additional)c(correctness)17 b(condition)c(on)h(the)g(implemen)o (tation)d(that)j(need)g(b)q(e)h(made)e(explicit.)75 1495 y Fm(2.8)70 b(Comm)n(unication)20 b(Ob)t(jects)75 1601 y Fs(An)g(opaque)g(comm)o(unication)h(ob)s(ject)e(iden)o(ti\014es)i(v)m (arious)g(prop)q(erties)f(of)g(a)f(comm)o(unication)i(op)q(er-)75 1657 y(ation,)i(suc)o(h)f(as)f(the)h(bu\013er)g(descriptor)g(that)f(is) h(asso)q(ciated)g(with)g(it,)h(its)f(con)o(text,)g(the)g(tag)f(and)75 1714 y(destination)16 b(parameters)f(to)g(b)q(e)h(used)g(for)e(a)h (send,)h(or)f(the)g(tag)g(and)h(source)f(parameters)g(to)f(b)q(e)i (used)75 1770 y(for)g(a)g(receiv)o(e.)24 b(In)17 b(addition,)g(this)g (ob)s(ject)f(stores)g(information)g(ab)q(out)g(the)h(status)e(of)h(the) h(last)f(com-)75 1827 y(m)o(unication)g(op)q(eration)g(that)e(w)o(as)h (p)q(erformed)g(with)h(this)g(ob)s(ject.)j(This)d(ob)s(ject)f(is)g (accessed)h(using)g(a)75 1883 y(comm)o(unication)g(handle.)166 1942 y(One)g(can)f(consider)h(comm)o(unication)g(op)q(erations)f(to)f (consist)i(of)f(the)g(follo)o(wing)g(sub)q(op)q(erations:)75 2059 y Ft(INIT\(op)q(eration,)k(params,)d(handle\))24 b Fs(Pro)q(cess)15 b(pro)o(vides)f(all)i(relev)m(an)o(t)e(parameters)g (for)g(its)g(par-)189 2115 y(ticipation)k(in)g(the)g(comm)o(unication)g (op)q(eration)f(\(t)o(yp)q(e)g(of)g(op)q(eration,)h(data)e(bu\013er,)i (tag,)e(par-)189 2172 y(ticipan)o(ts,)f(etc.\).)k(An)d(ob)s(ject)e(is)i (created)f(that)g(iden)o(ti\014es)i(the)e(op)q(eration.)75 2274 y Ft(ST)l(AR)l(T\(handle\))24 b Fs(The)16 b(comm)o(unication)g(op) q(eration)f(is)h(started)75 2376 y Ft(COMPLETE\(handle\))25 b Fs(The)16 b(comm)o(unication)g(op)q(eration)f(is)h(completed.)75 2478 y Ft(FREE\(handle\))25 b Fs(The)16 b(comm)o(unication)g(ob)s (ject,)e(and)h(asso)q(ciated)h(resources)f(are)g(freed.)75 2595 y(Correct)f(in)o(v)o(o)q(cation)i(of)f(these)g(sub)q(op)q (erations)h(is)g(a)f(sequence)h(of)f(the)g(form)551 2704 y Ft(INIT)i Fs(\()p Ft(ST)l(AR)l(T)g(COMPLETE)p Fs(\))1208 2685 y Fe(\003)1246 2704 y Ft(FREE)p Fr(:)p eop %%Page: 23 28 23 27 bop 75 -100 a Fq(2.8.)34 b(COMMUNICA)l(TION)16 b(OBJECTS)1004 b Fs(23)75 45 y(I.e.,)12 b(an)g(ob)s(ject)g(needs)h(b)q (e)f(created)h(b)q(efore)f(comm)o(unication)h(o)q(ccurs;)g(it)f(can)g (b)q(e)h(reused)g(only)g(after)e(the)75 102 y(previous)16 b(use)f(has)f(completed;)i(and)f(it)g(needs)h(to)e(b)q(e)h(freed)h(ev)o (en)o(tually)f(\(of)g(course,)f(one)h(can)g(assume)75 158 y(that)f(all)j(ob)s(jects)d(are)h(freed)h(at)e(program)g (termination,)h(b)o(y)h(default\).)166 217 y(The)h(ab)q(o)o(v)o(e)f (scenario)i(p)q(ertains)f(to)f Fi(p)n(ersistent)g Fs(ob)s(jects.)24 b(One)18 b(can)f(also)f(create)h Fi(ephemer)n(al)g Fs(ob-)75 273 y(jects.)h(Suc)o(h)12 b(an)f(ob)s(ject)g(p)q(ersists)h(only)f(un)o (til)i(the)e(comm)o(unication)h(op)q(eration)f(is)h(completed,)h(at)d (whic)o(h)75 330 y(p)q(oin)o(t)i(it)g(is)g(destro)o(y)o(ed.)18 b(Th)o(us)11 b(the)h(correct)f(in)o(v)o(o)q(cation)h(of)f(sub)q(op)q (erations)i(with)e(an)h(ephemeral)h(ob)s(ject)75 386 y(is)j Ft(INIT)h(ST)l(AR)l(T)g(COMPLETE)p Fs(.)166 445 y(A)i(user)g(ma)o(y)f(directly)j(in)o(v)o(ok)o(e)e(these)g(sub)q(op)q (erations.)32 b(This)20 b(w)o(ould)f(allo)o(w)g(the)g(amortization)75 501 y(of)j(the)g(o)o(v)o(erhead)g(of)g(setting)h(up)g(a)f(comm)o (unication)h(o)o(v)o(er)f(man)o(y)g(successiv)o(e)h(uses)g(of)f(the)g (same)75 558 y(handle,)14 b(and)e(allo)o(ws)g(the)h(o)o(v)o(erlap)f(of) f(comm)o(unication)i(and)g(computation.)18 b(Simpler)c(comm)o (unication)75 614 y(op)q(erations)i(com)o(bine)g(sev)o(eral)g(of)f (these)h(sub)q(op)q(erations)h(in)o(to)e(one)h(op)q(eration,)f(th)o(us) h(simplifying)i(the)75 670 y(use)13 b(of)g(comm)o(unication)h(primitiv) o(es.)20 b(Th)o(us,)13 b(one)h(only)f(needs)h(to)f(sp)q(ecify)h (precisely)h(the)e(seman)o(tics)g(of)75 727 y(these)h(sub)q(op)q (erations)g(in)h(order)e(to)g(sp)q(ecify)i(the)f(seman)o(tics)f(of)h (MPI)f(p)q(oin)o(t)h(to)f(p)q(oin)o(t)h(comm)o(unication)75 783 y(op)q(erations.)166 842 y(W)l(e)h(sa)o(y)f(that)f(a)i(comm)o (unication)g(op)q(eration)g(\(send)g(or)f(receiv)o(e\))h(is)g Ft(p)q(osted)g Fs(once)g(a)g Ft(start)f Fs(sub-)75 898 y(op)q(eration)h(has)f(b)q(een)i(in)o(v)o(ok)o(ed;)f(the)g(op)q (eration)f(is)i Ft(completed)f Fs(once)g(the)g Ft(complete)g Fs(sub)q(op)q(eration)75 955 y(has)i(completed.)28 b(A)17 b(send)h(and)g(a)f(receiv)o(e)h(op)q(eration)g Ft(matc)o(h)f Fs(if)h(the)f(receiv)o(e)i(pattern)e(sp)q(eci\014ed)i(b)o(y)75 1011 y(the)c(receiv)o(e)h(matc)o(hes)f(the)g(message)g(en)o(v)o(elop)q (e)i(created)e(b)o(y)g(the)g(send.)75 1145 y Fj(2.8.1)55 b(Comm)n(unication)17 b(Ob)s(ject)h(Creation)75 1318 y Fl(Discussion:)34 b Fk(This)14 b(section)g(has)g(not)g(y)o(et)g(b)q (een)h(appro)o(v)o(ed)166 1459 y Fs(An)j(ob)s(ject)e(for)h(a)g(send)h (op)q(eration)g(is)g(created)f(b)o(y)g(a)g(call)i(to)e Ft(MPI)p 1373 1459 16 2 v 18 w(INIT)p 1508 1459 V 18 w(SEND)p Fs(.)h(A)f(call)i(to)75 1516 y Ft(MPI)p 184 1516 V 18 w(INIT)p 319 1516 V 19 w(RECV)e Fs(is)i(similarly)h(used)e (for)f(creating)h(an)g(ob)s(ject)g(for)f(a)h(receiv)o(e)g(op)q (eration.)29 b(The)75 1572 y(creation)11 b(of)g(a)g(comm)o(unication)h (ob)s(ject)e(is)i(a)f(lo)q(cal)h(op)q(eration)f(that)g(need)h(not)f(in) o(v)o(olv)o(e)g(comm)o(unication)75 1629 y(with)16 b(a)e(remote)h(pro)q (cess.)166 1723 y Ft(MPI)p 275 1723 V 18 w(INIT)p 410 1723 V 19 w(SEND)k(\(handle,)h(bu\013er)p 932 1723 V 18 w(handle,)g(dest,)e(tag,)i(con)o(text,)f(mo)q(de,)g(p)q(ersis-)75 1779 y(tence\))166 1873 y Fs(Creates)14 b(a)h(send)h(comm)o(unication)g (ob)s(ject.)j(P)o(arameters)14 b(are)75 1976 y Ft(OUT)k(handle)24 b Fs(message)11 b(handle.)20 b(The)12 b(handle)h(should)f(not)f(b)q(e)i (asso)q(ciated)f(with)f(an)o(y)h(ob)s(ject)f(b)q(efore)189 2032 y(the)k(call.)75 2135 y Ft(IN)i(bu\013er)p 291 2135 V 18 w(handle)24 b Fs(handle)16 b(to)f(send)h(bu\013er)f(descriptor)75 2237 y Ft(IN)i(dest)23 b Fs(rank)15 b(in)h(con)o(text)e(of)h (destination)h(\(in)o(teger\))75 2340 y Ft(IN)h(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(sen)o(t)g(with)g(this)h(handle)g (\(in)o(teger\))75 2442 y Ft(IN)h(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(sen)o(t)g(with)h(this)f(handle)75 2545 y Ft(IN)i(mo)q(de)23 b Fs(send)11 b(mo)q(de)f(\(state)f(t)o(yp)q(e,)i (with)g(three)f(v)m(alues:)18 b Fh(MPI)p 1186 2545 15 2 v 17 w(STANDARD)p Fs(,)9 b Fh(MPI)p 1489 2545 V 16 w(READY)h Fs(and)g Fh(MPI)p 1790 2545 V 17 w(SECURE)p Fs(\))75 2647 y Ft(IN)17 b(p)q(ersistence)23 b Fs(handle)h(p)q (ersistence)g(\(state)d(t)o(yp)q(e,)j(with)f(t)o(w)o(o)e(v)m(alues:)35 b Fh(MPI)p 1527 2647 V 17 w(PERSISTENT)21 b Fs(and)189 2704 y Fh(MPI)p 264 2704 V 16 w(EPHEMERAL)p Fs(\))p eop %%Page: 24 29 24 28 bop 75 -100 a Fs(24)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Ft(MPI)p 275 45 16 2 v 18 w(INIT)p 410 45 V 19 w(RECV)i(\(handle,)i(bu\013er)p 938 45 V 18 w(handle,)g(source,)e(tag,)h(con)o(text,)g(mo)q(de,)g(p)q (er-)75 102 y(sistence\))166 195 y Fs(Create)c(a)f(receiv)o(e)i (handle.)22 b(P)o(arameters)13 b(are)75 293 y Ft(OUT)18 b(handle)24 b Fs(message)11 b(handle.)20 b(The)12 b(handle)h(should)f (not)f(b)q(e)i(asso)q(ciated)f(with)f(an)o(y)h(ob)s(ject)f(b)q(efore) 189 349 y(the)k(call.)75 447 y Ft(IN)i(bu\013er)p 291 447 V 18 w(handle)24 b Fs(handle)16 b(to)f(receiv)o(e)h(bu\013er)f (descriptor.)75 545 y Ft(IN)i(source)22 b Fs(rank)15 b(in)h(con)o(text)f(of)g(source,)g(or)f Fh(MPI)p 974 545 15 2 v 17 w(ANY)p 1063 545 V 17 w(SOURCE)g Fs(\(in)o(teger\).)75 643 y Ft(IN)j(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(receiv)o(ed) h(with)g(this)f(handle,)h(or)f Fh(MPI)p 1341 643 V 17 w(ANY)p 1430 643 V 16 w(TAG)g Fs(\(in)o(teger\).)75 741 y Ft(IN)i(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(receiv)o (ed)h(with)g(this)g(handle.)75 839 y Ft(IN)h(mo)q(de)23 b Fs(receiv)o(e)16 b(mo)q(de)g(\(state)e(t)o(yp)q(e,)h(with)g(t)o(w)o (o)f(v)m(alues:)21 b Fh(MPI)p 1233 839 V 17 w(STANDARD)p Fs(,)13 b(and)j Fh(MPI)p 1629 839 V 17 w(SECURE)p Fs(\))75 938 y Ft(IN)h(p)q(ersistence)23 b Fs(handle)h(p)q(ersistence)g(\(state) d(t)o(yp)q(e,)j(with)f(t)o(w)o(o)e(v)m(alues:)35 b Fh(MPI)p 1527 938 V 17 w(PERSISTENT)21 b Fs(and)189 994 y Fh(MPI)p 264 994 V 16 w(EPHEMERAL)p Fs(\))166 1092 y(See)16 b(Section)g(2.4.2)d (for)i(a)g(discussion)i(of)d(source,)h(tag)g(and)g(con)o(text.)166 1226 y Fl(Discussion:)33 b Fk(I)13 b(ha)o(v)o(e)h(not)f(included)g (prop)q(osals)h(for)f(partially)e(sp)q(eci\014ed)k(message)e(handles,)g (that)h(some)75 1275 y(p)q(eoples)h(seem)e(to)h(desire.)166 1326 y(I)g(ha)o(v)o(e)f(merged)h(all)f(handle)g(setup)i(in)o(to)e(one)h (call.)166 1377 y(The)d(functions)f Fg(MPI)p 492 1377 14 2 v 15 w(INIT)p 595 1377 V 15 w(SEND)g Fk(and)g Fg(MPI)p 851 1377 V 15 w(INIT)p 954 1377 V 15 w(RECV)g Fk(ha)o(v)o(e)g(the)h (same)f(list)g(of)g(parameters.)17 b(Jim)8 b(Co)o(wnie)75 1427 y(suggests)15 b(to)f(ha)o(v)o(e)g(only)f(one)h(function)f(with)h (an)g(extra)g(argumen)o(t.)166 1478 y(Ephemeral)f(\\p)q(ersisten)o(t)h (handles")g(are)f(not)h(terribly)f(useful)g(as)h(suc)o(h.)k(But)c (their)g(existence)h(allo)o(w)c(us)j(to)75 1528 y(describ)q(e)i(all)c (the)j(higher)f(lev)o(el)f(send/receiv)o(e)j(op)q(erations)e(in)g (terms)f(of)h(basic)g(op)q(erations)g(on)f(handles.)75 1738 y Fj(2.8.2)55 b(Comm)n(unication)17 b(Start)75 1862 y Ft(MPI)p 184 1862 16 2 v 18 w(ST)l(AR)l(T\(handle\))75 1995 y(IN)g(handle)24 b Fs(comm)o(unication)16 b(handle)166 2093 y(The)i Fh(MPI)p 337 2093 15 2 v 17 w(START)g Fs(function)h (starts)e(the)h(execution)i(of)d(a)h(comm)o(unication)h(op)q(eration)g (\(send)f(or)75 2150 y(receiv)o(e\).)32 b(A)19 b(sender)h(should)g(not) e(up)q(date)i(the)f(send)h(bu\013er)f(after)f(a)h(send)h(op)q(eration)f (has)g(started)75 2206 y(un)o(til)e(after)e(it)h(has)g(completed.)23 b(A)16 b(receiv)o(er)g(should)h(not)f(access)g(the)g(receiv)o(e)h (bu\013er)e(after)g(a)h(receiv)o(e)75 2262 y(op)q(eration)h(w)o(as)f (started)f(un)o(til)j(after)e(it)h(has)f(completed.)25 b(A)17 b(program)e(that)h(do)q(es)h(not)f(satisfy)h(these)75 2319 y(conditions)f(is)g(erroneous)f(and)h(its)f(outcome)g(is)g (undetermined.)75 2447 y Fj(2.8.3)55 b(Comm)n(unication)17 b(Completion)75 2570 y Ft(MPI)p 184 2570 16 2 v 18 w(W)-6 b(AIT)17 b(\()h(handle,)g(return)p 730 2570 V 18 w(status)p 884 2570 V 19 w(handle\))75 2704 y(IN)f(handle)24 b Fs(comm)o (unication)16 b(handle)p eop %%Page: 25 30 25 29 bop 75 -100 a Fq(2.8.)34 b(COMMUNICA)l(TION)16 b(OBJECTS)1004 b Fs(25)75 45 y Ft(IN)17 b(return)p 302 45 16 2 v 18 w(handle)24 b Fs(handle)17 b(that)e(is)i(asso)q(ciated)f (with)g(return)g(status)f(ob)s(ject.)21 b(the)16 b(return)g(status)189 102 y(ob)s(ject)e(is)i(up)q(dated)g(b)o(y)f(call.)166 192 y(A)h(call)i(to)e Ft(MPI)p 467 192 V 18 w(W)-6 b(AIT)16 b Fs(returns)g(when)h(the)f(send)h(op)q(eration)g(iden)o(ti\014ed)h(b)o (y)f Fh(handle)e Fs(is)i(com-)75 248 y(plete.)24 b(The)17 b(completion)g(of)f(a)g(send)h(op)q(eration)g(indicates)h(that)d(the)i (sender)g(is)f(no)o(w)g(free)h(to)e(up)q(date)75 305 y(the)i(lo)q(cations)h(in)g(the)f(send)g(bu\013er,)h(or)e(an)o(y)h (other)f(lo)q(cation)i(that)e(can)i(b)q(e)f(referenced)h(b)o(y)f(the)g (send)75 361 y(op)q(eration.)j(Ho)o(w)o(ev)o(er,)13 b(it)i(do)q(es)g (not)f(indicate)i(that)d(the)i(message)f(has)g(b)q(een)i(receiv)o(ed;)f (rather)f(it)h(ma)o(y)75 418 y(ha)o(v)o(e)g(b)q(een)h(bu\013ered)g(b)o (y)f(the)g(comm)o(unication)h(subsystem.)166 474 y(The)g(completion)g (of)g(a)f(receiv)o(e)h(op)q(eration)g(indicates)h(that)e(the)h(receiv)o (er)g(is)g(no)o(w)f(free)h(to)f(access)75 531 y(the)d(lo)q(cations)h (in)g(the)f(receiv)o(e)i(bu\013er,)e(whic)o(h)h(con)o(tain)f(the)h (receiv)o(ed)g(message,)f(or)g(an)o(y)f(other)h(lo)q(cation)75 587 y(that)g(can)g(b)q(e)h(referenced)h(b)o(y)e(the)h(receiv)o(e)g(op)q (eration.)19 b(It)13 b(do)q(es)g(not)f(indicate)i(that)d(the)i(matc)o (hing)f(send)75 643 y(op)q(eration)j(has)g(completed.)166 700 y(The)i(call)g(returns)g(a)f(handle)i(to)e(an)g(opaque)h(ob)s(ject) f(that)g(con)o(tains)g(information)h(on)f(the)h(com-)75 756 y(pleted)f(op)q(eration)g({)f(the)g Ft(return)i(status)e Fs(ob)s(ject.)166 848 y Ft(MPI)p 275 848 V 18 w(ST)l(A)l(TUS)i (\(handle,)h(\015ag,)g(return)p 974 848 V 18 w(handle\))75 974 y(IN)f(handle)24 b Fs(comm)o(unication)16 b(handle)75 1066 y Ft(OUT)i(\015ag)24 b Fs(logical)75 1159 y Ft(OUT)18 b(return)p 357 1159 V 19 w(handle)23 b Fs(handle)17 b(that)d(is)i(asso) q(ciated)f(with)h(return)f(status)f(ob)s(ject.)166 1249 y(A)21 b(call)h(to)f Ft(MPI)p 481 1249 V 18 w(ST)l(A)l(TUS)g Fs(returns)g Fh(flag=true)f Fs(if)h(the)g(op)q(eration)h(iden)o (ti\014ed)h(b)o(y)e Fh(handle)75 1306 y Fs(is)h(complete.)40 b(In)22 b(suc)o(h)g(case,)g(the)g(return)f(handle)i(p)q(oin)o(ts)f(to)f (an)g(opaque)h(ob)s(ject)f(that)g(con)o(tains)75 1362 y(information)c(on)g(the)g(completed)h(op)q(eration.)26 b(It)17 b(returns)g Fh(flag=false)p Fs(,)f(otherwise.)25 b(In)18 b(suc)o(h)f(case,)75 1418 y(the)e(v)m(alue)i(of)d(the)i(return) f(handle)h(is)g(unde\014ned.)166 1475 y(A)e(call)i(to)e Fh(MPI)p 427 1475 15 2 v 16 w(WAIT)g Fs(blo)q(c)o(ks)h(only)g(the)g (executing)g(thread.)20 b(If)14 b(the)h(executing)g(pro)q(cess)g(is)g (m)o(ulti-)75 1531 y(threaded,)g(then)h(other)e(threads)h(within)i(the) e(pro)q(cess)h(can)f(b)q(e)h(sc)o(heduled)h(for)d(execution.)166 1588 y(The)21 b(use)h(of)f(a)f(blo)q(c)o(king)j(receiv)o(e)f(op)q (eration)f(\()p Fh(MPI)p 1095 1588 V 17 w(WAIT)p Fs(\))f(allo)o(ws)h (the)h(op)q(erating)f(system)g(to)75 1644 y(desc)o(hedule)e(the)e(blo)q (c)o(k)o(ed)i(thread)e(and)g(sc)o(hedule)i(another)d(thread)h(for)g (execution,)h(if)g(suc)o(h)f(is)h(a)o(v)m(ail-)75 1701 y(able.)k(The)16 b(use)g(of)f(a)g(non)o(blo)q(c)o(king)i(receiv)o(e)g (op)q(eration)e(\()p Fh(MPI)p 1152 1701 V 17 w(STATUS)p Fs(\))f(allo)o(ws)i(the)g(user)f(to)g(sc)o(hedule)75 1757 y(alternativ)o(e)g(activities)i(within)f(a)f(single)i(thread)e(of) f(execution.)166 1814 y(The)h(in)o(tended)h(implemen)o(tation)g(of)e Fh(MPI)p 894 1814 V 17 w(STATUS)g Fs(is)h(for)f(that)g(op)q(eration)h (to)f(return)h(as)f(so)q(on)g(as)75 1870 y(p)q(ossible.)22 b(Ho)o(w)o(ev)o(er,)14 b(if)i(rep)q(eatedly)h(called)g(for)d(an)i(op)q (eration)f(that)g(is)h(enabled,)h(it)e(m)o(ust)g(ev)o(en)o(tually)75 1927 y(succeed.)166 1983 y(The)23 b(return)f(status)f(ob)s(ject)h(for)g (a)g(send)h(op)q(eration)g(carries)f(no)h(information.)41 b(The)23 b(return)75 2039 y(status)17 b(ob)s(ject)h(for)f(a)h(receiv)o (e)h(op)q(eration)g(carries)f(information)g(on)g(the)g(source,)h(tag)e (and)i(length)f(of)75 2096 y(the)i(receiv)o(ed)h(message.)34 b(These)20 b(\014elds)h(are)e(required)i(b)q(ecause)g(the)f(receiv)o(e) h(op)q(eration)f(ma)o(y)f(ha)o(v)o(e)75 2152 y(sp)q(eci\014ed)i Fh(DONTCARE)d Fs(in)i(either)f(source)g(or)g(tag)f(\014eld,)i(and)f (the)g(message)g(ma)o(y)f(ha)o(v)o(e)h(b)q(een)h(shorter)75 2209 y(than)15 b(the)g(receiv)o(e)h(bu\013er.)166 2301 y Ft(MPI)p 275 2301 16 2 v 18 w(QUER)l(Y\()i(handle,)g(coun)o(t,)f (source,)g(tag\))75 2426 y(IN)g(handle)24 b Fs(handle)16 b(to)f(return)g(status)f(ob)s(ject)75 2519 y Ft(OUT)k(coun)o(t)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(elemen)o(ts)g(receiv)o(ed.)75 2611 y Ft(OUT)j(source)23 b Fs(rank)14 b(of)h(message)g(sender)h(in)g (message)f(con)o(text)f(\(in)o(teger\).)75 2704 y Ft(OUT)k(tag)24 b Fs(tag)14 b(of)h(receiv)o(ed)h(message)f(\(in)o(teger\).)p eop %%Page: 26 31 26 30 bop 75 -100 a Fs(26)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Ft(MPI)p 275 45 16 2 v 18 w(CREA)l(TE)p 510 45 V 19 w(ST)l(A)l(TUS\()h(handle\)) 166 138 y Fs(Creates)d(a)h(new)h(return)f(status)f(ob)s(ject.)75 238 y Ft(OUT)k(handle)24 b Fs(handle)16 b(to)f(return)g(status)g(ob)s (ject.)166 413 y Fl(Discussion:)166 465 y Fk(The)e(use)g(of)f(a)h (return)g(status)h(ob)r(ject,)f(rather)g(than)g(a)f(list)g(of)g (parameters)h(ma)o(y)d(simplify)g(the)j(use)h(of)e(MPI)75 515 y(routines,)17 b(if)f(the)g(v)n(alues)g(stored)i(in)e(the)h(ob)r (ject)g(are)f(seldom)f(c)o(hec)o(k)o(ed.)27 b(A)16 b(prede\014ned)j (return)e(status)g(ob)r(ject)75 564 y(should)d(b)q(e)h(pro)o(vided,)f (to)g(ease)h(programming.)h(The)f(main)d(reason)j(for)f(the)h(use)g(of) f(a)g(return-status)i(ob)r(ject)f(is)75 614 y(that)f(one)h(w)o(an)o(ts) f(to)g(b)q(e)g(able)g(to)g(use)h(the)g(same)e Fg(MPI)p 923 614 14 2 v 15 w(WAIT)g Fk(and)h Fg(MPI)p 1186 614 V 15 w(STATUS)f Fk(calls)h(for)g(c)o(hec)o(king)g(on)g(di\013eren)o(t) 75 664 y(t)o(yp)q(es)h(of)f(op)q(erations,)f(e.g.)19 b(b)q(oth)14 b(sends)i(and)d(receiv)o(es,)j(and)e(p)q(erhaps)h(new)g (op)q(erations)f(in)g(the)g(future.)20 b(These)75 714 y(di\013eren)o(t)12 b(op)q(erations)g(return)g(di\013eren)o(t)g (information)c(\(alb)q(eit)j(w)o(e)h(could)f(decree)i(that)e(a)g(send)h (returns)h(the)f(same)75 764 y(information)f(as)j(a)g(receiv)o(e\).)166 815 y(The)i(issue)h(of)e(memory)f(allo)q(cation)g(for)i(return)p 939 815 13 2 v 16 w(status)h(handles)f(is)g(not)g(y)o(et)g(solv)o(ed.) 24 b(There)17 b(is)f(a)g(desire)75 865 y(that)e(the)g(user)h(will)e(b)q (e)h(able)g(to)g(allo)q(cate)f(space)i(for)f(them)f(\(e.g.,)f(on)i(the) h(stac)o(k\).)166 916 y(W)m(e)i(return)i(n)o(um)o(b)q(er)d(of)h(elemen) o(ts)g(receiv)o(ed,)j(rather)e(than)f(b)o(ytes)i(receiv)o(ed)f(b)q (ecause)h(\\n)o(um)o(b)q(er)e(of)g(ele-)75 966 y(men)o(ts")g(is)h (closer)h(to)e(the)i(application)d(seman)o(tic)i(lev)o(el)f(\(user)i (do)q(es)g(not)f(need)h(to)f(b)q(e)g(a)o(w)o(are)g(of)f(the)i(size)g (of)75 1015 y(elemen)o(ts\),)g(and)f(is)f(more)h(in)o(v)n(arian)o(t.)28 b(The)19 b(return)g(status)g(ob)r(ject)g(returns)h(the)e(n)o(um)o(b)q (er)g(of)f Fl(elemen)o(ts)e Fk(re-)75 1065 y(ceiv)o(ed.)k(If)14 b(there)h(w)o(as)f(no)g(truncation,)g(then)h(this)f(is)g(equal)g(to)f (the)i(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(sen)o(t.)19 b(On)c(the)f(other)75 1115 y(hand,)k(the)f(n)o(um)o(b)q(er)g(of)g(b)o (ytes)h(collected)g(from)d(the)j(sender)h(memory)m(,)c(the)j(n)o(um)o (b)q(er)e(of)h(b)o(ytes)h(sen)o(t)g(o)o(v)o(er)f(the)75 1165 y(wire,)12 b(and)g(the)g(n)o(um)o(b)q(er)f(of)g(b)o(ytes)i(stored) f(in)g(the)g(receiv)o(er)h(memory)d(ma)o(y)g(all)g(b)q(e)i(di\013eren)o (t,)h(in)e(a)h(heterogeneous)75 1215 y(en)o(vironmen)o(t.)23 b(The)17 b(n)o(um)o(b)q(er)e(of)g(elemen)o(ts)h(sen)o(t)h(can)f(b)q(e)g (computed)g(form)e(the)j(send)g(bu\013er)g(descriptor;)g(the)75 1265 y(n)o(um)o(b)q(er)c(of)g(elemen)o(ts)h(receiv)o(ed)h(can)f(b)q(e)h (computed)e(from)g(the)h(receiv)o(e)h(bu\013er)g(descriptor)g(and)f (the)g(length)g(\(in)75 1314 y(b)o(ytes\))h(of)e(the)i(receiv)o(ed)g (message.)166 1366 y Fg(MPI)p 235 1366 14 2 v 15 w(STATUS)i Fk(and)h Fg(MPI)p 550 1366 V 15 w(QUERY)f Fk(are)h(not)g(go)q(o)q(d)f (names)h({)f(they)i(can)f(b)q(e)h(easily)e(confused.)31 b(Leslie)19 b(Hart)75 1415 y(suggests)14 b Fg(MPI)p 305 1415 V 15 w(TEST)d Fk(or)i Fg(MPI)p 535 1415 V 15 w(CHECK)f Fk(for)g(the)h(\014rst.)18 b(P)o(erhaps)c(w)o(e)e(should)h(replace)g Fg(MPI)p 1458 1415 V 15 w(STATUS)f Fk(with)g Fg(MPI)p 1776 1415 V 15 w(TEST)75 1465 y Fk(and)i Fg(MPI)p 225 1465 V 15 w(QUERY)f Fk(with)g Fg(MPI)p 523 1465 V 15 w(STATUS)p Fk(.)75 1678 y Fj(2.8.4)55 b(Multiple)17 b(Completions)75 1766 y Fs(It)f(is)h(con)o(v)o(enien)o(t)f(to)g(b)q(e)g(able)h(to)f(w)o (ait)f(for)h(the)g(completion)h(of)e(an)o(y)h(or)g(all)h(the)f(op)q (erations)g(in)h(a)f(set,)75 1823 y(rather)d(than)h(ha)o(ving)f(to)g(w) o(ait)h(for)f(a)g(sp)q(eci\014c)j(message.)i(A)c(call)h(to)e Fh(MPI)p 1323 1823 15 2 v 17 w(WAITANY)f Fs(or)h Fh(MPI)p 1646 1823 V 17 w(STATUSANY)75 1879 y Fs(can)e(b)q(e)h(used)g(to)e(w)o (ait)h(for)f(the)i(completion)g(of)e(one)i(out)e(of)h(sev)o(eral)g(op)q (erations;)i(a)d(call)j(to)d Fh(MPI)p 1694 1879 V 17 w(WAITALL)75 1936 y Fs(can)15 b(b)q(e)h(used)g(to)f(w)o(ait)f(for)h (all)h(p)q(ending)h(op)q(erations)e(in)h(a)f(list.)166 2029 y Ft(MPI)p 275 2029 16 2 v 18 w(W)-6 b(AIT)l(ANY)16 b(\()i(list)p 677 2029 V 19 w(of)p 738 2029 V 19 w(handles,)g(coun)o (t,)f(index,)g(return)p 1421 2029 V 19 w(handle\))166 2122 y Fs(Blo)q(c)o(ks)h(un)o(til)g(one)g(of)f(the)g(op)q(erations)h (asso)q(ciated)g(with)f(the)h(comm)o(unication)g(handles)h(in)f(the)75 2179 y(arra)o(y)13 b(has)i(completed.)21 b(Returns)15 b(the)g(index)h(of)e(that)g(handle)i(in)f(the)g(arra)o(y)l(,)f(and)g (returns)h(the)g(status)75 2235 y(of)g(that)f(op)q(eration)i(in)g(the)f (ob)s(ject)g(asso)q(ciated)g(with)h(the)f(return)p 1223 2235 14 2 v 16 w(handle.)21 b(The)16 b(parameters)e(are:)75 2349 y Ft(IN)j(list)p 225 2349 16 2 v 20 w(of)p 287 2349 V 19 w(handles)23 b Fs(list)16 b(of)f(handles)h(to)f(comm)o(unication)h (ob)s(jects.)75 2448 y Ft(IN)h(coun)o(t)23 b Fs(list)16 b(length)g(\(in)o(teger\))75 2548 y Ft(OUT)i(index)23 b Fs(index)17 b(of)d(handle)j(for)d(op)q(eration)i(that)e(completed)i (\(in)o(teger\).)75 2647 y Ft(OUT)i(return)p 357 2647 V 19 w(handle)23 b Fs(handle)c(that)e(is)i(asso)q(ciated)f(with)g (return)g(status)f(ob)s(ject.)27 b(Set)18 b(to)f(return)189 2704 y(status)d(of)h(op)q(eration)g(that)g(completed.)p eop %%Page: 27 32 27 31 bop 75 -100 a Fq(2.8.)34 b(COMMUNICA)l(TION)16 b(OBJECTS)1004 b Fs(27)166 45 y(The)12 b(successful)i(execution)f(of)f Fh(MPI)p 784 45 15 2 v 16 w(WAITANY\(list)p 1088 45 V 16 w(of)p 1152 45 V 17 w(handles,)23 b(index,)g(return)p 1695 45 V 16 w(handle\))75 102 y Fs(has)15 b(the)g(same)g(e\013ect)g (as)g(the)h(successful)g(execution)h(of)d Fh(MPI)p 1139 102 V 17 w(WAIT\(handle[i],)22 b(return)p 1682 102 V 16 w(handle\))p Fs(,)75 158 y(where)17 b Fh(i)g Fs(is)g(the)g(v)m(alue) h(returned)f(b)o(y)g Fh(index)f Fs(and)g Fh(handle[i])g Fs(is)h(the)g Fh(i)p Fs(-th)g(handle)h(in)f(the)g(list,)h(and)75 214 y(the)d(cancellation)i(of)e(all)h(remaining)g(w)o(ait)f(op)q (erations.)166 271 y(If)21 b(more)g(then)g(one)g(op)q(eration)h(is)f (enabled)i(and)e(can)g(terminate,)h(one)f(is)h(arbitrarily)g(c)o(hosen) 75 327 y(\(sub)s(ject)15 b(to)f(the)i(restrictions)f(on)g(op)q(eration) h(termination)f(order,)g(and)g(fairness,)g(see)h(Section)g(2.13\).)166 384 y Fh(MPI)p 241 384 V 17 w(WAITANY)23 b(\()g(list)p 592 384 V 17 w(of)p 657 384 V 17 w(handles,)g(count,)g(index,)g(return) p 1367 384 V 16 w(status)p 1527 384 V 16 w(handle\))15 b Fs(is)147 457 y Fh({MPI_WAIT)22 b(\(handle[0],)h(return_handle\);)e (index)j(=)f(0})h(||)f(...)75 514 y(||)147 570 y({MPI_WAIT)f (\(handle[count-1],)g(return_handle\);)g(index)h(=)g(count-1})166 643 y Fs(\(\\)p Fo(jj)p Fs(")13 b(indicates)k(c)o(hoice;)f(one)f(of)g (the)g(alternativ)o(es)h(is)f(c)o(hosen,)g(nondeterministically)m(.\)) 166 735 y Ft(MPI)p 275 735 16 2 v 18 w(ST)l(A)l(TUSANY)h(\()i(list)p 739 735 V 20 w(of)p 801 735 V 19 w(handles,)f(coun)o(t,)g(index,)h (return)p 1484 735 V 18 w(handle\))166 827 y Fs(Causes)12 b(either)h(one)f(or)g(none)h(of)f(the)g(op)q(erations)g(asso)q(ciated)h (with)f(the)h(comm)o(unication)g(handles)75 884 y(to)i(return.)23 b(In)17 b(the)f(former)f(case,)h(it)g(has)g(the)g(same)g(return)g (seman)o(tics)g(as)g(a)g(call)h(to)e Fh(MPI)p 1664 884 15 2 v 17 w(WAIT)p 1777 884 V 17 w(ANY)p Fs(.)75 940 y(In)i(the)f(latter)g(case,)g(it)h(returns)f(a)g(v)m(alue)h(of)f({1)f (in)j Fh(index)d Fs(and)h Fh(return)p 1349 940 V 17 w(handle)f Fs(is)i(unde\014ned.)25 b(The)75 997 y(parameters)14 b(are:)75 1074 y Ft(IN)j(list)p 225 1074 16 2 v 20 w(of)p 287 1074 V 19 w(handles)23 b Fs(list)16 b(of)f(handles)h(to)f(comm)o (unication)h(ob)s(jects.)75 1160 y Ft(IN)h(coun)o(t)23 b Fs(list)16 b(length)g(\(in)o(teger\))75 1246 y Ft(OUT)i(index)23 b Fs(index)18 b(of)e(handle)h(for)f(op)q(eration)h(that)e(completed,)j (or)e(-1)g(if)h(none)f(completed)i(\(in)o(te-)189 1302 y(ger\).)75 1388 y Ft(OUT)g(return)p 357 1388 V 19 w(handle)23 b Fs(handle)c(that)e(is)i(asso)q(ciated)f(with)g(return)g(status)f(ob)s (ject.)27 b(Set)18 b(to)f(return)189 1444 y(status)d(of)h(op)q(eration) g(that)g(completed,)g(if)h(an)o(y;)f(unde\014ned)i(when)e Fh(index)24 b(=)f(-1)p Fs(.)166 1557 y Ft(MPI)p 275 1557 V 18 w(W)-6 b(AIT)l(ALL\(list)p 624 1557 V 20 w(of)p 686 1557 V 19 w(handles,)18 b(coun)o(t,)f(list)p 1136 1557 V 20 w(of)p 1198 1557 V 19 w(return)p 1363 1557 V 18 w(handles\))166 1649 y Fs(Blo)q(c)o(ks)c(un)o(til)h(all)g(comm)o (unication)g(op)q(erations)f(asso)q(ciated)g(with)g(handles)h(in)g(the) f(list)h(complete,)75 1705 y(and)h(return)h(the)f(status)f(of)h(all)h (these)f(op)q(erations.)20 b(The)c(parameters)e(are:)75 1783 y Ft(IN)j(list)p 225 1783 V 20 w(of)p 287 1783 V 19 w(handles)23 b Fs(list)16 b(of)f(handles)h(to)f(comm)o(unication)h (ob)s(jects.)75 1869 y Ft(IN)h(coun)o(t)23 b Fs(lists)16 b(length)g(\(in)o(teger\))75 1954 y Ft(OUT)i(list)p 280 1954 V 20 w(of)p 342 1954 V 19 w(return)p 507 1954 V 18 w(handles)24 b Fs(Must)18 b(ha)o(v)o(e)h(the)g(same)g(length)g(as)g (the)g(\014rst)g(list.)32 b(Eac)o(h)19 b(return)189 2011 y(status)12 b(ob)s(ject)g(is)i(set)e(to)h(the)g(return)f(status)h(of)f (the)h(corresp)q(onding)h(op)q(eration)f(in)h(the)f(\014rst)g(list.)166 2165 y Fl(Discussion:)166 2214 y Fk(The)h(fairness)f(requiremen)o(t)g (giv)o(en)g(in)g(Section)g(2.13)f(implies)g(that)h(the)h(use)g(of)e Fg(WAIT)p 1525 2214 14 2 v 15 w(ANY)h Fk(cannot)g(lead)g(to)75 2264 y(starv)n(ation:)21 b(If)15 b(the)h(sending)g(pro)q(cess)h(has)f (issued)g(a)f(send)i(complete)d(op)q(eration,)i(and)f(the)h(receiving)g (pro)q(cess)75 2314 y(rep)q(eatedly)21 b(p)q(ost)f(a)g(receiv)o(e)h (for)e(the)h(message)g(sen)o(t,)h(then)g(that)e(message)h(m)o(ust)f(b)q (e)h(ev)o(en)o(tually)f(receiv)o(ed.)75 2364 y(Section)c(2.13)e(has)i (not)f(y)o(et)h(b)q(een)h(discussed.)21 b(The)15 b(fairness)g (requiremen)o(t)g(can)f(b)q(e)i(attac)o(k)o(ed)e(either)i(for)e(b)q (eing)75 2414 y(to)q(o)j(w)o(eak)h(\(ev)o(en)o(tually)f(is)g(not)g(go)q (o)q(d)g(enough\),)h(or)g(to)q(o)f(strong)h(\(hard)f(to)g(implemen)o (t\).)26 b(In)18 b(the)g(later)f(case,)75 2463 y(if)g(the)g(requiremen) o(t)h(of)f(fairness)g(is)h(dropp)q(ed)g(from)d(MPI)j(implemen)o(tatio)o (ns,)d(then)j(some)f(mec)o(hanism)e(need)75 2513 y(b)q(e)i(pro)o(vided) f(to)f(the)i(user)g(to)f(ac)o(hiev)o(e)g(fairness)h(b)o(y)f(his)f(or)h (her)h(o)o(wn)f(devices.)25 b(One)17 b(suc)o(h)g(prop)q(osal)f(is)f(to) h(b)q(e)75 2563 y(able)e(to)g(sp)q(ecify)g(a)g(rotating)g(priorit)o(y)f (order)i(on)f(the)g(op)q(erations)h(p)q(osted)g(b)o(y)f(a)f Fg(MPI)p 1431 2563 V 15 w(WAITANY)p Fk(:)g(searc)o(h)i(the)f(list)75 2613 y(sequen)o(tially)m(,)e(starting)i(from)e(a)i(user)h(sp)q (eci\014ed)g(p)q(osition.)p eop %%Page: 28 33 28 32 bop 75 -100 a Fs(28)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fm(2.9)70 b(Blo)r(c)n(king)21 b(Comm)n(unication)75 150 y Fs(Blo)q(c)o(king)g (send)e(and)h(receiv)o(e)g(op)q(erations)f(com)o(bine)h(all)h(comm)o (unication)f(sub)q(op)q(erations)g(in)o(to)f(one)75 206 y(call.)h(The)12 b(op)q(eration)h(returns)f(only)g(when)h(the)f(comm)o (unication)h(completes)g(and)f(no)g(comm)o(unication)75 262 y(ob)s(ject)19 b(p)q(ersists)i(after)e(the)i(call)g(completed.)35 b(Ho)o(w)o(ev)o(er,)20 b(the)g(bu\013er)g(descriptor)h(ob)s(ject)e (needs)i(b)q(e)75 319 y(created)15 b(ahead)h(of)e(the)i(call.)166 377 y(W)l(e)f(use)h(the)f(follo)o(wing)h(naming)g(con)o(v)o(en)o(tion)f (for)f(suc)o(h)i(op)q(erations:)782 443 y Fd(2)782 516 y(6)782 543 y(4)833 484 y Fo(\000)831 540 y Ft(R)836 597 y(S)891 443 y Fd(3)891 516 y(7)891 543 y(5)926 468 y(")975 512 y Ft(SEND)971 568 y(RECV)1143 468 y Fd(#)166 706 y Fs(The)i(\014rst)f(letter)h(\(v)o(oid,)g Ft(R)g Fs(or)g Ft(S)p Fs(\))f(indicates)j(the)d(start)g(mo)q(de)h(\(standard,) g(ready)l(,)g(or)f(secure\).)75 763 y(Only)f(t)o(w)o(o)e(of)h(the)g (com)o(binations)h(\(standard)f(and)g(secure\))g(are)g(meaningful)i (for)d(receiv)o(es.)166 856 y Ft(MPI)p 275 856 16 2 v 18 w(SEND)k(\(bu\013er)p 610 856 V 18 w(handle,)g(dest,)f(tag,)h(con)o (text\))d Fs(is)75 993 y Fh(MPI_INIT_SEND\(handle,)21 b(buffer_handle,)h(dest,)h(tag,)g(context,)g(MPI_STANDARD,)75 1049 y(MPI_EPHEMERAL\))75 1106 y(MPI_START\(handle\))75 1162 y(MPI_WAIT\(handle,)f(null\))166 1298 y Ft(MPI)p 275 1298 V 18 w(RSEND)c(\(bu\013er)p 649 1298 V 18 w(handle,)g(dest,)f (tag,)h(con)o(text\))d Fs(is)75 1435 y Fh(MPI_INIT_SEND\(handle,)21 b(buffer_handle,)h(dest,)h(tag,)g(context,)g(MPI_READY,)g (MPI_EPHEMERAL\))75 1491 y(MPI_START\(handle\))75 1548 y(MPI_WAIT\(handle,)f(null\))166 1683 y Ft(MPI)p 275 1683 V 18 w(SSEND)c(\(bu\013er)p 639 1683 V 18 w(handle,)g(dest,)f (tag,)h(con)o(text\))d Fs(is)75 1820 y Fh(MPI_INIT_SEND\(handle,)21 b(buffer_handle,)h(dest,)h(tag,)g(context,)g(MPI_SECURE,)g (MPI_EPHEMERAL\))75 1877 y(MPI_START\(handle\))75 1933 y(MPI_WAIT\(handle,)f(null\))166 2068 y Ft(MPI)p 275 2068 V 18 w(RECV\(bu\013er)p 599 2068 V 18 w(handle,)c(source,)e(tag,)i (con)o(text,)f(return)p 1427 2068 V 19 w(handle\))f Fs(is)75 2205 y Fh(MPI_INIT_RECV\(handle,)21 b(buffer_handle,)h(source,)h(tag,)g (context,)g(MPI_STANDARD,)f(MPI_EPHEMERAL\))75 2262 y (MPI_START\(handle\))75 2318 y(MPI_WAIT\(handle,return_han)o(dle\))166 2454 y Ft(MPI)p 275 2454 V 18 w(SRECV\(bu\013er)p 628 2454 V 18 w(handle,)c(source,)e(tag,)i(con)o(text,)f(return)p 1456 2454 V 19 w(handle\))f Fs(is)75 2591 y Fh(MPI_INIT_RECV\(handle,) 21 b(buffer_handle,)h(source,)h(tag,)g(context,)g(MPI_SECURE,)f (MPI_EPHEMERAL\))75 2647 y(MPI_START\(handle\))75 2704 y(MPI_WAIT\(handle,return_han)o(dle\))p eop %%Page: 29 34 29 33 bop 75 -100 a Fq(2.10.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)845 b Fs(29)166 45 y Fl(Implemen)o(tati)o(on)12 b(note:)166 95 y Fk(While)f(these)i(functions)f(can)h(b)q(e)f(implemen)o(ted)e(via) h(calls)h(to)f(functions)h(that)g(implemen)o(t)e(sub)q(op)q(erations,) 75 145 y(as)k(describ)q(ed)h(in)e(this)g(subsection,)i(an)e(e\016cien)o (t)h(implemen)o(tatio)o(n)d(ma)o(y)g(optimize)h(a)o(w)o(a)o(y)h(these)i (m)o(ultiple)c(calls,)75 195 y(pro)o(vided)j(it)f(do)q(es)i(not)f(c)o (hange)g(the)g(b)q(eha)o(vior)g(of)f(correct)j(programs.)166 410 y Fl(Discussion:)166 460 y Fk(W)m(e)e(use)g(a)g(di\013eren)o(t)h (function)f(name,)e(rather)j(than)f(an)g(additional)e(mo)q(de)h (parameter,)h(in)f(order)i(to)f(sa)o(v)o(e)75 509 y(on)g(an)g (additional)f(parameter)h(\(p)q(erformance)g(and)g(user)i(con)o(v)o (enience\).)k(Lyndon)14 b(and)g(p)q(erhaps)i(other)f(prefer)75 559 y(few)o(er)g(functions)e(and)h(an)g(additional)e(parameter.)75 783 y Fm(2.10)70 b(Non)n(blo)r(c)n(king)22 b(Comm)n(unication)75 885 y Fs(Non)o(blo)q(c)o(king)f(send)f(and)f(receiv)o(e)i(op)q (erations)f(com)o(bine)g(the)g(\014rst)f(t)o(w)o(o)f(sub)q(op)q (erations)i(\()p Fh(INIT)f Fs(and)75 941 y Fh(START)p Fs(\))c(in)o(to)g(one)h(call.)22 b(They)16 b(use)g(ephemeral)h(comm)o (unication)f(ob)s(jects,)f(so)g(that)g(the)h(op)q(eration)f(is)75 998 y(completed,)g(and)g(the)g(asso)q(ciated)f(resources)h(are)f (freed,)h(b)o(y)f(using)h(one)g(of)f(the)h(functions)g Fh(MPI)p 1742 998 15 2 v 17 w(WAIT,)75 1054 y(MPI)p 150 1054 V 17 w(STATUS,)23 b(MPI)p 430 1054 V 16 w(WAITANY,)g(MPI)p 733 1054 V 17 w(STATUSANY)p Fs(,)c(or)i Fh(MPI)p 1132 1054 V 17 w(WAITALL)p Fs(.)e(Here,)j(to)q(o,)g(a)f(bu\013er)g(ob)s (ject)75 1111 y(has)15 b(to)g(b)q(e)g(created)h(ahead)f(of)g(the)g (comm)o(unication)h(initiation)h(op)q(eration.)166 1167 y(W)l(e)d(use)h(the)f(same)g(naming)g(con)o(v)o(en)o(tion)g(as)g(for)f (blo)q(c)o(king)j(op)q(erations:)j(a)14 b(pre\014x)h(of)e Ft(R)i Fs(\()p Ft(S)p Fs(\))e(indi-)75 1223 y(cates)h(the)g Fh(READY)g Fs(\()p Ft(SECURE)p Fs(\))g(mo)q(de.)20 b(In)15 b(addition,)g(a)f(pre\014x)h(of)f Ft(I)g Fs(is)g(used)h(to)f(indicate)i Fi(imme)n(diate)75 1280 y Fs(\(i.e.,)f(non)o(blo)q(c)o(king\))h (execution.)166 1372 y Ft(MPI)p 275 1372 16 2 v 18 w(ISEND)i(\(handle,) g(bu\013er)p 813 1372 V 18 w(handle,)g(dest,)f(tag,)g(con)o(text\))f Fs(is)75 1491 y Fh(MPI_INIT_SEND\(handle,)21 b(buffer_handle,)h(dest,)h (tag,)g(context,)g(MPI_STANDARD,)f(MPI_EPHEMERAL\))75 1548 y(MPI_START\(handle\))166 1668 y Ft(MPI)p 275 1668 V 18 w(IRSEND)c(\(handle,)g(bu\013er)p 852 1668 V 18 w(handle,)g(dest,)f(tag,)h(con)o(text\))d Fs(is)75 1787 y Fh(MPI_INIT_SEND\(handle,)21 b(buffer_handle,)h(dest,)h(tag,)g (context,)g(MPI_READY,)g(MPI_EPHEMERAL\))75 1844 y(MPI_START\(handle\)) 166 1964 y Ft(MPI)p 275 1964 V 18 w(ISSEND)18 b(\(handle,)g(bu\013er)p 842 1964 V 18 w(handle,)g(dest,)f(tag,)g(con)o(text\))f Fs(is)75 2083 y Fh(MPI_INIT_SEND\(handle,)21 b(buffer_handle,)h(dest,)h (tag,)g(context,)g(MPI_SECURE,)g(MPI_EPHEMERAL\))75 2140 y(MPI_START\(handle\))166 2259 y Ft(MPI)p 275 2259 V 18 w(IRECV\(handle,)13 b(bu\013er)p 797 2259 V 18 w(handle,)g(source,)f (tag,)h(con)o(text,)g(return)p 1607 2259 V 18 w(status)p 1761 2259 V 19 w(handle\))75 2316 y Fs(is)75 2436 y Fh (MPI_INIT_RECV\(handle,)21 b(buffer_handle,)h(source,)h(tag,)g (context,)g(MPI_STANDARD,)f(MPI_EPHEMERAL\))75 2492 y (MPI_START\(handle\))166 2612 y Ft(MPI)p 275 2612 V 18 w(ISRECV\(handle,)13 b(bu\013er)p 826 2612 V 18 w(handle,)g(source,)f (tag,)h(con)o(text,)g(return)p 1636 2612 V 18 w(status)p 1790 2612 V 19 w(handle\))75 2668 y Fs(is)p eop %%Page: 30 35 30 34 bop 75 -100 a Fs(30)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fh (MPI_INIT_RECV\(handle,)21 b(buffer_handle,)h(source,)h(tag,)g (context,)g(MPI_SECURE,)f(MPI_EPHEMERAL\))75 102 y(MPI_START\(handle\)) 75 253 y Fm(2.11)70 b(Con)n(tiguous)23 b(Blo)r(c)n(k)f(Comm)n (unication)f(Op)r(erations)75 358 y Fs(The)g(most)f(frequen)o(t)h(t)o (yp)q(e)f(of)h(bu\013er)g(used)g(is)g(a)g(bu\013er)f(with)h(one)g(con)o (tiguous)g(comp)q(onen)o(t.)37 b(W)l(e)75 414 y(sp)q(ecialize)17 b(the)d(functions)h(in)g(the)f(t)o(w)o(o)f(previous)i(subsections)g(to) f(this)g(case,)g(th)o(us)g(a)o(v)o(oiding)h(the)f(need)75 470 y(for)h(the)g(creation)h(of)f(a)g(bu\013er)g(descriptor)h(ob)s (ject.)k(W)l(e)c(use)f(the)h(same)f(naming)h(sc)o(heme)g(used)g(in)g (the)75 527 y(previous)g(subsections,)g(and)f(app)q(end)h(a)f Ft(C)g Fs(in)h(the)g(function)g(name,)e(for)h Fh(CONTIGUOUS)p Fs(.)166 620 y Ft(MPI)p 275 620 16 2 v 18 w(SENDC)i(\(start,)h(coun)o (t,)f(datat)o(yp)q(e,)h(dest,)f(tag,)h(con)o(text\))e Fs(is)75 757 y Fh(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h (MPI_EPHEMERAL\))75 813 y(MPI_APPEND\()g(buffer_handle,)g(start,)h (count,)g(datatype\))75 870 y(MPI_SEND)g(\(buffer_handle,)f(count,)h (dest,)g(tag,)g(context\))166 1005 y Ft(MPI)p 275 1005 V 18 w(RSENDC)18 b(\(handle,)g(start,)f(coun)o(t,)g(datat)o(yp)q(e,)h (dest,)f(tag,)h(con)o(text\))e Fs(is)75 1142 y Fh(MPI_CREATE_BUFFER\() 21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 1198 y(MPI_APPEND\()g (buffer_handle,)g(start,)h(count,)g(datatype\))75 1255 y(MPI_RSEND\()f(buffer_handle,)g(count,)h(dest,)h(tag,)f(context\))166 1390 y Ft(MPI)p 275 1390 V 18 w(SSENDC)18 b(\(handle,)g(start,)f(coun)o (t,)g(datat)o(yp)q(e,)h(dest,)f(tag,)h(con)o(text\))e Fs(is)75 1526 y Fh(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h (MPI_EPHEMERAL\))75 1583 y(MPI_APPEND\()g(buffer_handle,)g(start,)h (count,)g(datatype\))75 1639 y(MPI_SSEND\()f(buffer_handle,)g(count,)h (dest,)h(tag,)f(context\))166 1774 y Ft(MPI)p 275 1774 V 18 w(RECV)o(C)11 b(\(start,)h(coun)o(t,)h(datat)o(yp)q(e,)g(source,)f (tag,)h(con)o(text,)g(return)p 1643 1774 V 18 w(status)p 1797 1774 V 19 w(handle\))75 1831 y Fs(is)75 1968 y Fh (MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 2024 y(MPI_APPEND\()g(buffer_handle,)g(start,)h(count,)g(datatype\))75 2080 y(MPI_RECV\()g(buffer_handle,)f(source,)g(tag,)i(context,)f (return_status_handle\))166 2216 y Ft(MPI)p 275 2216 V 18 w(SRECV)o(C)11 b(\(start,)i(coun)o(t,)f(datat)o(yp)q(e,)i(source,) d(tag,)j(con)o(text,)e(return)p 1672 2216 V 18 w(status)p 1826 2216 V 19 w(handle\))75 2272 y Fs(is)75 2409 y Fh (MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 2465 y(MPI_APPEND\()g(buffer_handle,)g(start,)h(count,)g(datatype\))75 2522 y(MPI_SRECV\()f(buffer_handle,)g(source,)h(tag,)h(context,)e (return_status_handle\))166 2657 y Ft(MPI)p 275 2657 V 18 w(ISENDC)17 b(\(handle,)i(start,)e(coun)o(t,)g(datat)o(yp)q(e,)h (dest,)f(tag,)h(con)o(text\))d Fs(is)p eop %%Page: 31 36 31 35 bop 75 -100 a Fq(2.12.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1138 b Fs(31)75 45 y Fh(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h (MPI_EPHEMERAL\))75 102 y(MPI_APPEND\()g(buffer_handle,)g(start,)h (count,)g(datatype\))75 158 y(MPI_ISEND\()f(handle,)h(buffer_handle,)f (dest,)i(tag,)f(context\))166 289 y Ft(MPI)p 275 289 16 2 v 18 w(IRSENDC)17 b(\(handle,)i(start,)e(coun)o(t,)g(datat)o(yp)q (e,)h(dest,)f(tag,)h(con)o(text\))e Fs(is)75 420 y Fh (MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 476 y(MPI_APPEND\()g(buffer_handle,)g(start,)h(count,)g(datatype\))75 532 y(MPI_IRSEND\()f(handle,)h(buffer_handle,)f(dest,)h(tag,)h (context\))166 663 y Ft(MPI)p 275 663 V 18 w(ISSENDC)17 b(\(handle,)i(start,)e(coun)o(t,)g(datat)o(yp)q(e,)h(dest,)f(tag,)h (con)o(text\))d Fs(is)75 794 y Fh(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 850 y(MPI_APPEND\()g (buffer_handle,)g(start,)h(count,)g(datatype\))75 907 y(MPI_ISSEND\()f(handle,)h(buffer_handle,)f(dest,)h(tag,)h(context\)) 166 1038 y Ft(MPI)p 275 1038 V 18 w(IRECV)o(C\(handle,)12 b(start,)h(coun)o(t,)g(datat)o(yp)q(e,)g(source,)f(tag,)h(con)o(text,)f (return)p 1829 1038 V 19 w(status)p 1984 1038 V 19 w(handle\))75 1094 y Fs(is)75 1225 y Fh(MPI_CREATE_BUFFER)22 b(\(buffer_handle,)f (MPI_EPHEMERAL\))75 1281 y(MPI_APPEND\()h(buffer_handle,)g(start,)h (count,)g(datatype\))75 1338 y(MPI_IRECV\()f(handle,)h(buffer_handle,)f (source,)h(tag,)h(context\))166 1468 y Ft(MPI)p 275 1468 V 18 w(ISRECV)o(C\(handle,)j(start,)g(coun)o(t,)h(datat)o(yp)q(e,)g (source,)e(tag,)i(con)o(text,)f(re-)75 1525 y(turn)p 178 1525 V 19 w(status)p 333 1525 V 19 w(handle\))16 b Fs(is)75 1656 y Fh(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h (MPI_EPHEMERAL\))75 1712 y(MPI_APPEND\()g(buffer_handle,)g(start,)h (count,)g(datatype\))75 1769 y(MPI_ISRECV\()f(handle,)h(buffer_handle,) f(source,)h(tag,)g(context\))75 1914 y Fm(2.12)70 b(Prob)r(e)23 b(and)h(Cancel)75 2016 y Fs(The)d Fh(MPI)p 249 2016 15 2 v 17 w(PROBE)f Fs(op)q(eration)h(allo)o(ws)g(incoming)h(messages)f (to)f(b)q(e)i(c)o(hec)o(k)o(ed)f(for,)g(without)g(actually)75 2072 y(receiving)16 b(them.)k(The)15 b(user)g(can)g(then)h(decide)g (where)f(to)g(receiv)o(e)g(them,)g(based)g(on)g(the)g(information)75 2129 y(returned)h(b)o(y)f(the)g(prob)q(e)g(\(basically)l(,)i(the)e (information)g(on)g(the)h(message)e(en)o(v)o(elop)q(e\).)21 b(An)16 b(additional)75 2185 y(function,)k Fh(MPI)p 345 2185 V 17 w(GET)p 434 2185 V 17 w(LEN)e Fs(allo)o(ws)h(the)f(amoun)o(t) g(of)g(storage)g(needed)i(to)e(receiv)o(e)h(the)g(message)f(to)g(b)q(e) 75 2242 y(computed,)23 b(when)g(this)f(length)g(is)h(not)e(readily)i (computed)f(from)f(the)h(information)g(returned)g(b)o(y)75 2298 y Fh(MPI)p 150 2298 V 17 w(PROBE)p Fs(.)166 2355 y(The)e Fh(MPI)p 339 2355 V 17 w(CANCEL)f Fs(op)q(eration)h(allo)o(ws)h (p)q(ending)g(comm)o(unications)g(to)e(b)q(e)i(cancelled.)37 b(This)20 b(is)75 2412 y(required)13 b(for)f(clean)o(up.)20 b(P)o(osting)12 b(a)f(send)i(or)f(a)g(receiv)o(e)h(ties)f(user)g (resources)h(\(send)f(or)g(receiv)o(e)h(bu\013ers\),)75 2468 y(and)i(a)g(cancel)i(ma)o(y)d(b)q(e)i(needed)g(to)f(free)g(these)h (resources)f(gracefully)l(.)166 2560 y Ft(MPI)p 275 2560 16 2 v 18 w(PR)o(OBE\()j(source,)e(tag,)i(con)o(text,)f(\015ag,)h (datat)o(yp)q(e,)g(return)p 1488 2560 V 18 w(status\))75 2704 y(IN)f(source)22 b Fs(rank)15 b(in)h(con)o(text)f(of)g(source,)g (or)f Fh(MPI)p 974 2704 15 2 v 17 w(ANY)p 1063 2704 V 17 w(SOURCE)g Fs(\(in)o(teger\).)p eop %%Page: 32 37 32 36 bop 75 -100 a Fs(32)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Ft(IN)h(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(receiv)o(ed)h(with)g(this)f (handle,)h(or)f Fh(MPI)p 1341 45 15 2 v 17 w(ANY)p 1430 45 V 16 w(TAG)g Fs(\(in)o(teger\).)75 140 y Ft(IN)i(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(receiv)o(ed)h(with)g(this)g (handle.)75 235 y Ft(OUT)i(\015ag)24 b Fs(\(logical\))75 330 y Ft(IN)17 b(t)o(yp)q(e)23 b Fs(assumed)15 b(t)o(yp)q(e)g(of)g (data)g(in)h(message)e(\(status)g(v)m(ariable\).)75 426 y Ft(OUT)k(return)p 357 426 16 2 v 19 w(handle)23 b Fs(handle)17 b(that)d(is)i(asso)q(ciated)f(with)h(return)f(status)f(ob)s(ject.)166 533 y Fh(MPI)p 241 533 15 2 v 17 w(PROBE)k Fs(returns)g Fh(flag)23 b(=)h(true)18 b Fs(if)h(there)g(is)g(a)g(message)f(that)g (can)h(b)q(e)g(receiv)o(ed)h(and)f(that)75 590 y(matc)o(hes)h(the)g (pattern)g(sp)q(eci\014ed)j(b)o(y)d(the)g(parameters)g Fh(source)p Fs(,)g Fh(tag)p Fs(,)h(and)f Fh(context)p Fs(.)35 b(It)20 b(returns)75 646 y Fh(flag)j(=)h(false)p Fs(,)15 b(otherwise.)21 b(If)15 b Fh(MPI)p 731 646 V 17 w(PROBE)g Fs(returns)g Fh(flag)24 b(=)f(true)p Fs(,)15 b(then)h(the)f(length,)h(source)g(and)75 703 y(tag)g(of)g(the)h (message)g(matc)o(hed)g(are)f(returned)h(in)h(the)f(return)g(status)f (ob)s(ject.)24 b(These)18 b(are)e(the)h(same)75 759 y(v)m(alues)i(that) d(w)o(ould)i(ha)o(v)o(e)f(b)q(een)i(returned)f(b)o(y)f(a)h(call)g(to)f Fh(MPI)p 1176 759 V 17 w(RECV)g Fs(or)g(to)g Fh(MPI)p 1494 759 V 16 w(SRECV)g Fs(executed)i(at)75 816 y(the)d(same)f(p)q(oin) o(t)h(in)h(the)f(program)e(\(with)i(a)f(ca)o(v)o(eat)g(concerning)i (length;)f(see)g(b)q(elo)o(w\).)22 b(These)16 b(v)m(alues)75 872 y(can)j(b)q(e)h(deco)q(ded)g(from)e(the)h(return)g(status)f(ob)s (ject)g(using)i(the)f Fh(MPI)p 1302 872 V 16 w(RETURN)p 1462 872 V 17 w(STAT)f Fs(function.)32 b(The)75 929 y(v)m(alue)16 b(returned)g(in)g(the)f(return)g(status)g(ob)s(ject)f(is)i(unde\014ned) h(if)f Fh(flag=false)p Fs(.)166 985 y(The)21 b(length)g(v)m(alue)h (returned)g(b)o(y)e(the)h(return)g(status)f(ob)s(ject)g(is)h (\(correctly\))f(the)h(n)o(um)o(b)q(er)g(of)75 1042 y(elemen)o(ts)14 b(in)h(the)f(message,)f(pro)o(vided)i(that)e(all)h(elemen)o(ts)h(in)f (the)g(message)f(are)h(of)f(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)75 1098 y(b)o(y)h(the)g Fh(type)g Fs(parameter;)f(otherwise)h(the)h (length)g(v)m(alue)g(returned)g(is)f(unde\014ned.)166 1155 y(A)e(subsequen)o(t)i(receiv)o(e)f(executed)g(with)g(the)g(same)f (con)o(text,)g(and)g(the)h(source)f(and)h(tag)f(returned)75 1212 y(b)o(y)f(the)g(call)i(to)d Fh(MPI)p 418 1212 V 17 w(PROBE)g Fs(will)j(receiv)o(e)f(the)g(message)e(that)h(w)o(as)f (matc)o(hed)h(b)o(y)g(the)h(prob)q(e,)g(if)f(no)g(other)75 1268 y(in)o(terv)o(ening)k(receiv)o(e)g(o)q(ccurred)f(after)f(the)h (prob)q(e.)21 b(If)15 b(the)g(receiving)h(pro)q(cess)f(is)h(m)o (ultithreaded,)f(it)g(is)75 1324 y(the)g(user)h(resp)q(onsibilit)o(y)h (to)e(ensure)h(that)e(the)h(last)h(condition)g(holds.)166 1457 y Fl(Discussion:)166 1507 y Fk(Do)g(w)o(e)g(w)o(an)o(t)g Fg(MPI)p 471 1507 14 2 v 15 w(PROBE)f Fk(to)i(return)g(mo)q(de)e (\(standard)i(or)g(secure\)?)27 b(If)16 b(y)o(es,)h(w)o(e)g(need)g(to)f (carry)h(mo)q(de)75 1557 y(information)c(with)i(messages)g(\(it's)g (deja)h(vu)f(all)f(o)o(v)o(er)i(again\).)21 b(If)15 b(not,)h(then)g(it) f(is)g(up)g(to)h(the)g(user)g(to)g(enco)q(de)75 1607 y(this)e(information)d(\(in)i(the)i(tag\))f(so)g(that)g(receiv)o(er)h (can)f(decide)h(whether)g(to)f(use)h(a)e(secure)j(or)e(standard)g (send.)166 1657 y(MPI)h(guaran)o(tees)g(that)g(successiv)o(e)i (messages)e(sen)o(t)h(from)d(a)h(source)i(to)f(a)f(destination)h (within)e(the)j(same)75 1707 y(con)o(text)21 b(are)g(receiv)o(ed)h(in)e (the)h(order)g(they)g(are)g(sen)o(t.)39 b(Th)o(us,)22 b(MPI)e(m)o(ust)g(supp)q(ort,)i(either)g(explicitly)d(or)75 1757 y(implicitly)l(,)13 b(a)i(FIF)o(O)h(structure)i(to)d(manage)f (messages)i(b)q(et)o(w)o(een)h(eac)o(h)f(pair)f(of)g(messages,)g(for)h (eac)o(h)g(con)o(text.)75 1807 y Fg(MPI)p 144 1807 V 15 w(PROBE)h Fk(returns)i(information)c(on)j(the)h(\014rst)f(matc)o (hing)e(message)i(in)f(this)h(FIF)o(O;)g(this)g(will)f(also)g(b)q(e)h (the)75 1856 y(message)i(receiv)o(ed)i(b)o(y)e(the)h(\014rst)h (subsequen)o(t)g(receiv)o(e)g(with)e(the)h(same)e(source,)k(tag)d(and)h (con)o(text)g(as)f(the)75 1906 y(message)14 b(matc)o(hed)f(b)o(y)g Fg(MPI)p 528 1906 V 16 w(PROBE)p Fk(.)166 1956 y(Message)g(passing)f (in)g(MPI)g(can)h(b)q(e)g(implem)o(en)o(ted)d(without)i(app)q(ending)g (t)o(yp)q(e)h(information)c(to)j(messages.)75 2006 y(A)h(message)h(is)f (merely)f(a)h(string)h(of)f(b)o(ytes)h(and)f(the)h(in)o(terpretation)g (of)e(these)j(b)o(ytes)f(in)o(to)f(a)g(sequence)j(of)c(t)o(yp)q(ed)75 2056 y(elemen)o(ts)18 b(is)g(done)g(using)g(the)g(information)d(in)j (the)g(bu\013er)h(descriptors)h(at)e(eac)o(h)g(end.)31 b(The)18 b(abilit)o(y)e(to)i(use)75 2106 y(suc)o(h)g(implem)o(en)o (tation)c(strategy)j(is)g(deemed)g(to)g(b)q(e)g(an)g(imp)q(ortan)o(t)e (goal.)25 b(In)17 b(suc)o(h)h(implemen)o(tatio)o(n,)c(when)75 2156 y(a)j(message)g(arriv)o(es,)h(it)f(is)g(not)g(b)q(e)g(kno)o(wn)g (ho)o(w)g(man)o(y)e(elemen)o(ts)i(it)g(con)o(tains,)h(or)f(ev)o(en)h (ho)o(w)f(m)o(uc)o(h)f(storage)75 2205 y(is)e(needed)h(to)f(receiv)o(e) h(that)f(message)f(\(b)q(ecause)j(of)d(p)q(ossible)h(represen)o(tation) i(con)o(v)o(ersion)e(in)f(a)h(heterogeneous)75 2255 y(en)o(vironmen)o (t\).)i(The)c(prob)q(e)f(function)g(cannot)g(use)h(a)f(bu\013er)h (descriptor;)h(this)e(defeats)h(the)g(purp)q(ose)g(of)f(probing)75 2305 y(in)k(order)h(to)g(decide)g(where)h(to)e(receiv)o(e)i(a)e (message.)23 b(Therefore,)17 b(prob)q(e)f(cannot,)g(in)f(general,)h (return)g(correct)75 2355 y(length)10 b(information.)k(Still,)9 b(it)h(is)f(often)i(the)f(case)h(that)f(prob)q(e)h(is)f(used)h(to)f (decide)h(ho)o(w)e(m)o(uc)o(h)g(storage)i(to)e(allo)q(cate)75 2405 y(in)15 b(order)i(to)e(receiv)o(e)i(a)f(message.)23 b(Enco)q(ding)16 b(suc)o(h)g(information)d(in)i(the)h(message)g(tag)f (is)h(deemed)f(to)h(b)q(e)g(to)q(o)75 2455 y(a)o(wkw)o(ard,)g(and)h(it) f(is)g(deemed)h(imp)q(ortan)o(t)e(for)h Fg(MPI)p 923 2455 V 15 w(PROBE)g Fk(to)g(return)i(some)e(useful)h(size)g (information.)24 b(The)75 2504 y(curren)o(t)13 b(de\014nition)e(of)f Fg(MPI)p 512 2504 V 15 w(PROBE)g Fk(is)i(a)e(compromise)f(b)q(et)o(w)o (een)k(these)g(t)o(w)o(o)e(goals.)16 b(F)m(or)11 b(the)h(most)e(common) e(case)75 2554 y(of)13 b(messages)g(where)i(all)d(en)o(tries)j(ha)o(v)o (e)e(the)h(same)f(t)o(yp)q(e)h Fg(MPI)p 1039 2554 V 15 w(PROBE)f Fk(returns)i(the)f(correct)h(length)e(information;)75 2604 y(the)h(more)f(esoteric)i(case)f(is)g(handled)f(b)o(y)g(the)i Fg(MPI)p 883 2604 V 15 w(GET)p 964 2604 V 15 w(LEN)e Fk(that)g(is)h(describ)q(ed)h(b)q(elo)o(w.)j(The)c(curren)o(t)h (solution)75 2654 y(sa)o(v)o(es)h(us)f(the)h(need)g(for)f(one)g (additional)f(w)o(ord)h(p)q(er)h(message)f(that)g(w)o(ould)f(otherwise) j(b)q(e)e(needed)i(to)e(transfer)75 2704 y(the)f(message)g(length)g (\(in)f(elemen)o(ts\))h(with)g(the)g(message.)p eop %%Page: 33 38 33 37 bop 75 -100 a Fq(2.13.)34 b(CORRECTNESS)1280 b Fs(33)166 45 y Fk(There)15 b(w)o(as)f(a)f(request)j(for)d(a)h(blo)q(c)o (king)f(prob)q(e)h(function.)166 220 y Ft(MPI)p 275 220 16 2 v 18 w(GET)p 404 220 V 20 w(LEN\()k(coun)o(t,)f(return)p 872 220 V 18 w(status,)h(bu\013er)p 1194 220 V 18 w(descriptor\))166 312 y Fs(Computes)j(the)f(n)o(um)o(b)q(er)i(of)e(elemen)o(ts)i(that)e (w)o(ere)h(to)f(b)q(e)h(receiv)o(ed,)i(if)f(the)f(message)f(that)g(is) 75 368 y(asso)q(ciated)13 b(with)g(the)g(return)g(status)f(handle)j(w)o (ould)e(b)q(e)h(receiv)o(ed)g(in)f(the)h(bu\013er)e(asso)q(ciated)i (with)f(the)75 424 y(bu\013er)i(descriptor)h(handle.)75 506 y Ft(OUT)i(coun)o(t)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(elemen)o(ts)g (that)g(w)o(ere)g(to)f(b)q(e)i(receiv)o(ed)h(\(in)o(teger\))75 594 y Ft(IN)g(return)p 302 594 V 18 w(status)23 b Fs(handle)17 b(to)d(return)i(status)e(descriptor)75 683 y Ft(IN)j(bu\013er)p 291 683 V 18 w(descriptor)23 b Fs(handle)16 b(to)f(bu\013er)g (descriptor)166 840 y Fl(Discussion:)166 890 y Fk(In)k(order)g(to)f (supp)q(ort)i(this)f(function,)g(an)f(additional)f(\014eld)h(is)h (needed)h(in)e(the)h(return)h(status)g(ob)r(ject,)75 940 y(i.e.,)c(n)o(um)o(b)q(er)f(of)h(b)o(ytes)i(in)e(the)h(incoming)d (message)i(matc)o(hed)g(b)o(y)g(prob)q(e,)h(or)g(v)n(alue)f(of)g(datat) o(yp)q(e)g(parameter)75 989 y(pro)o(vided)e(b)o(y)f(prob)q(e)h (\(unless)h(w)o(e)f(add)f(this)h(parameter)f(to)h(the)g Fg(MPI)p 1169 989 14 2 v 15 w(GET)p 1250 989 V 15 w(LEN)f Fk(function,)g(whic)o(h)g(is)h(to)q(o)f(ugly)g(to)75 1039 y(b)q(ear\).)166 1214 y Ft(MPI)p 275 1214 16 2 v 18 w(CANCEL\()k(handle,)h(\015ag\))75 1338 y(IN)f(handle)24 b Fs(handle)16 b(to)f(comm)o(unication)h(ob)s(ject)75 1426 y Ft(OUT)i(\015ag)24 b Fs(\(logical\))166 1515 y(A)19 b(call)i(to)e Fh(MPI)p 442 1515 15 2 v 17 w(CANCEL)f Fs(cancels)j(a)e(p)q(ending)i(comm)o(unication)f(op)q(eration)g(\(send) g(or)f(receiv)o(e\).)75 1571 y(The)d(call)h(returns)e Fh(flag)24 b(=)f(true)15 b Fs(if)i(the)e(cancel)i(op)q(eration)f (succeeded,)h Fh(flag)23 b(=)h(false)15 b Fs(otherwise.)75 1628 y(It)h(m)o(ust)f(b)q(e)i(the)f(case)g(that)f(either)i(the)f (cancel)h(op)q(eration)f(succeeds)h(or)e(that)g(the)h(p)q(ending)i (comm)o(u-)75 1684 y(nication)g(op)q(eration)e(completes)i(\(but)e(not) g(b)q(oth\).)24 b(If)17 b(a)f(comm)o(unication)h(op)q(eration)g(w)o(as) f(cancelled)75 1741 y(successfully)l(,)k(the)e(e\013ect)g(is)g(as)g(if) g(this)g(op)q(eration)h(w)o(as)e(nev)o(er)h(executed.)28 b(If)19 b(a)e(send)i(w)o(as)e(cancelled)75 1797 y(successfully)22 b(then)e(the)g(message)f(sen)o(t)g(will)j(not)d(b)q(e)h(receiv)o(ed,)i (the)e(receiv)o(e)g(bu\013er)g(of)f(an)o(y)h(p)q(osted)75 1854 y(receiv)o(e)g(for)f(that)g(message)g(will)i(not)e(b)q(e)h (altered,)g(and)g(an)o(y)f(suc)o(h)h(receiv)o(e)g(has)f(to)g(b)q(e)h (satis\014ed)g(b)o(y)75 1910 y(another)15 b(send;)g(if)g(a)g(receiv)o (e)h(w)o(as)e(cancelled)j(successfully)l(,)g(then)e(the)g(receiv)o(e)h (bu\013er)f(p)q(osted)g(will)h(not)75 1966 y(b)q(e)g(altered,)f(and)h (an)o(y)e(send)i(matc)o(hing)f(this)h(receiv)o(e)g(has)f(to)g(b)q(e)h (satis\014ed)f(b)o(y)h(another)e(receiv)o(e.)166 2099 y Fl(Implemen)o(tati)o(on)e(note:)166 2149 y Fk(There)j(is)f(not)f(exp) q(ectation)i(that)f(a)g(cancel)g(op)q(eration)g(will)e(b)q(e)j(fast.)75 2372 y Fm(2.13)70 b(Correctness)75 2556 y Fl(Discussion:)17 b Fk(The)d(material)f(in)h(this)g(section)h(has)g(not)f(y)o(et)h(b)q (een)h(discussed)g(b)o(y)e(MPIF.)g(Some)g(or)g(all)f(of)h(it)g(is)75 2613 y(lik)o(ely)f(to)g(mo)o(v)o(e)g(to)g(Section)i(6.)i(It)d(is)g (incorp)q(orated)h(here)g(for)e(completeness.)p eop %%Page: 34 39 34 38 bop 75 -100 a Fs(34)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fj(2.13.1)55 b(Order)75 137 y Fs(MPI)19 b(preserv)o(es)f(the)h(order)g(of)f (messages)g(within)i(the)f(same)f(con)o(text,)h(b)q(et)o(w)o(een)g(an)o (y)f(\014xed)h(pair)g(of)75 193 y(pro)q(cesses.)26 b(In)18 b(other)e(w)o(ords,)h(if)g(pro)q(cess)g(A)h(executes)f(t)o(w)o(o)f (successiv)o(e)i(send)g Fh(start)e Fs(sub)q(op)q(erations)75 249 y(within)i(the)f(same)f(con)o(text,)g(pro)q(cess)h(B)g(executes)g (t)o(w)o(o)f(successiv)o(e)h(receiv)o(e)h Fh(start)e Fs(op)q(erations,)h(and)75 306 y(b)q(oth)e(receiv)o(es)g(matc)o(h)f (either)h(sends,)g(then)g(the)g(\014rst)f(receiv)o(e)i(will)g(receiv)o (e)f(the)g(message)f(sen)o(t)g(b)o(y)h(the)75 362 y(\014rst)f(send,)g (and)g(the)g(second)g(receiv)o(e)h(will)h(receiv)o(e)e(the)g(message)g (sen)o(t)g(b)o(y)f(the)h(second)h(send.)20 b(Th)o(us,)13 b(if)75 419 y(a)g(t)o(w)o(o)g(messages)g(from)f(the)i(same)f(source)h (can)g(satisfy)f(a)g(p)q(ending)j(receiv)o(e,)e(the)g(\014rst)f (message)g(sen)o(t)g(is)75 475 y(accepted;)i(if)h(a)f(message)f(can)h (satisfy)g(t)o(w)o(o)f(p)q(ending)j(receiv)o(es,)e(the)g(\014rst)g (receiv)o(e)h(p)q(osted)f(is)h(satis\014ed.)166 535 y(The)c(last)f (paragraph)g(assumes)g(that)g(the)h(send)g Fh(start)e Fs(op)q(erations)i(are)f(ordered)h(b)o(y)f(the)h(program)75 591 y(order)20 b(at)f(pro)q(cess)h(A,)g(and)g(the)g(receiv)o(e)h Fh(start)e Fs(op)q(erations)h(are)f(ordered)i(b)o(y)e(the)h(program)f (order)75 648 y(at)h(pro)q(cess)h(B.)g(If)g(a)g(pro)q(cess)g(is)g(m)o (ultithreaded)h(and)f(the)g(op)q(erations)h(are)e(executed)i(b)o(y)f (distinct)75 704 y(threads,)d(then)g(the)g(seman)o(tics)f(of)h(the)f (threaded)h(system)g(ma)o(y)f(not)g(de\014ne)i(an)e(order)h(b)q(et)o(w) o(een)g(the)75 760 y(t)o(w)o(o)c(op)q(erations,)h(in)h(whic)o(h)g(case) f(the)g(condition)i(is)f(v)o(oid.)75 899 y Fj(2.13.2)55 b(Progress)19 b(and)g(F)-5 b(airness)75 990 y Fs(W)l(e)20 b(can)g(mo)q(del)g(the)g(execution)h(of)e(MPI)h(programs)e(as)h(an)h (in)o(teraction)g(b)q(et)o(w)o(een)g(executing)h(pro-)75 1047 y(cesses)d(that)f(execute)h(eac)o(h)g(their)g(o)o(wn)f(program,)g (and)g(the)h Ft(comm)o(unication)j(subsystem)p Fs(.)26 b(The)75 1103 y(comm)o(unication)16 b(subsystem)g(ma)o(y)f(ha)o(v)o(e)g (v)m(arious)h(constrain)o(ts)g(on)f(the)h(amoun)o(t)f(of)g(resources)h (it)g(can)75 1160 y(use.)k(E.g.:)166 1219 y(Bounds)e(on)f(the)g(n)o(um) o(b)q(er)h(and)f(total)g(sizes)h(of)f(activ)o(e)h(comm)o(unication)g (ob)s(jects.)25 b(Suc)o(h)18 b(b)q(ound)75 1276 y(can)d(b)q(e)h (global,)g(p)q(er)f(no)q(de,)h(or)e(p)q(er)i(pair)g(of)e(comm)o (unicating)i(no)q(des.)166 1335 y(Bounds)e(on)g(the)g(n)o(um)o(b)q(er)g (and)h(total)e(sizes)i(of)e(messages)g(bu\013ered)i(in)f(the)g(system.) 19 b(Suc)o(h)c(b)q(ound)75 1391 y(can,)k(again,)g(b)q(e)g(global,)g(p)q (er)g(no)q(de,)g(or)f(p)q(er)h(pair)f(of)g(comm)o(unicating)h(no)q(de.) 30 b(In)19 b(addition,)h(a)e(mes-)75 1448 y(sage)f(ma)o(y)g(b)q(e)h (bu\013ered)g(at)f(the)g(sender,)i(at)d(the)i(receiv)o(er,)g(at)f(b)q (oth,)h(or)f(p)q(erhaps)h(at)f(another)g(place)75 1504 y(altogether.)166 1564 y(Th)o(us,)i(it)g(will)i(b)q(e)e(di\016cult)h (to)e(set)h(rules)g(on)g(resource)g(managemen)o(t)f(of)g(the)h(comm)o (unication)75 1620 y(subsystem.)29 b(Ho)o(w)o(ev)o(er,)17 b(it)i(is)g(generally)g(exp)q(ected)g(that)f(implemen)o(ters)h(will)h (pro)o(vide)f(information)75 1677 y(on)c(the)g(mec)o(hanism)g(used)h (for)e(resource)h(allo)q(cation,)h(and)f(that)f(query)i(and)f(set)f (functions)i(will)h(allo)o(w)75 1733 y(to)e(query)g(and)g(p)q(ossibly)i (con)o(trol)e(the)g(amoun)o(t)g(of)f(a)o(v)m(ailable)j(resources.)166 1792 y(W)l(e)g(pro)o(vide)h(in)g(this)f(section)h(a)f(set)g(of)g (minimal)h(requiremen)o(ts)g(on)f(the)h(comm)o(unication)f(sub-)75 1849 y(system.)26 b(Programs)15 b(that)i(execute)h(on)f(an)o(y)g (subsystem)g(that)f(ful\014ls)j(these)f(minimal)g(requiremen)o(ts)75 1905 y(are)12 b Ft(safe)h Fs(and)g(will)h(p)q(ort)f(to)f(an)o(y)g(MPI)h (implemen)o(tation.)21 b Ft(Unsafe)13 b Fs(programs)e(ma)o(y)h(execute) i(on)e(some)75 1962 y(MPI)k(implemen)o(tations,)h(dep)q(ending)h(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(and)g(the)g (implemen-)75 2018 y(tation)k(used)g(for)g(the)g(MPI)g(comm)o (unication)h(subsystem.)34 b(Finally)22 b Ft(erroneous)d Fs(programs)g(nev)o(er)75 2075 y(execute)e(correctly)l(.)22 b(\(While)17 b(it)f(is)g(desirable)i(to)d(detect)h(erroneous)g (programs,)e(it)i(is)h(not)e(p)q(ossible)j(to)75 2131 y(do)c(so)h(at)e(compile)j(time,)f(and)g(often)f(prohibitiv)o(e)i(to)e (do)h(so)f(a)g(run)h(time.)20 b(Th)o(us,)14 b(the)h(do)q(cumen)o(t)g (do)q(es)75 2188 y(not)f(sp)q(ecify)h(a)f(b)q(eha)o(vior)h(for)f (erroneous)g(programs,)f(although)h(the)g(desired)i(b)q(eha)o(vior)f (is)f(to)g(return)g(a)75 2244 y(useful)i(error)f(message.\))131 2365 y(1.)22 b(If)15 b(a)g(pro)q(cess)g(executes)h(an)f Fh(INIT)f Fs(op)q(eration,)h(then)g(the)h(op)q(eration)f(ev)o(en)o (tually)h(succeeds,)g(or)e(a)189 2421 y Fi(r)n(esour)n(c)n(e)f(exc)n (eption)f Fs(o)q(ccurs.)19 b(The)13 b(standard)f(do)q(es)g(not)h(sp)q (ecify)g(when)g(a)f(resource)h(exception)g(is)189 2478 y(allo)o(w)o(ed)h(to)g(o)q(ccur.)20 b(It)14 b(is)h(exp)q(ected)h(that)d (an)h(op)q(erational)h(de\014nition)h(will)g(b)q(e)f(made)g(a)o(v)m (ailable,)189 2534 y(in)20 b(the)g(form)e(of)h(test)g(programs)g(that)f (ha)o(v)o(e)h(to)g(execute)h(with)g(no)g(resource)f(exceptions.)34 b(It)189 2591 y(is)21 b(highly)i(desirable)f(to)f(ha)o(v)o(e)f (generous)h(b)q(ounds)h(on)f(the)g(n)o(um)o(b)q(er)h(of)e(concurren)o (tly)i(activ)o(e)189 2647 y(comm)o(unication)17 b(ob)s(jects)f(eac)o(h) g(pro)q(cess)h(ma)o(y)f(ha)o(v)o(e,)g(so)g(that,)g(in)i(practice,)f Fh(INIT)f Fs(op)q(erations)189 2704 y(will)g(alw)o(a)o(ys)f(b)q(e)h (guaran)o(teed)f(to)f(succeed.)p eop %%Page: 35 40 35 39 bop 75 -100 a Fq(2.13.)29 b(CORRECTNESS)1285 b Fs(35)131 45 y(2.)22 b(Eac)o(h)16 b(pro)q(cess)g(can)h(initiate)h(a)e (comm)o(unication)h(op)q(eration)g(for)e(eac)o(h)i(activ)o(e)f(comm)o (unication)189 102 y(ob)s(ject.)j(I.e.)h(correct)15 b Fh(START)f Fs(op)q(erations)i(alw)o(a)o(ys)e(succeed)i(\(ev)o(en)o (tually\).)131 190 y(3.)22 b(A)e(send)h(op)q(eration)f(is)h Ft(enabled)h Fs(if)e(the)h(sending)g(pro)q(cess)g(has)f(issued)i(a)e Fh(COMPLETE)f Fs(op)q(era-)189 246 y(tion)f(and)g(the)f(receiving)j (pro)q(cess)e(has)g(issued)g(a)g Fh(START)f Fs(op)q(eration)h(for)f(a)h (matc)o(hing)f(receiv)o(e.)189 303 y(Symmetrically)l(,)k(a)d(receiv)o (e)i(op)q(eration)f(is)g Ft(enabled)g Fs(if)h(the)e(receiving)j(pro)q (cess)e(has)f(issued)i(a)189 359 y Fh(COMPLETE)10 b Fs(op)q(eration)i (and)f(the)h(sending)g(pro)q(cess)g(has)f(issued)i(a)e Fh(START)g Fs(op)q(eration)g(for)g(a)g(matc)o(h-)189 415 y(ing)19 b(send.)31 b(An)19 b(enabled)h(op)q(eration)e(ma)o(y)g(b)q (ecome)i Ft(disabled)f Fs(either)h(b)q(ecause)f(it)g(completes)189 472 y(successfully)e(or,)e(in)h(the)f(case)h(of)f(a)g(receiv)o(e,)h(b)q (ecause)g(the)g(matc)o(hing)f(message)g(is)h(successfully)189 528 y(receiv)o(ed)g(b)o(y)f(another)g(receiv)o(e)g(op)q(eration)h(or,)e (in)i(the)f(case)g(of)g(a)f(send,)i(b)q(ecause)g(the)f(matc)o(hing)189 585 y(receiv)o(e)h(successfully)h(receiv)o(es)f(another)f(message.)189 657 y Ft(An)24 b(enabled)j(op)q(eration)f(either)g(completes)f (successfully)h(or)e(b)q(ecomes)i(p)q(erma-)189 714 y(nen)o(tly)17 b(disabled.)189 786 y Fs(I.e.,)d(an)h(enabled)i(op)q(eration)e(either)h (ev)o(en)o(tually)g(succeeds,)g(or)e(b)q(ecomes)i(disabled)h (\(progress\);)189 842 y(and)i(an)g(op)q(eration)g(cannot)g(b)q(e)h (enabled)h(in\014nitely)g(man)o(y)e(times)g(without)g(ev)o(er)g (succeeding)189 899 y(\(fairness\).)131 987 y(4.)j(A)15 b Fh(FREE)f Fs(op)q(eration)i(alw)o(a)o(ys)e(succeeds)j(\(ev)o(en)o (tually\).)166 1073 y(The)i(four)f(conditions)i(guaran)o(tee)d (progress)h(in)i(the)e(comm)o(unication)i(subsystem.)30 b(The)18 b(third)75 1130 y(condition)e(guaran)o(tee)f(\(w)o(eak\))f (fairness)h(among)g(comp)q(eting)h(comm)o(unication)g(requests.)166 1186 y(Examples)g(\(in)o(v)o(olving)g(t)o(w)o(o)e(pro)q(cessors)h(with) g(ranks)g(0)g(and)g(1\))166 1243 y(The)g(follo)o(wing)h(program)e(is)i (safe,)f(and)g(should)h(alw)o(a)o(ys)f(succeed.)75 1379 y Fh(CALL)23 b(MPI_RANK\(rank,)f(context\))75 1435 y(IF)i (\(rank.EQ.0\))e(THEN)147 1492 y(CALL)h(MPI_SENDC\(sendbuf,)e(count,)i (1,)h(tag,)f(context\))147 1548 y(CALL)g(MPI_RECVC\(recvbuf,)e(count,)i (1,)h(tag,)f(context\))75 1604 y(ELSE)95 b(!)24 b(rank.EQ.1)147 1661 y(CALL)f(MPI_RECVC\(recvbuf,)e(count,)i(0,)h(tag,)f(context\))147 1717 y(CALL)g(MPI_SENDC\(sendbuf,)e(count,)i(0,)h(tag,)f(context\))75 1774 y(END)g(IF)166 1853 y Fs(The)15 b(follo)o(wing)h(program)e(is)i (erroneous,)f(and)g(should)h(alw)o(a)o(ys)f(fail.)75 1996 y Fh(CALL)23 b(MPI_RANK\(rank,)f(context\))75 2053 y(IF)i(\(rank.EQ.0\))e(THEN)147 2109 y(CALL)h(MPI_RECVC\(recvbuf,)e (count,)i(1,)h(tag,)f(context\))147 2166 y(CALL)g(MPI_SENDC\(sendbuf,)e (count,)i(1,)h(tag,)f(context\))75 2222 y(ELSE)95 b(!)24 b(rank.EQ.1)147 2279 y(CALL)f(MPI_RECVC\(recvbuf,)e(count,)i(0,)h(tag,) f(context\))147 2335 y(CALL)g(MPI_SENDC\(sendbuf,)e(count,)i(0,)h(tag,) f(context\))75 2391 y(END)g(IF)166 2478 y Fs(The)15 b(receiv)o(e)g(op)q (eration)g(of)f(the)h(\014rst)f(pro)q(cess)h(m)o(ust)f(complete)i(b)q (efore)f(its)f(send,)h(and)g(can)g(com-)75 2534 y(plete)j(only)g(if)g (the)f(matc)o(hing)g(send)h(of)f(the)g(second)h(pro)q(cessor)f(is)h (executed;)g(the)g(receiv)o(e)g(op)q(eration)75 2591 y(of)c(the)h(second)g(pro)q(cess)g(m)o(ust)f(complete)h(b)q(efore)g (its)f(send)i(and)e(can)h(complete)g(only)g(if)g(the)g(matc)o(hing)75 2647 y(send)h(of)f(the)g(\014rst)g(pro)q(cess)g(is)h(executed.)21 b(This)15 b(program)f(will)j(deadlo)q(c)o(k.)166 2704 y(The)10 b(follo)o(wing)h(program)f(is)g(unsafe,)h(and)g(ma)o(y)e (succeed)j(or)e(fail,)h(dep)q(ending)i(on)d(implemen)o(tation.)p eop %%Page: 36 41 36 40 bop 75 -100 a Fs(36)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 102 y Fh(CALL)23 b(MPI_RANK\(rank,)f(context\))75 158 y(IF)i(\(rank.EQ.0\))e(THEN)147 214 y(CALL)h(MPI_SENDC\(sendbuf,)e(count,)i(1,)h(tag,)f(context\))147 271 y(CALL)g(MPI_RECVC\(recvbuf,)e(count,)i(1,)h(tag,)f(context\))75 327 y(ELSE)95 b(!)24 b(rank.EQ.1)147 384 y(CALL)f(MPI_SENDC\(sendbuf,)e (count,)i(0,)h(tag,)f(context\))147 440 y(CALL)g(MPI_RECVC\(recvbuf,)e (count,)i(0,)h(tag,)f(context\))75 497 y(END)g(IF)166 659 y Fs(The)11 b(message)g(sen)o(t)g(b)o(y)g(eac)o(h)g(pro)q(cess)g (has)g(to)f(b)q(e)i(copied)g(out)f(b)q(efore)g(the)g(send)h(op)q (eration)f(returns)75 716 y(and)k(the)g(receiv)o(e)h(op)q(eration)f (starts.)k(F)l(or)14 b(the)h(program)f(to)g(complete,)h(it)h(is)f (necessary)g(that)g(at)f(least)75 772 y(one)i(of)g(the)h(t)o(w)o(o)e (messages)g(sen)o(t)h(is)h(bu\013ered)g(out)f(of)f(either)i(pro)q (cesses')g(address)f(space.)23 b(Th)o(us,)17 b(this)75 829 y(program)c(can)i(succeed)g(only)g(if)g(the)f(comm)o(unication)h (system)f(has)g(su\016cien)o(t)h(bu\013er)g(space)f(to)g(bu\013er)75 885 y Fh(count)g Fs(w)o(ords)h(of)g(data.)166 942 y(If)g(additional)h (requiremen)o(ts)f(will)i(b)q(ecome)e(part)f(of)h(the)g(standard)f (\(e.g.,)f(b)q(ounds)j(on)f(the)f(mini-)75 998 y(mal)i(n)o(um)o(b)q(er) f(of)g(concurren)o(tly)h(activ)o(e)f(handles)i(that)e(need)h(b)q(e)g (supp)q(orted,)f(then)h(further)f(programs)75 1055 y(b)q(ecome)h(safe.) 75 1198 y Fm(2.14)70 b(Missing)75 1299 y Ft(send-receiv)o(e)45 b Fs(a)15 b(function)h(of)f(the)g(form)f(\(for)g(blo)q(c)o(king)j(comm) o(unication\))166 1356 y Fh(MPI)p 241 1356 15 2 v 17 w(SEND)p 354 1356 V 16 w(RECV\()23 b(send)p 609 1356 V 17 w(buffer,)g(destination,)f(recv)p 1223 1356 V 17 w(buffer,)h(source,)g(tag,)g(context\))166 1412 y Fs(It)14 b(ma)o(y)g(b)q(e)g(sup)q(er\015uous)i(b)q(ecause)f(the)f(collectiv)o(e) i(comm)o(unication)f(shift)g(has)f(similar)h(function-)75 1469 y(alit)o(y)l(.)36 b(Y)l(et)21 b(it)f(allo)o(ws)h(to)e(generate)i (more)f(easily)h(comm)o(unication)g(patterns)f(that)g(w)o(ould)g(lead)i (to)75 1525 y(deadlo)q(c)o(k)16 b(if)g(implemen)o(ted)h(naiv)o(ely)l(,) f(with)f(blo)q(c)o(king)i(comm)o(unication.)166 1582 y(An)g(imp)q(ortan)o(t)f(v)m(arian)o(t)g(is)h(the)f(case)h(where)g Fh(send)p 1069 1582 V 16 w(buffer)23 b(=)h(recv)p 1396 1582 V 17 w(buffer)15 b Fs(\(this)i(can)f(either)75 1638 y(b)q(e)j(a)e(sub)q(case)i(of)e(the)h(general)h(send)p 735 1638 14 2 v 17 w(recv)f(function,)h(or)e(a)h(separate)f(function,)i (if)f(send)h(bu\013er)f(and)75 1695 y(receiv)o(e)k(bu\013er)e(are)h (required)h(to)e(b)q(e)h(disjoin)o(t)h(in)f(the)g(general)h (function\).)37 b(This)21 b(in)o(tro)q(duces)h(new)75 1751 y(functionalit)o(y)16 b(that)f(is)h(not)e(a)o(v)m(ailable,)j (otherwise,)e(and)g(whic)o(h)h(seems)g(to)e(b)q(e)i(often)f(required.) 166 1807 y(An)g Fh(exchange)f Fs(is)i(the)f(sp)q(ecial)i(case)f(where)f (source)g(=)h(destination.)p eop %%Page: 37 42 37 41 bop 75 358 a Fn(Chapter)34 b(3)75 567 y Fu(Collecti)q(v)m(e)42 b(Comm)m(unication)892 791 y Fs(Al)16 b(Geist)876 847 y(Marc)e(Snir)75 998 y Fm(3.1)70 b(In)n(tro)r(duction)75 1102 y Fs(This)22 b(section)f(is)h(a)e(draft)h(of)f(the)h(curren)o(t)g (prop)q(osal)g(for)g(collectiv)o(e)i(comm)o(unication.)38 b(Collectiv)o(e)75 1158 y(comm)o(unication)12 b(is)f(de\014ned)i(to)d (b)q(e)i(comm)o(unication)g(that)e(in)o(v)o(olv)o(es)h(a)g(group)g(of)g (pro)q(cesses.)18 b(Examples)75 1215 y(are)c(broadcast)g(and)g(global)i (sum.)j(A)c(collectiv)o(e)h(op)q(eration)f(is)f(executed)i(b)o(y)e(ha)o (ving)h(all)g(pro)q(cesses)g(in)75 1271 y(the)j(group)f(call)h(the)g (comm)o(unication)g(routine,)h(with)e(matc)o(hing)h(parameters.)26 b(Routines)18 b(can)g(\(but)75 1328 y(are)f(not)g(required)h(to\))e (return)i(as)f(so)q(on)g(as)g(their)g(participation)i(in)f(the)f (collectiv)o(e)i(comm)o(unication)75 1384 y(is)h(complete.)34 b(The)19 b(completion)i(of)e(a)g(call)i(indicates)g(that)e(the)g (caller)i(is)f(no)o(w)f(free)g(to)g(access)h(the)75 1441 y(lo)q(cations)i(in)g(the)f(comm)o(unication)g(bu\013er,)h(or)f(an)o(y) g(other)f(lo)q(cation)i(that)e(can)h(b)q(e)h(referenced)g(b)o(y)75 1497 y(the)c(collectiv)o(e)j(op)q(eration.)29 b(Ho)o(w)o(ev)o(er,)18 b(it)h(do)q(es)f(not)g(indicate)i(that)e(other)g(pro)q(cesses)h(in)g (the)f(group)75 1554 y(ha)o(v)o(e)c(started)g(the)h(op)q(eration)g (\(unless)h(otherwise)f(indicated)h(in)g(the)f(description)h(of)f(the)f (op)q(eration\).)75 1610 y(Ho)o(w)o(ev)o(er,)i(the)g(successful)i (completion)g(of)e(a)h(collectiv)o(e)h(comm)o(unication)f(call)h(ma)o (y)e(dep)q(end)i(on)f(the)75 1667 y(execution)f(of)f(a)g(matc)o(hing)g (call)h(at)f(all)h(pro)q(cesses)g(in)g(the)f(group.)166 1724 y(The)g(syn)o(tax)g(and)g(seman)o(tics)g(of)g(the)g(collectiv)o(e) i(op)q(erations)f(is)f(de\014ned)i(so)e(as)f(to)h(b)q(e)h(consisten)o (t)75 1781 y(with)g(the)f(syn)o(tax)f(and)i(seman)o(tics)f(of)g(the)g (p)q(oin)o(t)h(to)e(p)q(oin)o(t)i(op)q(erations.)166 1839 y(The)f(reader)f(is)h(referred)f(to)g(the)g(p)q(oin)o(t-to-p)q (oin)o(t)h(comm)o(unication)g(section)g(of)f(the)h(curren)o(t)f(MPI)75 1895 y(draft)g(for)f(information)i(concerning)g(comm)o(unication)g (bu\013ers)g(and)f(their)h(manipulations.)21 b(The)15 b(con-)75 1952 y(text)d(section)h(describ)q(es)g(the)g(formation,)e (manipulation,)j(and)f(query)f(functions)h(\(suc)o(h)g(as)e(group)h (size\))75 2008 y(that)i(are)h(a)o(v)m(ailable)i(for)e(groups)g(and)g (group)g(ob)s(jects.)166 2066 y(The)23 b(collectiv)o(e)i(comm)o (unication)f(routines)f(are)g(built)h(ab)q(o)o(v)o(e)f(the)g(p)q(oin)o (t-to-p)q(oin)o(t)h(routines.)75 2122 y(While)14 b(v)o(endors)d(ma)o(y) h(optimize)h(certain)g(collectiv)o(e)h(routines)e(for)g(their)g(arc)o (hitectures,)h(a)f(complete)h(li-)75 2179 y(brary)h(of)g(the)h (collectiv)o(e)h(comm)o(unication)g(routines)f(can)f(b)q(e)i(written)e (en)o(tirely)i(using)f(p)q(oin)o(t-to-p)q(oin)o(t)75 2235 y(comm)o(unication)21 b(functions.)37 b(W)l(e)20 b(are)h(using)g(naiv)o(e)g(implemen)o(tations)h(of)e(the)g(collectiv)o (e)j(calls)e(in)75 2292 y(terms)f(of)f(p)q(oin)o(t)i(to)e(p)q(oin)o(t)i (op)q(erations)f(in)h(order)f(to)f(pro)o(vide)i(an)f(op)q(erational)h (de\014nition)h(of)d(their)75 2348 y(seman)o(tics.)166 2406 y(The)c(follo)o(wing)h(comm)o(unication)g(functions)g(are)f(prop)q (osed.)143 2505 y Fo(\017)23 b Fs(Broadcast)14 b(from)g(one)i(mem)o(b)q (er)f(to)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 2604 y Fo(\017)23 b Fs(Barrier)15 b(across)f(all)j(group)d(mem)o(b)q (ers)143 2704 y Fo(\017)23 b Fs(Gather)14 b(data)h(from)f(all)i(group)f (mem)o(b)q(ers)g(to)g(one)g(mem)o(b)q(er.)952 2828 y(37)p eop %%Page: 38 43 38 42 bop 75 -100 a Fs(38)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)143 45 y Fo(\017)23 b Fs(Scatter)14 b(data)h(from)f(one)i(mem)o(b)q(er)f(to)g(all)h(mem)o (b)q(ers)f(of)g(a)g(group.)143 137 y Fo(\017)23 b Fs(Global)15 b(op)q(erations)g(suc)o(h)h(as)e(sum,)h(max,)f(min,)i(etc.,)e(w)o(ere)h (the)g(result)g(is)h(kno)o(wn)e(b)o(y)h(all)h(group)189 193 y(mem)o(b)q(ers)e(and)h(a)g(v)m(ariation)g(where)g(the)g(result)g (is)g(kno)o(wn)f(b)o(y)h(only)g(one)g(mem)o(b)q(er.)20 b(The)15 b(abilit)o(y)189 250 y(to)f(ha)o(v)o(e)h(user)g(de\014ned)i (global)f(op)q(erations.)143 342 y Fo(\017)23 b Fs(Sim)o(ultaneous)14 b(shift)g(of)f(data)g(around)g(the)g(group,)h(the)f(simplest)i(example) f(b)q(eing)h(all)f(mem)o(b)q(ers)189 398 y(sending)i(their)g(data)e(to) h(\(rank+1\))f(with)i(wrap)e(around.)143 490 y Fo(\017)23 b Fs(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g (called)h(parallel)h(pre\014x\).)143 582 y Fo(\017)23 b Fs(Broadcast)14 b(from)g(all)j(mem)o(b)q(ers)e(to)f(all)j(mem)o(b)q (ers)e(of)g(a)g(group.)143 674 y Fo(\017)23 b Fs(Scatter)18 b(data)f(from)h(all)h(mem)o(b)q(ers)g(to)f(all)h(mem)o(b)q(ers)f(of)g (a)g(group)h(\(also)f(called)i(complete)f(ex-)189 730 y(c)o(hange)c(or)g(index\).)166 819 y(T)l(o)g(simplify)i(the)f (collectiv)o(e)h(comm)o(unication)g(in)o(terface)e(it)h(is)g(designed)h (with)f(t)o(w)o(o)e(la)o(y)o(ers.)21 b(The)75 876 y(lo)o(w)e(lev)o(el)i (routines)e(ha)o(v)o(e)g(all)i(the)e(generalit)o(y)h(of,)f(and)h(mak)o (e)e(use)i(of,)g(the)f(comm)o(unication)h(bu\013er)75 932 y(routines)15 b(of)g(the)f(p)q(oin)o(t-to-p)q(oin)o(t)i(section)f (whic)o(h)h(allo)o(ws)f(arbitrarily)g(complex)h(messages)e(to)g(b)q(e)i (con-)75 989 y(structed.)k(The)14 b(second)h(lev)o(el)g(routines)g(are) f(similar)i(to)d(the)i(upp)q(er)g(lev)o(el)g(p)q(oin)o(t-to-p)q(oin)o (t)g(routines)g(in)75 1045 y(that)f(they)i(send)g(only)f(a)g(con)o (tiguous)g(bu\013er.)75 1187 y Fm(3.2)70 b(Group)24 b(F)-6 b(unctions)75 1289 y Fs(A)17 b(full)h(description)g(of)e(the)h(group)f (formation)g(and)h(manipulation)h(functions)f(can)g(b)q(e)g(found)g(in) h(the)75 1345 y(con)o(text)f(c)o(hapter)h(of)f(the)g(MPI)h(do)q(cumen)o (t.)28 b(Here)18 b(w)o(e)f(describ)q(e)i(only)g(those)e(group)g (functions)i(that)75 1402 y(are)c(used)h(in)g(the)f(seman)o(tic)g (description)i(of)e(the)g(collectiv)o(e)i(comm)o(unication)f(routines.) 166 1458 y(An)g(initial)j(group)d(con)o(taining)h(all)g(pro)q(cesses)g (is)g(supplied)h(b)o(y)e(default)h(in)g(MPI.)f(MPI)g(pro)o(vides)75 1515 y(a)f(pro)q(cedure)g(that)f(returns)h(the)g(handle)h(to)e(this)i (initial)h(group.)i(The)c(pro)q(cesses)g(in)h(the)f(inital)h(group)75 1571 y(eac)o(h)22 b(ha)o(v)o(e)f(a)h(unique)h(rank)e(in)i(the)f(group)f (represen)o(ted)h(b)o(y)g(in)o(tegers)g(\(0,)g(1,)h(2,)f(...,)g([n)o (um)o(b)q(er-of-)75 1628 y(pro)q(cesses)16 b(-)f(1].)166 1720 y Ft(MPI)p 275 1720 16 2 v 18 w(ALLGR)o(OUP\(group\))i Fs(Return)g(the)f(descriptor)h(of)e(the)i(inital)g(group)f(con)o (taining)h(all)75 1811 y(pro)q(cesses.)75 1900 y Ft(OUT)h(group)23 b Fs(handle)16 b(to)f(descriptor)h(ob)s(ject)e(of)h(initial)i(group.) 166 2025 y Ft(MPI)p 275 2025 V 18 w(RANK\(group,)g(rank\))d Fs(Return)i(the)f(rank)g(of)g(the)g(calling)i(pro)q(cess)e(within)h (the)f(sp)q(eci-)75 2117 y(\014ed)h(group.)75 2206 y Ft(IN)h(group)23 b Fs(group)15 b(handle)75 2298 y Ft(OUT)j(rank)23 b Fs(in)o(teger)166 2422 y Ft(MPI)p 275 2422 V 18 w(GSIZE\(group,)18 b(size\))e Fs(Return)g(the)f(n)o(um)o(b)q(er)h(of)f(pro)q(cesses)h (that)e(b)q(elong)j(to)e(the)g(sp)q(ec-)75 2514 y(i\014ed)h(group.)75 2603 y Ft(IN)h(group)23 b Fs(group)15 b(handle)75 2695 y Ft(OUT)j(size)23 b Fs(in)o(teger)p eop %%Page: 39 44 39 43 bop 75 -100 a Fq(3.3.)34 b(COMMUNICA)l(TION)16 b(FUNCTIONS)942 b Fs(39)75 45 y Fm(3.3)70 b(Comm)n(unication)20 b(F)-6 b(unctions)75 147 y Fs(The)11 b(prop)q(osed)g(comm)o(unication)h (functions)g(are)e(divided)j(in)o(to)e(t)o(w)o(o)f(la)o(y)o(ers.)18 b(The)11 b(lo)o(w)o(est)f(lev)o(el)j(uses)e(the)75 203 y(same)j(bu\013er)g(descriptor)h(ob)s(jects)e(a)o(v)m(ailable)j(in)f(p) q(oin)o(t-to-p)q(oin)o(t)g(to)f(create)g(noncon)o(tiguous,)g(m)o (ultiple)75 259 y(data)k(t)o(yp)q(e)h(messages.)31 b(The)19 b(second)h(lev)o(el)g(is)g(similar)g(to)e(the)h(blo)q(c)o(k)h (send/receiv)o(e)g(p)q(oin)o(t-to-p)q(oin)o(t)75 316 y(op)q(erations)g(in)g(that)f(it)h(supp)q(orts)g(only)g(con)o(tiguous)g (bu\013ers)g(of)f(data.)32 b(F)l(or)19 b(eac)o(h)h(comm)o(unication)75 372 y(op)q(eration,)15 b(w)o(e)g(list)h(these)f(t)o(w)o(o)f(lev)o(el)j (of)e(calls)h(together.)75 514 y Fm(3.4)70 b(Sync)n(hronization)75 616 y Ft(Barrier)17 b(sync)o(hronization)75 737 y(MPI)p 184 737 16 2 v 18 w(BARRIER\()h(group)g(\))166 829 y Fs(MPI)p 258 829 14 2 v 16 w(BARRIER)g(blo)q(c)o(ks)f(the)f(calling)i (pro)q(cess)e(un)o(til)h(all)g(group)e(mem)o(b)q(ers)i(ha)o(v)o(e)e (called)j(it;)e(the)75 885 y(call)g(returns)f(at)g(an)o(y)g(pro)q(cess) g(only)h(after)e(all)j(group)e(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered) g(the)g(call.)75 972 y Ft(IN)i(group)23 b Fs(group)15 b(handle)166 1058 y Fh(MPI)p 241 1058 15 2 v 17 w(BARRIER\()22 b(group)i(\))15 b Fs(is)75 1145 y Fh(MPI_CREATE_BUFFER\(buffer_h)o (andle,)20 b(MPI_BUFFER,)j(MPI_PERSISTENT\);)75 1201 y(MPI_GSIZE\()f(group,)i(&size)f(\);)75 1257 y(MPI_RANK\()g(group,)g (&rank)g(\);)75 1314 y(if)h(\(rank==0\))75 1370 y({)147 1427 y(for)f(\(i=1;)g(i)h(<)g(size;)f(i++\))218 1483 y(MPI_RECV\(buffer_handle,)e(i,)i(tag,)h(group,)f(return_handle\);)147 1540 y(for)g(\(i=1;)g(i)h(<)g(size;)f(i++\))218 1596 y(MPI_SEND\(buffer_handle,)e(i,)i(tag,)h(group\);)75 1653 y(})75 1709 y(else)75 1766 y({)147 1822 y (MPI_SEND\(buffer_handle,)c(0,)k(tag,)f(group\);)147 1878 y(MPI_RECV\(buffer_handle,)d(0,)k(tag,)f(group,)g (return_handle\);)75 1935 y(})75 1991 y(MPI_FREE\(buffer_handle\);)75 2133 y Fm(3.5)70 b(Data)23 b(mo)n(v)n(e)f(functions)75 2235 y Ft(Broadcast)75 2356 y(MPI)p 184 2356 16 2 v 18 w(BCAST\()c(bu\013er)p 555 2356 V 17 w(handle,)g(group,)f(ro)q(ot)h(\)) 166 2448 y Fh(MPI)p 241 2448 15 2 v 17 w(BCAST)12 b Fs(broadcasts)f(a)h (message)h(from)e(the)i(pro)q(cess)g(with)g(rank)f Fh(root)g Fs(to)g(all)h(other)g(pro)q(cesses)75 2504 y(of)18 b(the)g(group.)29 b(It)18 b(is)h(called)h(b)o(y)e(all)h(mem)o(b)q(ers)g(of)e(group)h (using)h(the)g(same)f(argumen)o(ts)f(for)h Fh(group,)75 2561 y(and)23 b(root)p Fs(.)d(On)15 b(return)h(the)f(con)o(ten)o(ts)f (of)h(the)g(bu\013er)g(of)g(the)g(pro)q(cess)h(with)f(rank)g Fh(root)g Fs(is)g(con)o(tained)75 2617 y(in)h(the)f(bu\013er)g(of)g (the)h(calling)g(pro)q(cess.)75 2704 y Ft(INOUT)i(bu\013er)p 407 2704 16 2 v 18 w(handle)24 b Fs(Handle)16 b(for)e(bu\013er)i(where) f(from)g(message)f(is)i(sen)o(t)f(or)g(receiv)o(ed.)p eop %%Page: 40 45 40 44 bop 75 -100 a Fs(40)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(IN)h(group)23 b Fs(group)15 b(handle)75 137 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(broadcast)g(ro)q(ot)f(\(in)o(teger\))166 262 y Ft(MPI)p 275 262 16 2 v 18 w(BCASTC\()j(buf,)g(len,)h(t)o(yp)q (e,)f(group,)f(ro)q(ot)i(\))166 354 y Fh(MPI)p 241 354 15 2 v 17 w(BCASTC)i Fs(b)q(eha)o(v)o(es)h(lik)o(e)i(broadcast,)e (restricted)h(to)e(a)h(blo)q(c)o(k)h(bu\013er.)37 b(It)22 b(is)f(called)i(b)o(y)e(all)75 410 y(pro)q(cesses)16 b(with)f(the)g(same)g(argumen)o(ts)g(for)f Fh(len,)24 b(group)14 b Fs(and)h Fh(root)p Fs(.)75 500 y Ft(INOUT)j(bu\013er)k Fs(Starting)15 b(address)g(of)g(bu\013er)g(\(c)o(hoice)h(t)o(yp)q(e\)) 75 592 y Ft(IN)h(len)23 b Fs(Num)o(b)q(er)16 b(of)f(en)o(tries)g(in)h (bu\013er)g(\(in)o(teger\))75 684 y Ft(IN)h(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 776 y Ft(IN)j(group)23 b Fs(group)15 b(handle)75 868 y Ft(in)j(ro)q(ot)23 b Fs(rank)15 b(of)g(broadcast)f(ro)q(ot)h(\(in)o(teger\))166 957 y Fh(MPI)p 241 957 V 17 w(BCAST\()23 b(buffer)p 569 957 V 16 w(handle,)g(group,)g(root)g(\))15 b Fs(is)75 1047 y Fh(MPI_GSIZE\()22 b(group,)i(&size)f(\);)75 1103 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 1160 y(MPI_IRECV\(handle,)f (buffer_handle,)f(root,)j(tag,)f(group,)g(return_handle\);)75 1216 y(if)h(\(rank==root\))147 1272 y(for)f(\(i=0;)g(i)h(<)g(size;)f (i++\))218 1329 y(MPI_SEND\(buffer_handle,)e(i,)i(tag,)h(group\);)75 1385 y(MPI_WAIT\(handle\))75 1505 y Ft(Circular)18 b(shift)75 1626 y(MPI)p 184 1626 16 2 v 18 w(CSHIFT\()g(in)o(buf,)f(outbuf,)g (group,)g(shift\))166 1718 y Fs(Pro)q(cess)22 b(with)h(rank)g Fh(i)f Fs(sends)h(the)g(data)f(in)i(its)e(input)i(bu\013er)f(to)f(pro)q (cess)h(with)g(rank)f(\()p Fh(i)15 b Fs(+)75 1774 y Fh(shift)p Fs(\))d(mo)q(d)g Fh(group)p 447 1774 15 2 v 17 w(size)p Fs(,)h(who)g(receiv)o(es)h(the)g(data)e(in)i(its)g(output)f(bu\013er.) 20 b(All)14 b(pro)q(cesses)g(mak)o(e)f(the)75 1831 y(call)k(with)f(the) f(same)h(v)m(alues)g(for)f Fh(group)p Fs(,)g(and)h Fh(shift)p Fs(.)k(The)c Fh(shift)f Fs(v)m(alue)i(can)f(b)q(e)g(p)q(ositiv)o(e,)g (zero,)f(or)75 1887 y(negativ)o(e.)75 1987 y Ft(IN)i(in)o(buf)23 b Fs(handle)17 b(to)d(input)i(bu\013er)f(descriptor)75 2079 y Ft(IN)i(outbuf)23 b Fs(handle)17 b(to)d(output)h(bu\013er)h (descriptor)75 2171 y Ft(IN)h(group)23 b Fs(handle)16 b(to)f(group)75 2263 y Ft(IN)i(shift)23 b Fs(in)o(teger)166 2399 y Ft(MPI)p 275 2399 16 2 v 18 w(CSHIFT1\()18 b(buf,)f(group,)f (shift\))166 2491 y Fs(Pro)q(cess)f(with)g(rank)f Fh(i)h Fs(sends)g(the)g(data)f(in)h(its)g(bu\013er)g(to)f(pro)q(cess)h(with)g (rank)f(\()p Fh(i)9 b Fs(+)g Fh(shift)p Fs(\))j(mo)q(d)75 2547 y Fh(group)p 198 2547 15 2 v 16 w(size)p Fs(,)i(who)g(receiv)o(es) h(the)f(data)f(in)i(the)f(same)g(bu\013er.)20 b(All)15 b(pro)q(cesses)f(mak)o(e)g(the)g(call)h(with)g(the)75 2604 y(same)g(v)m(alues)h(for)f Fh(group)p Fs(,)f(and)h Fh(shift)p Fs(.)k(The)d Fh(shift)e Fs(v)m(alue)j(can)e(b)q(e)h(p)q (ositiv)o(e,)g(zero,)e(or)h(negativ)o(e.)75 2704 y Ft(INOUT)j(buf)23 b Fs(handle)16 b(to)f(bu\013er)g(descriptor)p eop %%Page: 41 46 41 45 bop 75 -100 a Fq(3.5.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b Fs(41)75 45 y Ft(IN)17 b(group)23 b Fs(handle)16 b(to)f(group)75 141 y Ft(IN)i(shift)23 b Fs(in)o(teger)166 286 y Ft(MPI)p 275 286 16 2 v 18 w(CSHIFTC\()17 b(in)o(buf,)g(outbuf,)h(len,)f(t)o(yp)q(e,)g(group,)g(shift\))166 379 y Fs(Beha)o(v)o(es)j(lik)o(e)h Fh(MPI)p 509 379 15 2 v 17 w(CSHIFT)p Fs(,)d(with)i(bu\013ers)g(restricted)g(to)g(b)q(e)g (blo)q(c)o(ks)h(of)e(n)o(umeric)i(units.)35 b(All)75 435 y(pro)q(cesses)16 b(mak)o(e)e(the)i(call)g(with)f(the)h(same)f(v)m (alues)h(for)f Fh(len,)23 b(group)p Fs(,)14 b(and)h Fh(shift)p Fs(.)75 532 y Ft(IN)i(in)o(buf)23 b Fs(initial)17 b(lo)q(cation)f(of)f (input)h(bu\013er)75 628 y Ft(OUT)i(outbuf)24 b Fs(initial)17 b(lo)q(cation)f(of)f(output)g(bu\013er)75 724 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(\(and)e (output\))f(bu\013ers)75 821 y Ft(IN)j(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 917 y Ft(IN)j(group)23 b Fs(handle)16 b(to)f(group)75 1014 y Ft(IN)i(shift)23 b Fs(in)o(teger)166 1145 y Ft(MPI)p 275 1145 16 2 v 18 w(CSHIFTC1\()17 b(buf,)g(len,)h(t)o(yp)q(e,)f(group,)f(shift\))166 1238 y Fs(Beha)o(v)o(es)i(lik)o(e)h Fh(MPI)p 505 1238 15 2 v 17 w(CSHIFT1)p Fs(,)e(with)i(bu\013ers)f(restricted)g(to)g(b)q (e)h(blo)q(c)o(ks)g(of)e(n)o(umeric)i(units.)30 b(All)75 1294 y(pro)q(cesses)16 b(mak)o(e)e(the)i(call)g(with)f(the)h(same)f(v)m (alues)h(for)f Fh(len,)23 b(group)p Fs(,)14 b(and)h Fh(shift)p Fs(.)75 1391 y Ft(INOUT)j(buf)23 b Fs(initial)17 b(lo)q(cation)f(of)f (bu\013er)75 1487 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(\(and)e(output\))f(bu\013ers)75 1584 y Ft(IN)j(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er) 75 1680 y Ft(IN)j(group)23 b Fs(handle)16 b(to)f(group)75 1776 y Ft(IN)i(shift)23 b Fs(in)o(teger)166 1873 y Fh(MPI)p 241 1873 V 17 w(CSHIFT\()g(inbuf,)g(outbuf,)g(group,)g(shift\))14 b Fs(is)75 1970 y Fh(MPI_GSIZE\()22 b(group,)i(&size)f(\);)75 2026 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 2083 y(MPI_ISEND\()f (handle,)h(inbuf,)g(mod\(rank+shift,)f(size\),)h(tag,)h(group\);)75 2139 y(MPI_RECV\()f(outbuf,)g(mod\(rank-shift,size\),)e(tag,)i(group,)g (return_handle\))75 2195 y(MPI_WAIT\(handle\);)75 2319 y Ft(End-o\013)18 b(shift)75 2442 y(MPI)p 184 2442 16 2 v 18 w(EOSHIFT\()g(in)o(buf,)f(outbuf,)h(group,)f(shift\))166 2534 y Fs(Pro)q(cess)f(with)g(rank)g Fh(i)p Fs(,)f(max\()p Fh(0)p Fr(;)8 b Fo(\000)p Fh(shift)p Fs(\))k Fo(\024)i Fh(i)g Fr(<)g Fh(min)p Fs(\()p Fh(size)p Fr(;)8 b Fh(size)g Fo(\000)j Fh(shift)p Fs(\),)k(sends)i(the)f(data)75 2591 y(in)d(its)f(input)i(bu\013er)e(to)g(pro)q(cess)g(with)h(rank)f Fh(i+)23 b(shift)p Fs(,)12 b(who)g(receiv)o(es)h(the)f(data)g(in)h(its) f(output)g(bu\013er.)75 2647 y(The)17 b(output)g(bu\013er)g(of)f(pro)q (cesses)i(whic)o(h)f(do)g(not)g(receiv)o(e)h(data)e(is)h(left)g(unc)o (hanged.)26 b(All)19 b(pro)q(cesses)75 2704 y(mak)o(e)c(the)g(call)h (with)g(the)f(same)g(v)m(alues)h(for)f Fh(group)p Fs(,)f(and)h Fh(shift)p Fs(.)p eop %%Page: 42 47 42 46 bop 75 -100 a Fs(42)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(IN)h(in)o(buf)23 b Fs(handle)17 b(to)d(input)i(bu\013er)f(descriptor)75 142 y Ft(IN)i(outbuf)23 b Fs(handle)17 b(to)d(output)h(bu\013er)h (descriptor)75 239 y Ft(IN)h(group)23 b Fs(handle)16 b(to)f(group)75 337 y Ft(IN)i(shift)23 b Fs(in)o(teger)166 482 y Ft(MPI)p 275 482 16 2 v 18 w(EOSHIFT1\()18 b(buf,)f(group,)g (shift\))166 575 y Fs(Pro)q(cess)f(with)g(rank)g Fh(i)p Fs(,)f(max\()p Fh(0)p Fr(;)8 b Fo(\000)p Fh(shift)p Fs(\))k Fo(\024)i Fh(i)g Fr(<)g Fh(min)p Fs(\()p Fh(size)p Fr(;)8 b Fh(size)g Fo(\000)j Fh(shift)p Fs(\),)k(sends)i(the)f(data)75 632 y(in)g(its)f(bu\013er)g(to)f(pro)q(cess)h(with)h(rank)e Fh(i+)24 b(shift)p Fs(,)14 b(who)g(receiv)o(es)i(the)f(data)g(in)g(the) g(same)g(bu\013er.)20 b(The)75 688 y(output)15 b(bu\013er)g(of)g(pro)q (cesses)h(whic)o(h)g(do)g(not)f(receiv)o(e)h(data)e(is)i(left)g(unc)o (hanged.)21 b(All)c(pro)q(cesses)e(mak)o(e)75 745 y(the)g(call)i(with)e (the)g(same)g(v)m(alues)i(for)d Fh(group)p Fs(,)g(and)i Fh(shift)p Fs(.)75 855 y Ft(INOUT)i(buf)23 b Fs(handle)16 b(to)f(bu\013er)g(descriptor)75 952 y Ft(IN)i(group)23 b Fs(handle)16 b(to)f(group)75 1049 y Ft(IN)i(shift)23 b Fs(in)o(teger)166 1195 y Ft(MPI)p 275 1195 V 18 w(EOSHIFTC\()18 b(in)o(buf,)f(outbuf,)h(len,)f(t)o(yp)q(e,)g(group,)g(shift\))166 1288 y Fs(Beha)o(v)o(es)h(lik)o(e)h Fh(MPI)p 505 1288 15 2 v 17 w(EOSHIFT)p Fs(,)e(with)h(bu\013ers)h(restricted)f(to)g(b)q (e)g(blo)q(c)o(ks)h(of)f(n)o(umeric)h(units.)30 b(All)75 1344 y(pro)q(cesses)16 b(mak)o(e)e(the)i(call)g(with)f(the)h(same)f(v)m (alues)h(for)f Fh(len,)23 b(group)p Fs(,)14 b(and)h Fh(shift)p Fs(.)75 1441 y Ft(IN)i(in)o(buf)23 b Fs(initial)17 b(lo)q(cation)f(of)f (input)h(bu\013er)75 1539 y Ft(OUT)i(outbuf)24 b Fs(initial)17 b(lo)q(cation)f(of)f(output)g(bu\013er)75 1636 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(\(and)e (output\))f(bu\013ers)75 1733 y Ft(IN)j(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 1830 y Ft(IN)j(group)23 b Fs(handle)16 b(to)f(group)75 1927 y Ft(IN)i(shift)23 b Fs(in)o(teger)166 2060 y Ft(MPI)p 275 2060 16 2 v 18 w(EOSHIFTC1\()18 b(buf,)f(len,)h(t)o(yp)q(e,)e(group,)h(shift\))166 2153 y Fs(Beha)o(v)o(es)h(lik)o(e)g Fh(MPI)p 504 2153 15 2 v 17 w(EOSHIFT1)p Fs(,)f(with)h(bu\013er)g(restricted)g(to)f(b)q (e)h(blo)q(c)o(ks)h(of)e(n)o(umeric)i(units.)28 b(All)75 2209 y(pro)q(cesses)16 b(mak)o(e)e(the)i(call)g(with)f(the)h(same)f(v)m (alues)h(for)f Fh(len,)23 b(group)p Fs(,)14 b(and)h Fh(shift)p Fs(.)75 2306 y Ft(INOUT)j(buf)23 b Fs(initial)17 b(lo)q(cation)f(of)f (bu\013er)75 2403 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(bu\013er)75 2500 y Ft(IN)h(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 2598 y Ft(IN)j(group)23 b Fs(handle)16 b(to)f(group)75 2695 y Ft(IN)i(shift)23 b Fs(in)o(teger)p eop %%Page: 43 48 43 47 bop 75 -100 a Fq(3.5.)29 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1082 b Fs(43)75 45 y Ft(Gather)75 166 y(MPI)p 184 166 16 2 v 18 w(GA)l(THER\()18 b(in)o(buf,)f(list)p 684 166 V 20 w(of)p 746 166 V 19 w(outbufs,)g(group,)g(ro)q(ot,)g (return)p 1404 166 V 18 w(status\))166 258 y Fs(Eac)o(h)g(pro)q(cess)g (\(including)i(the)e(ro)q(ot)f(pro)q(cess\))h(sends)h(the)f(con)o(ten)o (t)f(of)h(its)g(input)h(bu\013er)f(to)f(the)75 315 y(ro)q(ot)d(pro)q (cess.)19 b(The)14 b(ro)q(ot)f(pro)q(cess)h(places)g(all)h(the)f (incoming)h(messages)e(in)h(the)g(lo)q(cation)g(sp)q(eci\014ed)i(b)o(y) 75 371 y(the)g(output)f(bu\013er)h(handle)h(corresp)q(onding)g(to)e (the)h(sender's)g(rank.)21 b(F)l(or)15 b(example,)h(the)g(ro)q(ot)f (places)75 428 y(the)f(data)g(from)f(pro)q(cess)h(with)h(rank)f(3)f(in) i(the)g(lo)q(cation)f(sp)q(eci\014ed)j(b)o(y)d(the)g(third)h(bu\013er)f (descriptor)g(in)75 484 y(the)i(list)g(of)f(outbufs.)20 b(The)c(list)p 620 484 14 2 v 17 w(of)p 674 484 V 16 w(outbufs)g(argumen)o(t)e(is)i(ignored)g(for)f(all)i(non-ro)q(ot)e(pro) q(cesses.)21 b(The)75 541 y(routine)c(is)h(called)g(b)o(y)f(all)g(mem)o (b)q(ers)g(of)g(group)f(using)i(the)e(same)h(argumen)o(ts)f(for)g Fh(group)p Fs(,)g(and)h Fh(root)p Fs(.)75 597 y(The)e(input)i(bu\013er) e(of)g(eac)o(h)g(pro)q(cess)g(ma)o(y)g(ha)o(v)o(e)g(a)f(di\013eren)o(t) i(length.)75 690 y Ft(IN)h(in)o(buf)23 b Fs(handle)17 b(to)d(input)i(bu\013er)f(descriptor)75 783 y Ft(IN)i(list)p 225 783 16 2 v 20 w(of)p 287 783 V 19 w(outbufs)23 b Fs(list)16 b(of)f(bu\013er)g(descriptor)g(handles)i(\(ro)q(ot\))75 877 y Ft(IN)g(group)23 b Fs(group)15 b(handle)75 970 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(receiving)i(pro)q(cess)e(\(in) o(teger\))75 1064 y Ft(OUT)j(return)p 357 1064 V 19 w(status)23 b Fs(return)15 b(status)f(handle)166 1239 y Fl(Discussion:)47 b Fk(Do)17 b(w)o(e)h(w)o(an)o(t)f(the)g(collectiv)o(e)h(routines)g(to)f (ha)o(v)o(e)g(return)h(status)g(handles?)29 b(And)17 b(if)g(so)75 1296 y(what)d(information)d(do)i(w)o(e)i(w)o(an)o(t)e(the) h(handle)g(to)g(b)q(e)g(able)g(to)g(return?)166 1470 y Ft(MPI)p 275 1470 V 18 w(GA)l(THER)o(C\()j(in)o(buf,)g(outbuf,)h (inlen,)g(t)o(yp)q(e,)f(group,)f(ro)q(ot\))166 1562 y Fh(MPI)p 241 1562 15 2 v 17 w(GATHERC)g Fs(b)q(eha)o(v)o(es)i(lik)o(e)h Fh(MPI)p 772 1562 V 17 w(GATHER)e Fs(restricted)h(to)f(blo)q(c)o(k)h (bu\013ers,)g(and)g(with)g(the)g(addi-)75 1619 y(tional)f(restriction)h (that)e(all)i(input)g(bu\013ers)e(should)i(ha)o(v)o(e)f(the)g(same)f (length.)26 b(All)18 b(pro)q(cesses)f(should)75 1675 y(pro)o(vided)f(the)f(same)g(v)m(alues)i(for)d Fh(inlen,)23 b(group)p Fs(,)14 b(and)i Fh(root)e Fs(.)75 1768 y Ft(IN)j(in)o(buf)23 b Fs(\014rst)15 b(v)m(ariable)h(of)f(input)h(bu\013er)g(\(c)o(hoice\)) 75 1862 y Ft(OUT)i(outbuf)24 b Fs(\014rst)14 b(v)m(ariable)j(of)e (output)g(bu\013er)g({)g(signi\014can)o(t)h(only)g(at)e(ro)q(ot)h (\(matc)o(hes)f(t)o(yp)q(e\))75 1955 y Ft(IN)j(inlen)24 b Fs(Num)o(b)q(er)16 b(of)e(\(w)o(ord\))g(v)m(ariables)j(in)f(input)g (bu\013er)f(\(in)o(teger\))75 2049 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 2142 y Ft(IN)j(group)23 b Fs(group)15 b(handle)75 2235 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(receiving)i(pro)q(cess)e(\(in)o(teger\))166 2328 y Fh(MPI)p 241 2328 V 17 w(GATHERC\()22 b(inbuf,)h(outbuf,)g (inlen,)g(type,)h(group,)f(root\))38 b Fs(is)75 2421 y Fh(MPI_GSIZE\()22 b(&size,)i(group\);)75 2478 y(MPI_RANK\()f(&rank,)g (group\);)75 2534 y(MPI_ISENDC\(handle,)e(inbuf,)i(inlen,)g(root,)h (tag,)f(group\);)75 2591 y(if)h(\(rank==root\))147 2647 y(for)f(\(i=0;)g(i)h(<)g(size;)f(i++\))147 2704 y({)p eop %%Page: 44 49 44 48 bop 75 -100 a Fs(44)708 b Fq(CHAPTER)15 b(3.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)218 45 y Fh(MPI_RECVC\(outbuf,)22 b(inlen,)h(i,)g(tag,)h(group,)f(return_status\);)218 102 y(outbuf)g(+=)h(inlen;)147 158 y(})75 214 y(MPI_WAIT\(handle\);)75 336 y Ft(Scatter)75 458 y(MPI)p 184 458 16 2 v 18 w(SCA)l(TTER\()18 b(list)p 557 458 V 20 w(of)p 619 458 V 19 w(in)o(bufs,)e(outbuf,)i (group,)f(ro)q(ot,)g(return)p 1425 458 V 19 w(status\))166 550 y Fs(The)e(ro)q(ot)f(pro)q(cess)i(sends)f(the)g(con)o(ten)o(t)g(of) f(its)i Fh(i)p Fs(-th)f(input)h(bu\013er)f(to)f(the)h(pro)q(cess)g (with)h(rank)f Fh(i)p Fs(;)75 607 y(eac)o(h)g(pro)q(cess)h(\(including) i(the)e(ro)q(ot)e(pro)q(cess\))h(stores)g(the)h(incoming)g(message)f (in)i(its)e(output)g(bu\013er.)75 663 y(The)f(routine)g(is)g(called)h (b)o(y)f(all)h(mem)o(b)q(ers)e(of)h(the)f(group)h(using)g(the)g(same)f (argumen)o(ts)g(for)g Fh(group)p Fs(,)g(and)75 720 y Fh(root)p Fs(.)75 815 y Ft(IN)k(list)p 225 815 V 20 w(of)p 287 815 V 19 w(in)o(bufs)22 b Fs(list)16 b(of)f(bu\013er)g(descriptor)h (handles)75 910 y Ft(IN)h(outbuf)23 b Fs(bu\013er)16 b(descriptor)f(handle)75 1005 y Ft(IN)i(group)23 b Fs(handle)75 1100 y Ft(IN)17 b(ro)q(ot)23 b Fs(rank)15 b(of)g(sending)i(pro)q(cess)e (\(in)o(teger\))75 1195 y Ft(OUT)j(return)p 357 1195 V 19 w(status)23 b Fs(return)15 b(status)f(handle)166 1290 y Fh(MPI)p 241 1290 15 2 v 17 w(SCATTER\()22 b(list)p 568 1290 V 17 w(of)p 633 1290 V 17 w(inbufs,)h(outbuf,)g(group,)g (root,)g(return)p 1486 1290 V 16 w(status\))15 b Fs(is)75 1385 y Fh(MPI_GSIZE\()22 b(group,)i(&size)f(\);)75 1441 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 1498 y(MPI_IRECV\(handle,)f (outbuf,)g(root,)i(tag,)f(group\);)75 1554 y(if)h(\(rank=root\))147 1611 y(for)f(\(i=0;)g(i)h(<)g(size;)f(i++\))218 1667 y(MPI_SEND\(inbuf[i],)f(i,)h(tag,)g(group\);)75 1724 y(MPI_WAIT\(handle,)f(return_status\);)166 1854 y Ft(MPI)p 275 1854 16 2 v 18 w(SCA)l(TTER)o(C\()17 b(in)o(buf,)g(outbuf,)h(len,)f (t)o(yp)q(e,)g(group,)g(ro)q(ot\))166 1946 y Fh(MPI)p 241 1946 15 2 v 17 w(SCATTERC)e Fs(b)q(eha)o(v)o(es)i(lik)o(e)h Fh(MPI)p 793 1946 V 17 w(SCATTER)e Fs(restricted)h(to)f(blo)q(c)o(k)i (bu\013ers,)f(and)g(with)g(the)g(ad-)75 2003 y(ditional)j(restriction)g (that)e(all)h(output)g(bu\013ers)g(ha)o(v)o(e)f(the)h(same)f(length.)32 b(The)19 b(input)g(bu\013er)g(blo)q(c)o(k)75 2059 y(of)e(the)h(ro)q(ot) e(pro)q(cess)i(is)g(partitioned)g(in)o(to)g Fh(n)f Fs(consecutiv)o(e)i (blo)q(c)o(ks,)f(eac)o(h)g(consisting)g(of)f Fh(len)g Fs(w)o(ords.)75 2116 y(The)d Fh(i)p Fs(-th)h(blo)q(c)o(k)f(is)h(sen)o (t)f(to)g(the)g Fh(i)p Fs(-th)g(pro)q(cess)g(in)i(the)e(group)g(and)g (stored)g(in)h(its)f(output)g(bu\013er.)20 b(The)75 2172 y(routine)e(is)h(called)g(b)o(y)f(all)h(mem)o(b)q(ers)f(of)g(the)g (group)f(using)i(the)f(same)f(argumen)o(ts)g(for)h Fh(group,)23 b(len)p Fs(,)75 2229 y(and)15 b Fh(root)p Fs(.)75 2324 y Ft(IN)i(in)o(buf)23 b Fs(\014rst)15 b(en)o(try)g(in)h(input)g (bu\013er)f({)g(signi\014can)o(t)h(only)g(at)e(ro)q(ot)h(\(c)o (hoice\).)75 2419 y Ft(OUT)j(outbuf)24 b Fs(\014rst)14 b(en)o(try)h(in)h(output)f(bu\013er)g(\(c)o(hoice\).)75 2514 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(to)g (b)q(e)h(stored)e(in)i(output)f(bu\013er)h(\(in)o(teger\))75 2609 y Ft(IN)h(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er) 75 2704 y Ft(IN)j(group)23 b Fs(handle)p eop %%Page: 45 50 45 49 bop 75 -100 a Fq(3.5.)29 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1082 b Fs(45)75 45 y Ft(IN)17 b(ro)q(ot)23 b Fs(rank)15 b(of)g(sending)i(pro)q(cess)e(\(in)o(teger\))166 142 y Fh(MPI)p 241 142 15 2 v 17 w(SCATTERC\()22 b(inbuf,)h(outbuf,)g (outlen,)g(type,)g(group,)g(root\))39 b Fs(is)75 239 y Fh(MPI_GSIZE\()22 b(&size,)i(group\);)75 295 y(MPI_RANK\()f(&rank,)g (group\);)75 352 y(MPI_IRECVC\()f(handle,)h(outbuf,)g(outlen,)g(type,)g (root,)g(tag,)h(group,)f(return_handle\);)75 408 y(if)h(\(rank=root\)) 147 465 y(for)f(\(i=0;)g(i)h(<)g(size;)f(i++\))147 521 y({)218 578 y(MPI_SENDC\(inbuf,)f(outlen,)h(type,)g(i,)h(tag,)f (group\);)218 634 y(inbuf)g(+=)h(outlen;)147 690 y(})75 747 y(MPI_WAIT\(handle\);)75 871 y Ft(All-to-all)d(scatter)75 993 y(MPI)p 184 993 16 2 v 18 w(ALLSCA)l(TTER\()d(list)p 659 993 V 20 w(of)p 721 993 V 19 w(in)o(bufs,)f(list)p 978 993 V 20 w(of)p 1040 993 V 19 w(outbufs,)g(group,)g(return)p 1571 993 V 18 w(status\))166 1086 y Fs(Eac)o(h)h(pro)q(cess)g(in)h(the) f(group)f(sends)i(its)f Fh(i)p Fs(-th)g(bu\013er)g(in)h(its)f(input)h (bu\013er)f(list)g(to)g(the)g(pro)q(cess)75 1142 y(with)i(rank)g Fh(i)g Fs(\(itself)g(included\);)k(eac)o(h)c(pro)q(cess)g(places)h(the) f(incoming)h(messages)e(in)i(the)f(lo)q(cation)75 1199 y(sp)q(eci\014ed)h(b)o(y)d(output)h(bu\013er)f(handle)i(corresp)q (onding)f(to)f(the)h(rank)f(of)g(the)h(sender.)31 b(F)l(or)17 b(example,)75 1255 y(eac)o(h)d(pro)q(cess)h(places)g(the)f(data)f(from) h(pro)q(cess)g(with)h(rank)f(3)g(in)h(the)f(lo)q(cation)h(sp)q (eci\014ed)h(b)o(y)e(the)g(third)75 1312 y(bu\013er)j(descriptor)g(in)h (the)f(list)h(of)e(outbufs.)25 b(The)17 b(routine)g(is)h(called)g(b)o (y)f(all)h(mem)o(b)q(ers)f(of)f(the)h(group)75 1368 y(using)f(the)f (same)g(argumen)o(ts)f(for)h Fh(group)p Fs(.)75 1465 y Ft(IN)i(list)p 225 1465 V 20 w(of)p 287 1465 V 19 w(in)o(bufs)22 b Fs(list)16 b(of)f(bu\013er)g(descriptor)h(handles)75 1561 y Ft(IN)h(list)p 225 1561 V 20 w(of)p 287 1561 V 19 w(outbufs)23 b Fs(list)16 b(of)f(bu\013er)g(descriptor)g(handles)75 1658 y Ft(IN)i(group)23 b Fs(handle)75 1754 y Ft(OUT)18 b(return)p 357 1754 V 19 w(status)23 b Fs(return)15 b(status)f(handle) 166 1886 y Ft(MPI)p 275 1886 V 18 w(ALLSCA)l(TTER)o(C\()k(in)o(buf,)f (outbuf,)g(len,)h(t)o(yp)q(e,)f(group\))166 1979 y Fh(MPI)p 241 1979 15 2 v 17 w(ALLSCATTERC)g Fs(b)q(eha)o(v)o(es)i(lik)o(e)g Fh(MPI)p 870 1979 V 17 w(ALLSCATTER)e Fs(restricted)i(to)f(blo)q(c)o(k) i(bu\013ers,)f(and)g(with)75 2035 y(the)f(additional)i(restriction)f (that)e(all)i(blo)q(c)o(ks)g(sen)o(t)f(from)f(one)i(pro)q(cess)f(to)f (another)h(ha)o(v)o(e)g(the)g(same)75 2092 y(length.)i(The)14 b(input)g(bu\013er)f(blo)q(c)o(k)i(of)e(eac)o(h)g(pro)q(cess)h(is)g (partitioned)g(in)o(to)f Fh(n)g Fs(consecutiv)o(e)i(blo)q(c)o(ks,)f (eac)o(h)75 2148 y(consisting)19 b(of)f Fh(len)g Fs(w)o(ords.)29 b(The)18 b Fh(i)p Fs(-th)g(blo)q(c)o(k)i(is)e(sen)o(t)h(to)e(the)i Fh(it)p Fs(-th)f(pro)q(cess)g(in)i(the)e(group.)29 b(Eac)o(h)75 2205 y(pro)q(cess)17 b(concatenates)g(the)g(incoming)h(messages,)e(in)i (the)f(order)g(of)f(the)h(senders')g(ranks,)g(and)g(store)75 2261 y(them)c(in)i(its)f(output)f(bu\013er.)19 b(The)14 b(routine)g(is)g(called)h(b)o(y)e(all)i(mem)o(b)q(ers)f(of)f(the)g (group)g(using)i(the)e(same)75 2318 y(argumen)o(ts)h(for)h Fh(group)p Fs(,)f(and)h Fh(len)p Fs(.)75 2414 y Ft(IN)i(in)o(buf)23 b Fs(\014rst)15 b(en)o(try)g(in)h(input)g(bu\013er)f(\(matc)o(hes)g(t)o (yp)q(e\).)75 2511 y Ft(OUT)j(outbuf)24 b Fs(\014rst)14 b(en)o(try)h(in)h(output)f(bu\013er)g(\(matc)o(hes)g(t)o(yp)q(e\).)75 2607 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(sen)o (t)g(from)g(eac)o(h)g(pro)q(cess)g(to)g(eac)o(h)g(other)g(\(in)o (teger\).)75 2704 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)p eop %%Page: 46 51 46 50 bop 75 -100 a Fs(46)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(IN)h(group)23 b Fs(handle)166 140 y Fh(MPI)p 241 140 15 2 v 17 w(ALLSCATTERC\()f (inbuf,)h(outbuf,)g(len,)g(type,)g(group\))15 b Fs(is)75 236 y Fh(MPI_GSIZE\()22 b(group,)i(&size)f(\);)75 292 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 349 y(for)g(\(i=0;)h(i)f(<)h (rank;)f(i++\))147 405 y({)170 462 y(MPI_IRECVC\(recv_handles[i],)e (outbuf,)i(len,)g(type,)g(tag,)g(group,)g(return_handle\);)170 518 y(outbuf)g(+=)h(len;)147 575 y(})75 631 y(for)f(\(i=0;)h(i)f(<)h (size;)f(i++\))147 688 y({)170 744 y(MPI_ISENDC\(send_handle[i],)e (inbuf,)i(len,)g(type,)g(i,)h(tag,)f(group\);)170 800 y(inbuf)h(+=)f(len;)147 857 y(})75 913 y(MPI_WAITALL\(send_handle\);)75 970 y(MPI_WAITALL\(recv_handle\);)75 1092 y Ft(All-to-all)e(broadcast) 75 1214 y(MPI)p 184 1214 16 2 v 18 w(ALLCAST\()d(in)o(buf,)f(list)p 702 1214 V 20 w(of)p 764 1214 V 19 w(outbufs,)g(group,)g(return)p 1295 1214 V 18 w(status\))166 1306 y Fs(Eac)o(h)c(pro)q(cess)h(in)g (the)g(group)f(broadcasts)g(its)h(input)g(bu\013er)g(to)f(all)h(pro)q (cesses)g(\(including)i(itself)t(\);)75 1363 y(eac)o(h)e(pro)q(cess)f (places)i(the)f(incoming)g(messages)f(in)i(the)e(lo)q(cation)i(sp)q (eci\014ed)g(b)o(y)f(output)f(bu\013er)h(handle)75 1419 y(corresp)q(onding)19 b(to)d(the)i(rank)f(of)h(the)f(sender.)28 b(F)l(or)17 b(example,)h(eac)o(h)g(pro)q(cess)g(places)g(the)g(data)f (from)75 1475 y(pro)q(cess)j(with)f(rank)g(3)g(in)h(the)g(lo)q(cation)g (sp)q(eci\014ed)h(b)o(y)e(the)h(third)g(bu\013er)f(descriptor)h(in)g (the)f(list)h(of)75 1532 y(outbufs.)25 b(The)17 b(routine)h(is)f (called)i(b)o(y)e(all)h(mem)o(b)q(ers)f(of)f(the)h(group)g(using)h(the) f(same)f(argumen)o(ts)h(for)75 1588 y Fh(group)p Fs(.)75 1684 y Ft(IN)g(in)o(buf)23 b Fs(bu\013er)15 b(descriptor)h(handle)g (for)f(input)h(bu\013er)75 1779 y Ft(IN)h(list)p 225 1779 V 20 w(of)p 287 1779 V 19 w(outbufs)23 b Fs(list)16 b(of)f(bu\013er)g(descriptor)g(handles)75 1874 y Ft(IN)i(group)23 b Fs(handle)75 1969 y Ft(OUT)18 b(return)p 357 1969 V 19 w(status)23 b Fs(return)15 b(status)f(handle)166 2100 y Ft(MPI)p 275 2100 V 18 w(ALLCASTC\()j(in)o(buf,)h(outbuf,)f(len,)h(t) o(yp)q(e,)e(group\))166 2192 y Fh(MPI)p 241 2192 15 2 v 17 w(ALLCASTC)f Fs(b)q(eha)o(v)o(es)i(lik)o(e)h Fh(MPI)p 793 2192 V 17 w(ALLCAST)e Fs(restricted)h(to)f(blo)q(c)o(k)i (bu\013ers,)f(and)g(with)g(the)g(ad-)75 2249 y(ditional)h(restriction)f (that)e(all)j(blo)q(c)o(ks)f(sen)o(t)f(from)f(one)i(pro)q(cess)g(to)e (another)h(ha)o(v)o(e)g(the)h(same)f(length.)75 2305 y(Eac)o(h)h(pro)q(cess)g(concatenates)f(the)h(incoming)h(messages,)e (in)i(the)f(order)f(of)h(the)f(senders')h(ranks,)g(and)75 2361 y(store)c(them)h(in)g(its)g(output)f(bu\013er.)20 b(The)14 b(routine)g(is)g(called)h(b)o(y)f(all)g(mem)o(b)q(ers)g(of)f (the)h(group)f(using)i(the)75 2418 y(same)g(argumen)o(ts)f(for)h Fh(group)p Fs(,)f(and)h Fh(len)p Fs(.)75 2513 y Ft(IN)i(in)o(buf)23 b Fs(\014rst)15 b(en)o(try)g(in)h(input)g(bu\013er)f(\(c)o(hoice\).)20 b(ro)q(ot)15 b(\(in)o(teger\))75 2608 y Ft(OUT)j(outbuf)24 b Fs(\014rst)14 b(en)o(try)h(in)h(output)f(bu\013er)g(\(c)o(hoice\).)75 2704 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(sen)o (t)g(from)g(eac)o(h)g(pro)q(cess)g(to)g(eac)o(h)g(other)g(\(including)j (itself)t(\).)p eop %%Page: 47 52 47 51 bop 75 -100 a Fq(3.6.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b Fs(47)75 45 y Ft(IN)17 b(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 139 y Ft(IN)j(group)23 b Fs(group)15 b(handle)166 233 y Fh(MPI)p 241 233 15 2 v 17 w(ALLCASTC\()22 b(inbuf,)h(outbuf,)g(len,)h(type,)f(group\))14 b Fs(is)75 327 y Fh(MPI_GSIZE\()22 b(group,)i(&size)f(\);)75 383 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 440 y(for)g(\(i=0;)h(i)f(<)h (rank;)f(i++\))147 496 y({)170 553 y(MPI_IRECVC\(recv_handles[i],)e (outbuf,)i(len,)g(type,)g(tag,)g(group,)g(return_handle\);)170 609 y(outbuf)g(+=)h(len;)147 665 y(})75 722 y(for)f(\(i=0;)h(i)f(<)h (size;)f(i++\))147 778 y({)170 835 y(MPI_ISENDC\(send_handle[i],)e (inbuf,)i(len,)g(type,)g(i,)h(tag,)f(group\);)147 891 y(})75 948 y(MPI_WAITALL\(send_handle\);)75 1004 y (MPI_WAITALL\(recv_handle\);)75 1148 y Fm(3.6)70 b(Global)22 b(Compute)g(Op)r(erations)75 1249 y Ft(Reduce)75 1370 y(MPI)p 184 1370 16 2 v 18 w(REDUCE\()d(in)o(buf,)e(outbuf,)g(group,)g (ro)q(ot,)h(op\))166 1462 y Fs(Com)o(bines)f(the)h(v)m(alues)g(pro)o (vided)g(in)g(the)f(input)h(bu\013er)g(of)e(eac)o(h)i(pro)q(cess)f(in)h (the)f(group,)g(using)75 1519 y(the)f(op)q(eration)g Fh(op)p Fs(,)g(and)g(returns)f(the)h(com)o(bined)h(v)m(alue)h(in)e(the) g(output)g(bu\013er)g(of)f(the)h(pro)q(cess)h(with)75 1575 y(rank)f Fh(root)p Fs(.)21 b(Eac)o(h)16 b(pro)q(cess)g(can)g(pro)o (vide)g(one)g(v)m(alue,)h(or)f(a)f(sequence)i(of)f(v)m(alues,)h(in)f (whic)o(h)h(case)f(the)75 1632 y(com)o(bine)f(op)q(eration)f(is)h (executed)g(p)q(oin)o(t)o(wise)g(on)f(eac)o(h)g(en)o(try)g(of)g(the)g (sequence.)21 b(F)l(or)13 b(example,)i(if)g(the)75 1688 y(op)q(eration)i(is)g Fh(max)g Fs(and)g(input)g(bu\013ers)g(con)o (tains)g(t)o(w)o(o)f(\015oating)g(p)q(oin)o(t)i(n)o(um)o(b)q(ers,)f (then)g(outbuf\(1\))f(=)75 1744 y(global)e(max\(in)o(buf\(1\)\))e(and)i (outbuf\(2\))e(=)h(global)h(max\(in)o(buf\(2\)\).)19 b(All)14 b(input)g(bu\013ers)g(should)g(de\014ne)75 1801 y(sequences)g(of)f(equal)g(length)h(of)e(en)o(tries)i(of)e(t)o(yp)q(es) h(that)f(matc)o(h)h(the)g(t)o(yp)q(e)g(of)f(the)h(op)q(erands)g(of)g Fh(op)p Fs(.)19 b(The)75 1857 y(output)f(bu\013er)g(should)i(de\014ne)f (a)f(sequence)h(of)f(the)g(same)g(length)h(of)f(en)o(tries)h(of)f(t)o (yp)q(es)g(that)g(matc)o(h)75 1914 y(the)g(t)o(yp)q(e)g(of)g(the)g (result)h(of)e Fh(op)p Fs(.)29 b(\(Note)17 b(that,)h(here)g(as)g(for)g (all)h(other)e(comm)o(unication)i(op)q(erations,)75 1970 y(the)d(t)o(yp)q(e)f(of)g(en)o(tries)h(inserted)g(in)h(a)e(message)g (dep)q(end)i(on)e(the)h(information)g(pro)o(vided)g(b)o(y)f(the)h (input)75 2027 y(bu\013er)d(descriptor,)g(and)h(not)e(on)h(the)g (declarations)h(of)f(these)g(v)m(ariables)h(in)g(the)f(calling)i (program.)j(The)75 2083 y(t)o(yp)q(es)13 b(of)g(the)g(v)m(ariables)i (in)f(the)f(calling)i(program)d(need)i(not)f(matc)o(h)g(the)g(t)o(yp)q (es)g(de\014ned)i(b)o(y)e(the)g(bu\013er)75 2140 y(descriptor,)22 b(but)f(in)g(suc)o(h)g(case)g(the)g(outcome)f(of)h(a)f(reduce)h(op)q (eration)g(ma)o(y)f(b)q(e)i(implemen)o(tation)75 2196 y(dep)q(enden)o(t.\))166 2253 y(The)16 b(op)q(eration)g(de\014ned)i(b)o (y)e Fh(op)f Fs(is)i(asso)q(ciativ)o(e)f(and)g(comm)o(utativ)o(e,)f (and)i(the)f(implemen)o(tation)75 2309 y(can)i(tak)o(e)f(adv)m(an)o (tage)g(of)g(asso)q(ciativit)o(y)h(and)g(comm)o(utativit)o(y)f(in)h (order)f(to)g(c)o(hange)h(order)f(of)h(ev)m(alu-)75 2365 y(ation.)29 b(The)18 b(routine)h(is)g(called)h(b)o(y)e(all)h(group)f (mem)o(b)q(ers)h(using)g(the)f(same)g(argumen)o(ts)f(for)h Fh(group,)75 2422 y(root)d Fs(and)g Fh(op)p Fs(.)75 2516 y Ft(IN)i(in)o(buf)23 b Fs(handle)17 b(to)d(input)i(bu\013er)75 2610 y Ft(IN)h(outbuf)23 b Fs(handle)17 b(to)d(output)h(bu\013er)h({)f (signi\014can)o(t)h(only)f(at)g(ro)q(ot)75 2704 y Ft(IN)i(group)23 b Fs(handle)16 b(to)f(group)p eop %%Page: 48 53 48 52 bop 75 -100 a Fs(48)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(IN)h(ro)q(ot)23 b Fs(rank)15 b(of)g(ro)q(ot)g(pro)q(cess)g(\(in)o(teger\))75 138 y Ft(IN)i(op)23 b Fs(op)q(eration)166 229 y(The)18 b(bu\013er)g(descriptor)g(con)o(tains)g(data)g(t)o(yp)q(e)g (information)g(so)f(that)h(the)g(correct)f(form)g(of)h(the)75 285 y(op)q(eration)d(can)h(b)q(e)g(p)q(erformed.)k(W)l(e)15 b(list)h(b)q(elo)o(w)g(the)f(op)q(erations)g(whic)o(h)h(are)f(supp)q (orted.)75 376 y Ft(MPI)p 184 376 16 2 v 18 w(MAX)21 b Fs(maxim)o(um)75 469 y Ft(MPI)p 184 469 V 18 w(MIN)h Fs(minim)o(um)75 562 y Ft(MPI)p 184 562 V 18 w(MIN)g Fs(minim)o(um)75 654 y Ft(MPI)p 184 654 V 18 w(SUM)h Fs(sum)75 747 y Ft(MPI)p 184 747 V 18 w(PR)o(OD)g Fs(pro)q(duct)75 840 y Ft(MPI)p 184 840 V 18 w(AND)f Fs(and)16 b(\(logical)g(or)e (bit-wise)j(in)o(teger\))75 932 y Ft(MPI)p 184 932 V 18 w(OR)23 b Fs(or)15 b(\(logical)h(or)f(bit-wise)h(in)o(teger\))75 1025 y Ft(MPI)p 184 1025 V 18 w(X)o(OR)22 b Fs(xor)15 b(\(logical)h(or)f(bit-wise)h(in)o(teger\))75 1118 y Ft(MPI)p 184 1118 V 18 w(MAXLOC)22 b Fs(rank)15 b(of)g(pro)q(cess)g (with)h(maxim)o(um)f(v)m(alue)75 1210 y Ft(MPI)p 184 1210 V 18 w(MINLOC)23 b Fs(rank)14 b(of)h(pro)q(cess)h(with)f(minim)o (um)i(v)m(alue)166 1337 y Ft(MPI)p 275 1337 V 18 w(REDUCEC\()h(in)o (buf,)f(outbuf,)h(len,)f(t)o(yp)q(e,)g(group,)g(ro)q(ot,)h(op\))166 1429 y Fs(Is)d(same)g(as)g Fh(MPI)p 460 1429 15 2 v 17 w(REDUCE)p Fs(,)f(restricted)h(to)g(a)g(blo)q(c)o(k)h(bu\013er.)75 1520 y Ft(IN)h(in)o(buf)23 b Fs(\014rst)15 b(lo)q(cation)h(in)g(input)g (bu\013er)75 1612 y Ft(OUT)i(outbuf)24 b Fs(\014rst)14 b(lo)q(cation)i(in)g(output)f(bu\013er)h({)e(signi\014can)o(t)j(only)e (at)g(ro)q(ot)75 1705 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(and)e(output)g(bu\013er)g(\(in)o (teger\))75 1798 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 1890 y Ft(IN)j(group)23 b Fs(handle)16 b(to)f(group)75 1983 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(ro)q(ot)g(pro)q(cess)g(\(in)o(teger\))75 2076 y Ft(IN)i(op)23 b Fs(op)q(eration)166 2202 y Ft(MPI)p 275 2202 16 2 v 18 w(USER)p 435 2202 V 20 w(REDUCE\()18 b(in)o(buf,)f(outbuf,)h(group,)f(ro)q(ot,)g(function\))166 2294 y Fs(Same)h(as)f(the)i(reduce)f(op)q(eration)h(function)f(ab)q(o)o (v)o(e)g(except)g(that)g(a)f(user)h(supplied)j(function)d(is)75 2351 y(used.)i Fh(function)11 b Fs(is)i(an)g(asso)q(ciativ)o(e)g(and)f (comm)o(utativ)o(e)g(function)i(with)f(t)o(w)o(o)e(argumen)o(ts.)18 b(The)13 b(t)o(yp)q(es)75 2407 y(of)h(the)h(t)o(w)o(o)e(argumen)o(ts)h (and)g(of)h(the)f(returned)h(v)m(alue)h(of)e(the)h(function,)g(and)g (the)f(t)o(yp)q(es)h(of)f(all)i(en)o(tries)75 2463 y(in)j(the)g(input)g (and)g(output)f(bu\013ers)h(all)g(agree.)29 b(The)19 b(output)f(bu\013er)h(has)f(the)g(same)h(length)g(as)f(the)75 2520 y(input)e(bu\013er.)75 2611 y Ft(IN)h(in)o(buf)23 b Fs(handle)17 b(to)d(input)i(bu\013er)75 2704 y Ft(IN)h(outbuf)23 b Fs(handle)17 b(to)d(output)h(bu\013er)h({)f(signi\014can)o(t)h(only)f (at)g(ro)q(ot)p eop %%Page: 49 54 49 53 bop 75 -100 a Fq(3.6.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b Fs(49)75 45 y Ft(IN)17 b(group)23 b Fs(handle)16 b(to)f(group)75 142 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(ro)q(ot)g(pro)q(cess)g(\(in)o(teger\))166 274 y Ft(MPI)p 275 274 16 2 v 18 w(USER)p 435 274 V 20 w(REDUCEC\()j(in)o (buf,)f(outbuf,)h(len,)f(t)o(yp)q(e,)g(group,)g(ro)q(ot,)g(function\)) 166 367 y Fs(Is)e(same)g(as)g Fh(MPI)p 460 367 15 2 v 477 367 V 34 w(USER)p 590 367 V 17 w(REDUCE)p Fs(,)f(restricted)h(to)g (a)g(blo)q(c)o(k)h(bu\013er.)75 464 y Ft(IN)h(in)o(buf)23 b Fs(\014rst)15 b(lo)q(cation)h(in)g(input)g(bu\013er)75 561 y Ft(OUT)i(outbuf)24 b Fs(\014rst)14 b(lo)q(cation)i(in)g(output)f (bu\013er)h({)e(signi\014can)o(t)j(only)e(at)g(ro)q(ot)75 658 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h (input)h(and)e(output)g(bu\013er)g(\(in)o(teger\))75 755 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er) 75 852 y Ft(IN)j(group)23 b Fs(handle)16 b(to)f(group)75 949 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(ro)q(ot)g(pro)q(cess)g (\(in)o(teger\))75 1046 y Ft(IN)i(function)24 b Fs(user)16 b(pro)o(vided)g(function)166 1219 y Fl(Discussion:)166 1270 y Fk(Do)f(w)o(e)h(also)f(w)o(an)o(t)g(a)g(v)o(ersion)h(of)f (reduce)i(that)e(broadcasts)i(the)f(result)g(to)g(all)e(pro)q(cesses)k (in)d(the)i(group?)75 1319 y(\(This)10 b(can)f(b)q(e)h(ac)o(hiev)o(ed)g (b)o(y)f(a)h(reduce)h(follo)o(w)o(ed)d(b)o(y)h(a)g(broadcast,)i(but)f (a)f(com)o(bined)f(function)h(ma)o(y)f(b)q(e)i(somewhat)75 1369 y(more)j(e\016cien)o(t.\))19 b(These)c(w)o(ould)e(b)q(e)h(resp)q (ectiv)o(ely:)166 1455 y Fl(MPI)p 265 1455 V 17 w(GOP\()h(in)o(buf,)e (outbuf,)h(group,)g(op\))166 1577 y(MPI)p 265 1577 V 17 w(GOPC\()h(in)o(buf,)e(outbuf,)h(len,)h(t)o(yp)q(e,)g(group,)g(op\)) 166 1698 y(MPI)p 265 1698 V 17 w(USER)p 413 1698 V 17 w(GOP\()f(in)o(buf,)g(outbuf,)f(group,)i(function\))166 1820 y(MPI)p 265 1820 V 17 w(USER)p 413 1820 V 17 w(GOPC\()g(in)o(buf,) e(outbuf,)h(len,)h(t)o(yp)q(e,)g(group,)g(function\))166 1906 y Fk(Do)e(w)o(e)h(w)o(an)o(t)f(a)g(user)i(pro)o(vided)e Fc(function)h Fk(\(t)o(w)o(o)f(IN)h(parameters,)f(one)h(OUT)g(v)n (alue\),)f(or)g(a)g(user)i(pro)o(vided)75 1955 y(pro)q(cedure)i(that)f (o)o(v)o(erwrites)g(the)g(second)h(input)e(\(ie.)23 b(one)15 b(IN)h(param,)d(one)j(INOUT)g(param,)e(the)i(equiv)n(alen)o(t)75 2005 y(of)i(C)g Fg(a)k(op=)f(b)d Fk(t)o(yp)q(e)h(assignmen)o(t\)?)31 b(The)19 b(second)h(c)o(hoice)f(ma)o(y)d(allo)o(w)h(a)h(more)g (e\016cien)o(t)h(implemen)o(tatio)o(n,)75 2055 y(without)14 b(c)o(hanging)f(the)h(seman)o(tics)g(of)f(the)h(MPI)g(call.)75 2262 y Ft(Scan)75 2385 y(MPI)p 184 2385 16 2 v 18 w(SCAN\()j(in)o(buf,) g(outbuf,)h(group,)e(op)i(\))166 2478 y Fs(MPI)p 258 2478 14 2 v 16 w(SCAN)13 b(is)g(used)g(to)f(p)q(erform)h(a)f(parallel)i (pre\014x)f(with)g(resp)q(ect)h(to)d(an)i(asso)q(ciativ)o(e)g (reduction)75 2534 y(op)q(eration)h(on)h(data)e(distributed)j(across)e (the)g(group.)19 b(The)c(op)q(eration)f(returns)g(in)i(the)e(output)g (bu\013er)75 2591 y(of)g(the)g(pro)q(cess)g(with)h(rank)e Fh(i)h Fs(the)g(reduction)h(of)f(the)g(v)m(alues)h(in)g(the)f(input)i (bu\013ers)e(of)f(pro)q(cesses)i(with)75 2647 y(ranks)h Fh(0,...,i)p Fs(.)21 b(The)c(t)o(yp)q(e)f(of)g(op)q(erations)g(supp)q (orted)h(and)f(their)h(seman)o(tic,)f(and)g(the)g(constrain)o(ts)75 2704 y(on)f(input)h(and)g(output)f(bu\013ers)g(are)g(as)g(for)f Fh(MPI)p 921 2704 15 2 v 17 w(REDUCE)p Fs(.)p eop %%Page: 50 55 50 54 bop 75 -100 a Fs(50)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(IN)h(in)o(buf)23 b Fs(handle)17 b(to)d(input)i(bu\013er)75 142 y Ft(IN)h(outbuf)23 b Fs(handle)17 b(to)d(output)h(bu\013er)75 239 y Ft(IN)i(group)23 b Fs(handle)16 b(to)f(group)75 336 y Ft(IN)i(op)23 b Fs(op)q(eration)166 469 y Ft(MPI)p 275 469 16 2 v 18 w(SCANC\()12 b(in)o(buf,)h(outbuf,)g(len,)h(t)o(yp)q(e,)e(group,)h(op)g (\))e Fs(Same)g(as)f Fh(MPI)p 1555 469 15 2 v 17 w(SCAN)p Fs(,)g(restricted)75 561 y(to)15 b(blo)q(c)o(k)h(bu\013ers.)75 671 y Ft(IN)h(in)o(buf)23 b Fs(\014rst)15 b(input)h(bu\013er)f(elemen)o (t)h(\(c)o(hoice\))75 768 y Ft(OUT)i(outbuf)24 b Fs(\014rst)14 b(output)h(bu\013er)h(elemen)o(t)f(\(c)o(hoice\))75 865 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h (input)h(and)e(output)g(bu\013er)g(\(in)o(teger\))75 962 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er) 75 1059 y Ft(IN)j(group)23 b Fs(handle)16 b(to)f(group)75 1156 y Ft(IN)i(op)23 b Fs(op)q(eration)166 1302 y Ft(MPI)p 275 1302 16 2 v 18 w(USER)p 435 1302 V 20 w(SCAN\()16 b(in)o(buf,)i(outbuf,)f(group,)g(function)i(\))166 1395 y Fs(Same)12 b(as)h(the)f(scan)h(op)q(eration)f(function)i(ab)q(o)o(v)o (e)e(except)h(that)f(a)g(user)h(supplied)h(function)g(is)f(used.)75 1451 y Fh(function)k Fs(is)i(an)f(asso)q(ciativ)o(e)g(and)h(comm)o (utativ)o(e)e(function)i(with)g(t)o(w)o(o)e(argumen)o(ts.)28 b(The)18 b(t)o(yp)q(es)g(of)75 1508 y(the)d(t)o(w)o(o)f(argumen)o(ts)g (and)i(of)f(the)g(returned)h(v)m(alues)g(all)g(agree.)75 1605 y Ft(IN)h(in)o(buf)23 b Fs(handle)17 b(to)d(input)i(bu\013er)75 1702 y Ft(IN)h(outbuf)23 b Fs(handle)17 b(to)d(output)h(bu\013er)75 1799 y Ft(IN)i(group)23 b Fs(handle)16 b(to)f(group)75 1896 y Ft(IN)i(function)24 b Fs(user)16 b(pro)o(vided)g(function)166 2028 y Ft(MPI)p 275 2028 V 18 w(USER)p 435 2028 V 20 w(SCANC\()g(in)o(buf,)h(outbuf,)h(len,)f(t)o(yp)q(e,)g(group,)g (function\))166 2121 y Fs(Is)e(same)g(as)g Fh(MPI)p 460 2121 15 2 v 17 w(USER)p 573 2121 V 17 w(SCAN)p Fs(,)f(restricted)h(to)g (a)g(blo)q(c)o(k)h(bu\013er.)75 2218 y Ft(IN)h(in)o(buf)23 b Fs(\014rst)15 b(lo)q(cation)h(in)g(input)g(bu\013er)75 2315 y Ft(OUT)i(outbuf)24 b Fs(\014rst)14 b(lo)q(cation)i(in)g(output)f (bu\013er)75 2412 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(and)e(output)g(bu\013er)g(\(in)o (teger\))75 2509 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 2607 y Ft(IN)j(group)23 b Fs(handle)16 b(to)f(group)75 2704 y Ft(IN)i(function)24 b Fs(user)16 b(pro)o(vided)g(function)p eop %%Page: 51 56 51 55 bop 75 -100 a Fq(3.7.)29 b(CORRECTNESS)1308 b Fs(51)166 45 y Fl(Discussion:)166 102 y Fk(Do)15 b(w)o(e)h(w)o(an)o(t)f(scan)h (op)q(erations)g(executed)i(b)o(y)d(segmen)o(ts?)24 b(\(The)16 b(HPF)g(de\014nition)f(of)g(pre\014x)i(and)e(su\016x)75 158 y(op)q(eration)h(migh)o(t)e(b)q(e)i(handy)g({)g(in)f(addition)g(to) h(the)h(scanned)g(v)o(ector)f(of)g(v)n(alues)g(there)h(is)f(a)f(mask)g (that)h(tells)75 214 y(where)f(segmen)o(ts)f(start)g(and)g(end.\))75 440 y Fm(3.7)70 b(Correctness)75 624 y Fl(Discussion:)34 b Fk(This)14 b(is)f(still)g(v)o(ery)i(preliminary)166 763 y Fs(The)k(seman)o(tics)g(of)f(the)h(collectiv)o(e)h(comm)o (unication)g(op)q(erations)f(can)f(b)q(e)i(deriv)o(ed)f(from)f(their)75 820 y(op)q(erational)g(de\014nition)i(in)f(terms)e(of)g(p)q(oin)o (t-to-p)q(oin)o(t)i(comm)o(unication.)28 b(It)18 b(is)g(assumed)g(that) f(mes-)75 876 y(sages)e(p)q(ertaining)i(to)e(one)g(op)q(eration)h (cannot)f(b)q(e)i(confused)f(with)g(messages)f(p)q(ertaining)h(to)f (another)75 933 y(op)q(eration.)20 b(Also)14 b(messages)f(p)q (ertaining)i(to)e(t)o(w)o(o)g(distinct)i(o)q(ccurrences)g(of)e(the)h (same)g(op)q(eration)g(can-)75 989 y(not)h(b)q(e)h(confused,)f(if)h (the)g(t)o(w)o(o)d(o)q(ccurrences)k(ha)o(v)o(e)e(distinct)h (parameters.)j(The)d(relev)m(an)o(t)g(parameters)75 1046 y(for)e(this)h(purp)q(ose)h(are)e Fh(group)p Fs(,)g Fh(root)g Fs(and)h Fh(op)p Fs(.)k(messages)14 b(p)q(ertaining)i(to)e(another)h(o) q(ccurrence)h(of)e(the)75 1102 y(same)k(op)q(eration,)g(with)g (di\013eren)o(t)g(parameters.)27 b(The)18 b(implemen)o(ter)i(can,)e(of) f(course,)i(use)f(another,)75 1158 y(more)d(e\016cien)o(t)h(implemen)o (tation,)g(as)f(long)g(as)g(it)g(has)g(the)h(same)f(e\013ect.)166 1291 y Fl(Discussion:)166 1341 y Fk(This)c(statemen)o(t)h(do)q(es)g (not)g(y)o(et)g(apply)e(to)i(the)g(curren)o(t,)h(incomplete)e(and)g (somewhat)g(careless)i(de\014nitions)75 1391 y(I)h(pro)o(vided)g(in)f (this)h(draft.)166 1440 y(The)i(de\014nition)f(ab)q(o)o(v)o(e)g(means)g (that)g(messages)h(p)q(ertaining)f(to)g(a)g(collectiv)o(e)h(comm)o(uni) o(cation)d(carry)j(in-)75 1490 y(formation)11 b(iden)o(tifying)h(the)j (op)q(eration)e(itself,)g(and)h(the)g(v)n(alues)g(of)f(the)h Fg(group)f Fk(and,)g(where)i(relev)n(an)o(t,)e Fg(root)g Fk(or)75 1540 y Fg(op)g Fk(parameters.)19 b(Is)14 b(this)g(acceptable?) 166 1679 y Fs(A)h(few)g(examples:)75 1782 y Fh(MPI_BCASTC\(buf,)22 b(len,)h(type,)g(group,)g(0\);)75 1839 y(MPI_BCASTC\(buf,)f(len,)h (type,)g(group,)g(1\);)166 1942 y Fs(Tw)o(o)13 b(consecutiv)o(e)h (broadcasts,)f(in)h(the)g(same)f(group,)g(with)h(the)g(same)f(tag,)g (but)g(di\013eren)o(t)h(ro)q(ots.)75 1999 y(Since)k(the)e(op)q (erations)g(are)g(distinguishable,)i(messages)e(from)f(one)h(broadcast) g(cannot)g(b)q(e)g(confused)75 2055 y(with)g(messages)e(from)h(the)g (other)g(broadcast;)f(the)h(program)f(is)i(safe)f(and)g(will)i(execute) f(as)f(exp)q(ected.)75 2158 y Fh(MPI_BCASTC\(buf,)22 b(len,)h(type,)g(group,)g(0\);)75 2215 y(MPI_BCASTC\(buf,)f(len,)h (type,)g(group,)g(0\);)166 2318 y Fs(Tw)o(o)18 b(consecutiv)o(e)i (broadcasts,)f(in)h(the)f(same)g(group,)h(with)f(the)g(same)g(tag)f (and)i(ro)q(ot.)30 b(Since)75 2374 y(p)q(oin)o(t-to-p)q(oin)o(t)14 b(comm)o(unication)g(preserv)o(es)g(the)f(order)h(of)f(messages)g (here,)g(to)q(o,)g(messages)g(from)g(one)75 2431 y(broadcast)g(will)i (not)e(b)q(e)h(confused)g(with)g(messages)f(from)g(the)g(other)g (broadcast;)g(the)h(program)e(is)i(safe)75 2487 y(and)h(will)i(execute) f(as)f(in)o(tended.)75 2591 y Fh(MPI_RANK\(&rank,)22 b(group\))75 2647 y(if)i(\(rank==0\))123 2704 y({)p eop %%Page: 52 57 52 56 bop 75 -100 a Fs(52)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)147 45 y Fh(MPI_BCASTC\(buf,)21 b(len,)j(type,)f(group,)g(0\);)147 102 y(MPI_SENDC\(buf,)f(len,)h (type,)g(2,)h(tag,)f(group\);)123 158 y(})75 214 y(elseif)g (\(rank==1\))123 271 y({)147 327 y(MPI_RECVC\(buf,)f(len,)h(type,)g (MPI_DONTCARE,)f(tag,)h(group\);)147 384 y(MPI_BCASTC\(buf,)e(len,)j (type,)f(group,)g(0\);)147 440 y(MPI_RECVC\(buf,)f(len,)h(type,)g (MPI_DONTCARE,)f(tag,)h(group\);)123 497 y(})75 553 y(else)123 610 y({)147 666 y(MPI_SENDC\(buf,)f(len,)h(type,)g(2,)h(tag,)f (group\);)147 723 y(MPI_BCASTC\(buf,)e(len,)j(type,)f(group,)g(0\);)123 779 y(})166 885 y Fs(Pro)q(cess)17 b(zero)g(executes)h(a)f(broadcast)f (follo)o(w)o(ed)i(b)o(y)f(a)g(send)g(to)g(pro)q(cess)g(one;)h(pro)q (cess)g(t)o(w)o(o)e(ex-)75 942 y(ecutes)h(a)f(send)h(to)f(pro)q(cess)h (one,)f(follo)o(w)o(ed)h(b)o(y)f(a)g(broadcast;)g(and)h(pro)q(cess)g (one)f(executes)h(a)f(receiv)o(e,)75 998 y(a)21 b(broadcast)g(and)g(a)g (receiv)o(e.)39 b(A)21 b(p)q(ossible)i(outcome)e(is)h(for)f(the)g(op)q (erations)h(to)e(b)q(e)i(matc)o(hed)f(as)75 1055 y(illustrated)c(b)o(y) e(the)g(diagram)g(b)q(elo)o(w.)170 1274 y Fh(0)549 b(1)525 b(2)457 1387 y(/)24 b(-)f(>)48 b(receive)285 b(/)24 b(-)g(send)409 1443 y(/)597 b(/)75 1500 y(broadcast)70 b(/)215 b(broadcast)166 b(/)71 b(broadcast)337 1556 y(/)573 b(/)123 1613 y(send)71 b(-)310 b(receive)47 b(<)23 b(-)166 1832 y Fs(The)14 b(reason)g(is)h(that)f(broadcast)f(is)i(not)f(a)g(sync)o(hronous)g(op)q (eration;)h(the)f(call)h(at)f(a)g(pro)q(cess)h(ma)o(y)75 1888 y(return)j(b)q(efore)g(the)f(other)h(pro)q(cesses)g(ha)o(v)o(e)f (en)o(tered)h(the)g(broadcast.)27 b(Th)o(us,)18 b(the)f(message)h(sen)o (t)f(b)o(y)75 1945 y(pro)q(cess)g(zero)f(can)h(arriv)o(e)g(to)f(pro)q (cess)g(one)h(b)q(efore)g(the)g(message)f(sen)o(t)g(b)o(y)h(pro)q(cess) g(t)o(w)o(o,)e(and)i(b)q(efore)75 2001 y(the)e(call)i(to)d(broadcast)h (on)g(pro)q(cess)g(one.)p eop %%Page: 53 58 53 57 bop 75 367 a Fn(Chapter)34 b(4)75 586 y Fu(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)850 819 y Fs(Rolf)16 b(Hemp)q(el)75 1024 y Fm(4.1)70 b(In)n(tro)r(duction)75 1147 y Fs(A)14 b(pro)q(cess)f(group)g(in)i(MPI)e(is)h(a)f(collection)j(of)d(n)h(pro)q (cesses.)19 b(Eac)o(h)14 b(pro)q(cess)f(in)i(the)e(group)g(is)h (assigned)75 1203 y(a)g(rank)g(b)q(et)o(w)o(een)h(0)g(and)f(n-1.)20 b(In)15 b(the)g(ma)s(jorit)o(y)e(of)h(parallel)i(applications)g(this)f (linear)h(ordering)f(do)q(es)75 1260 y(not)e(re\015ect)g(the)g(comm)o (unication)h(structure)e(of)h(the)g(pro)q(cesses)g(whic)o(h)h(is)f (usually)i(determined)f(b)o(y)f(the)75 1316 y(underlying)20 b(problem)e(geometry)f(and)h(the)f(n)o(umerical)i(algorithm)f(used.)28 b(Often)18 b(the)f(pro)q(cesses)h(are)75 1373 y(arranged)d(in)h(top)q (ological)g(patterns)e(as)h(simple)i(as)e(a)g(t)o(w)o(o{)e(or)i (three{dimensional)i(grid.)166 1440 y(F)l(or)12 b(the)h(user)g(who)g (programs)f(an)h(application)h(on)f(suc)o(h)g(a)g(pro)q(cess)g (structure)g(the)g(most)f(natural)75 1496 y(w)o(a)o(y)k(of)g(referring) i(to)e(individual)k(pro)q(cesses)d(is)h(to)e(sp)q(ecify)i(their)g(co)q (ordinates)f(in)h(the)f(pro)q(cess)g(grid.)75 1553 y(Usually)22 b(a)e(pro)q(cess)g(do)q(es)h(not)f(ha)o(v)o(e)g(to)g(kno)o(w)g(the)g (PID)h(n)o(um)o(b)q(ers)f(of)g Fi(al)r(l)g Fs(pro)q(cesses)h(in)g(the)g (grid,)75 1609 y(b)q(ecause)h(comm)o(unication)g(only)g(tak)o(es)f (place)h(with)g(pro)q(cesses)f(in)h(a)f(certain)h(neigh)o(b)q(orho)q(o) q(d,)i(and)75 1666 y(p)q(erhaps)18 b(with)g(a)f(few)h(distan)o(t)f(pro) q(cesses.)27 b(Non{scalable)19 b(features)e(lik)o(e)i(the)e(explicit)j (listing)f(of)e(all)75 1722 y(participating)c(pro)q(cesses)g(in)g(the)g (application)h(could,)g(therefore,)e(b)q(e)h(a)o(v)o(oided,)g(if)f (there)h(w)o(ere)f(a)g(means)75 1779 y(to)j(sp)q(ecify)h(the)f (information)h(whic)o(h)g(is)f(really)i(required.)166 1846 y(Additionally)l(,)g(the)e(user{de\014ned)h(pro)q(cess)f (structure)g(can)g(b)q(e)h(exploited)g(b)o(y)f(the)g(system)f(in)i(the) 75 1902 y(assignmen)o(t)d(of)g(pro)q(cesses)h(to)f(hardw)o(are)f(pro)q (cessors,)h(if)h(this)g(helps)h(to)d(impro)o(v)o(e)i(the)f(comm)o (unication)75 1959 y(p)q(erformance)g(on)h(a)f(giv)o(en)h(mac)o(hine.) 20 b(Ho)o(w)12 b(this)i(mapping)g(is)g(done)g(in)g(detail)g(is)g (outside)g(the)g(scop)q(e)f(of)75 2015 y(this)k(standardization)h(prop) q(osal,)f(since)h(it)f(is)h(to)q(o)e(m)o(uc)o(h)h(dep)q(enden)o(t)i(on) d(the)h(particular)h(hardw)o(are.)75 2072 y(The)d(description)i(of)d(a) h(\\virtual)g(pro)q(cess)g(top)q(ology",)f(ho)o(w)o(ev)o(er,)g(only)h (dep)q(ends)i(on)e(the)g(application,)75 2128 y(and)k(is,)g(therefore,) g(mac)o(hine{indep)q(endent.)32 b(The)19 b(functions)g(whic)o(h)h(are)e (prop)q(osed)h(in)g(this)g(pap)q(er)75 2185 y(only)d(deal)g(with)f (this)h(side)g(of)f(the)g(mapping)h(problem.)166 2252 y(There)e(are)g(w)o(ell{kno)o(wn)h(tec)o(hniques)g(for)f(mapping)h (grid/torus)e(structures)h(to)g(hardw)o(are)f(top)q(o-)75 2308 y(logies)18 b(suc)o(h)f(as)g(h)o(yp)q(ercub)q(es)h(or)f(grids.)26 b(But)17 b(ev)o(en)g(for)g(more)f(complicated)j(graph)d(structures)h (go)q(o)q(d)75 2365 y(heuristics)i(often)e(yield)i(near{optimal)e (results)h([14)o(].)26 b(On)18 b(the)f(other)g(hand,)h(if)g(there)f(is) h(no)f(w)o(a)o(y)g(for)75 2421 y(the)c(user)g(to)f(sp)q(ecify)j(the)e (logical)h(pro)q(cess)f(arrangemen)o(t)f(as)g(a)h(\\virtual)g(top)q (ology",)f(this)i(information)75 2478 y(is)k(most)f(di\016cult)j(to)d (reco)o(v)o(er)g(later.)28 b(A)18 b(random)g(mapping)g(is)g(most)g(lik) o(ely)h(the)f(consequence.)29 b(On)75 2534 y(some)15 b(mac)o(hines)h(this)g(will)h(lead)f(to)f(unnecessary)h(con)o(ten)o (tion)g(in)g(the)f(in)o(terconnect)i(net)o(w)o(ork.)i(Some)75 2591 y(details)g(ab)q(out)e(predicted)i(and)f(measured)g(p)q (erformance)f(impro)o(v)o(emen)o(ts)h(whic)o(h)g(result)g(from)f(go)q (o)q(d)75 2647 y(pro)q(cess{to{pro)q(cessor)e(mapping)h(on)f(mo)q(dern) h(w)o(ormhole{routing)f(arc)o(hitectures)h(can)g(b)q(e)g(found)g(in)75 2704 y([5)o(,)f(4].)952 2828 y(53)p eop %%Page: 54 59 54 58 bop 75 -100 a Fs(54)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fm(4.2)70 b(Virtual)21 b(T)-6 b(op)r(ologies)75 159 y Fs(In)17 b(the)g(most)e(general)i(case)g (the)f(connection)i(structure)e(of)g(a)g(set)h(of)f(pro)q(cesses)h(can) f(b)q(e)h(represen)o(ted)75 215 y(b)o(y)c(a)g(graph.)19 b(The)14 b(no)q(des)f(stand)h(for)e(the)i(pro)q(cesses,)f(and)h(the)f (edges)h(connect)f(pro)q(cesses)h(whic)o(h)g(com-)75 272 y(m)o(unicate)h(with)g(eac)o(h)g(other.)20 b(MPI)14 b(pro)o(vides)i(message{passing)e(b)q(et)o(w)o(een)h(an)o(y)g(pair)g (of)f(pro)q(cesses)h(in)75 328 y(a)i(group.)27 b(There)18 b(is)g(no)g(requiremen)o(t)g(for)f(op)q(ening)i(a)e(c)o(hannel)i (explicitly)l(.)30 b(Therefore,)18 b(a)f(\\missing)75 385 y(link")g(in)f(the)f(user{de\014ned)i(pro)q(cess)f(graph)f(do)q(es) h(not)f(prev)o(en)o(t)g(the)g(corresp)q(onding)i(pro)q(cesses)f(from)75 441 y(exc)o(hanging)h(messages.)k(It)16 b(only)g(means)g(that)f(this)i (connection)g(is)f(neglected)h(in)g(the)f(virtual)h(top)q(ol-)75 498 y(ogy)l(.)24 b(As)16 b(a)g(consequence,)i(an)f(automatic)f(mapping) h(to)q(ol)g(will)h(not)e(try)g(to)g(assign)h(the)g(pro)q(cesses)g(to)75 554 y(neigh)o(b)q(oring)g(pro)q(cessors.)166 617 y(The)g(graph)g (de\014nition)i(function)f(MPI)p 868 617 14 2 v 16 w(GRAPH)f(presen)o (ted)h(in)g(this)f(pap)q(er)h(do)q(es)f(not)g(include)75 673 y(an)o(y)e(w)o(eigh)o(ting)g(of)f(the)h(comm)o(unication)h(v)o (olume)f(along)g(the)g(di\013eren)o(t)g(edges.)20 b(Pro)q(cesses)15 b(are)f(either)75 730 y(neigh)o(b)q(ors)22 b(or)e(not.)37 b(Exp)q(erience)23 b(with)f(similar)g(tec)o(hniques)g(in)g(P)l(ARMA)o (CS)f([11)o(,)g(12])f(sho)o(w)h(that)75 786 y(this)h(information)g(is)g (usually)h(su\016cien)o(t)g(for)e(a)g(go)q(o)q(d)h(mapping.)40 b(Additionally)l(,)25 b(a)d(more)f(precise)75 843 y(sp)q(eci\014cation) 16 b(w)o(ould)f(b)q(e)g(more)e(di\016cult)j(for)e(the)g(user)g(to)g (set)g(up,)g(and)h(it)f(w)o(ould)h(mak)o(e)e(the)i(in)o(terface)75 899 y(functions)h(more)f(complicated.)166 1045 y Fl(Discussion:)f Fk(One)f(could)f(also)f(ha)o(v)o(e)h(t)o(w)o(o)g(graph)g(mapping)d (functions,)j(one)h(of)e(whic)o(h)h(uses)h(comm)o(unica-)75 1101 y(tion)36 b(w)o(eigh)o(ting,)k(whereas)e(the)f(other)g(do)q(es)g (not.)85 b(The)37 b(mapping)d(function)i(with)h(w)o(eigh)o(ting,)75 1158 y(MPI)p 158 1158 13 2 v 15 w(MAP)p 270 1158 V 16 w(W)o(GRAPH,)10 b(w)o(ould)g(then)i(ha)o(v)o(e)f(an)g(additional)f (input)h(argumen)o(t)f Fg(weights)g Fk(with)h(the)g(same)g(struc-)75 1214 y(ture)k(as)f(the)h(neigh)o(b)q(or)f(list)g Fg(nglist)p Fk(.)j(On)e(the)g(higher)f(abstraction)g(lev)o(el)g(the)h(function)f (MPI)p 1587 1214 V 15 w(GRAPH)g(w)o(ould)75 1271 y(b)q(e)h(complemen)o (ted)d(b)o(y)h(MPI)p 545 1271 V 16 w(W)o(GRAPH)g(in)g(the)i(same)e(w)o (a)o(y)m(.)166 1416 y Fs(In)k(principle)i(a)c(function)i(for)f(the)g (de\014nition)i(of)e(suc)o(h)g(a)g(graph)g(structure)g(w)o(ould)g(b)q (e)h(su\016cien)o(t)75 1473 y(for)c(all)i(applications.)21 b(Ho)o(w)o(ev)o(er,)13 b(in)h(man)o(y)g(applications)h(the)f(graph)g (structure)f(is)h(v)o(ery)g(regular,)g(and)75 1529 y(the)e(detailed)i (set{up)f(of)e(the)i(graph)f(w)o(ould)h(b)q(e)g(incon)o(v)o(enien)o(t)g (for)f(the)g(user)h(and)f(resource{consuming)75 1586 y(during)g(run)g(time.)19 b(A)12 b(large)f(fraction)g(of)g(all)i (parallel)g(applications)g(use)f(pro)q(cess)g(top)q(ologies)f(lik)o(e)i (rings,)75 1642 y(or)g(t)o(w)o(o{)f(or)h(higher{dimensional)k(grids)c (or)h(tori.)19 b(These)14 b(structures)f(are)g(completely)i(de\014ned)g (b)o(y)f(the)75 1698 y(n)o(um)o(b)q(er)g(of)g(dimensions)i(and)e(the)g (n)o(um)o(b)q(ers)h(of)f(pro)q(cesses)g(in)h(eac)o(h)f(co)q(ordinate)h (direction.)21 b(Also,)14 b(the)75 1755 y(mapping)h(of)e(grids)h(and)h (tori)e(is)i(generally)g(an)f(easier)g(problem)h(then)f(that)f(of)h (general)g(graphs.)19 b(Th)o(us,)75 1811 y(it)14 b(is)h(desirable)h(to) d(do)h(the)h(mapping)g(as)e(grids)i(and)f(tori)g(rather)g(than)g (obscuring)h(the)f(information)g(in)75 1868 y(a)h(graph.)166 1931 y(Therefore,)f(in)h(addition)g(to)e(the)i(general)f(graph)g (cartesian)g(structures)g(of)g(arbitrary)f(dimension)75 1987 y(are)j(supp)q(orted.)22 b(The)16 b(top)q(ology)g(de\014nition)i (function)e(MPI)p 1140 1987 14 2 v 17 w(CAR)l(T)g(can)g(b)q(e)h(used)f (to)f(describ)q(e)j(b)q(oth)75 2044 y(grids)11 b(and)f(tori)g(b)o(y)g (sp)q(ecifying)j(p)q(er)d(co)q(ordinate)h(direction)h(whether)e(the)g (pro)q(cess)h(structure)f(is)h(p)q(erio)q(dic)75 2100 y(or)i(not.)20 b(Similar)15 b(functions)g(are)e(con)o(tained)i(in)g (EXPRESS)g([15)o(])e(and)h(P)l(ARMA)o(CS.)g(Note)g(that)f(an)h(n{)75 2156 y(dimensional)h(h)o(yp)q(ercub)q(e)g(is)f(an)f(n{dimensional)i (torus)e(with)g(2)g(pro)q(cesses)h(p)q(er)f(co)q(ordinate)h(direction.) 75 2213 y(Th)o(us,)h(sp)q(ecial)i(supp)q(ort)e(for)g(h)o(yp)q(ercub)q (e)h(structures)f(is)h(not)f(necessary)l(.)166 2276 y(Based)g(on)f(the) h(p)q(ositions)g(in)g(the)g(virtual)g(top)q(ology)f(a)g(pro)q(cess)h (can)g(lo)q(ok)g(up)g(the)f(PID)h(n)o(um)o(b)q(ers)75 2332 y(of)d(other)h(pro)q(cesses)g(b)o(y)f(calling)j(MPI)p 729 2332 V 16 w(INQPID.)e(The)g(pro)q(cesses)g(are)g(sp)q(eci\014ed)i (b)o(y)d(their)h(logical)h(no)q(de)75 2389 y(n)o(um)o(b)q(ers)20 b(\(graph\))e(or)h(rectangular)g(pro)q(cess)h(co)q(ordinates)g (\(grid/torus\).)31 b(Th)o(us,)20 b(it)g(is)f(p)q(ossible)j(to)75 2445 y(inquire)17 b(for)e(the)g(PID)g(n)o(um)o(b)q(ers)h(of)f(only)g (those)g(\(few\))g(pro)q(cesses)g(with)h(whic)o(h)g(messages)f(ha)o(v)o (e)g(to)f(b)q(e)75 2502 y(exc)o(hanged.)166 2647 y Fl(Discussion:)h Fk(In)e(the)h(PT2PT)g(discussion)g(it)f(is)g(not)g(y)o(et)h(decided)g (ho)o(w)f(pro)q(cesses)j(are)d(addressed.)20 b(This)75 2704 y(could,)10 b(for)g(example,)f(b)q(e)i(done)f(b)o(y)g(unique)g (iden)o(ti\014ers,)h(or)f(b)o(y)g(their)g(ranks)h(in)e(a)h(group.)17 b(The)10 b(top)q(ology)f(handling)p eop %%Page: 55 60 55 59 bop 75 -100 a Fq(4.3.)34 b(EMBEDDING)15 b(IN)g(MPI)1188 b Fs(55)75 45 y Fk(will)12 b(b)q(e)j(adapted)f(to)g(whatev)o(er)g (decision)h(will)d(b)q(e)i(tak)o(en)g(in)g(PT2PT.)75 270 y Fm(4.3)70 b(Em)n(b)r(edding)22 b(in)g(MPI)75 371 y Fs(The)17 b(supp)q(ort)g(for)f(virtual)i(top)q(ologies)f(has)f(to)h (b)q(e)g(fully)h(in)o(tegrated)f(in)g(the)g(framew)o(ork)f(of)g(MPI.)g (In)75 428 y(particular,)21 b(the)f(virtual)g(top)q(ology)f(functions)i (do)e(not)h(in)o(terfere)g(with)g(the)f(functions)i(for)e(pro)q(cess)75 484 y(group)i(creation)g(and)g(with)g(those)f(for)h(collectiv)o(e)h (comm)o(unication.)38 b(The)21 b(curren)o(t)g(design)g(of)g(the)75 541 y(pro)q(cess)15 b(top)q(ology)g(functions)h(is)g(based)g(on)f(the)g (follo)o(wing)h(assumptions:)143 629 y Fo(\017)23 b Fs(MPI)15 b(is)h(free)g(to)f(c)o(ho)q(ose)h(an)o(y)f(pro)q(cess)h(in)h(a)e(group) g(for)g(an)o(y)h(lo)q(cation)g(in)h(the)f(top)q(ology)l(.)21 b(There)189 685 y(are)16 b(no)g(sp)q(ecial)i(pro)q(cesses)f(whic)o(h)g (ha)o(v)o(e)f(to)g(b)q(e)h(aligned)g(with)g(top)q(ology)f (substructures.)23 b(This)189 742 y(assumption)d(\014ts)g(b)q(est)h(to) e(a)h(\\lo)q(cal)h(SPMD)f(mo)q(del")h(in)g(whic)o(h)g(all)g(pro)q (cesses)g(in)g(the)f(same)189 798 y(group)15 b(execute)g(the)h(same)f (program.)143 890 y Fo(\017)23 b Fs(One)g(imp)q(ortan)o(t)f(feature)g (of)g(MPI)g(are)g(the)h(pro)q(cess)f(groups.)42 b(The)22 b(treatmen)o(t)f(of)h(virtual)189 946 y(top)q(ologies)f(has)g(to)g(tak) o(e)g(this)g(concept)h(in)o(to)f(accoun)o(t.)38 b(The)22 b(functions)g(presen)o(ted)f(in)i(this)189 1003 y(pap)q(er)16 b(are)g(not)g(only)h(compatible)h(with)e(the)h(pro)q(cess)f(groups,)g (they)g(ev)o(en)h(tak)o(e)f(adv)m(an)o(tage)f(of)189 1059 y(them)e(for)g(the)g(de\014nition)i(of)e(mapping)h(o)o(v)o(erla)o (ys)f(and)g(dynamic)i(remapping)f(during)g(execution)189 1116 y(time.)42 b(Virtual)23 b(pro)q(cess)g(top)q(ologies)g(help)h(in)f (the)g(sub)q(division)i(of)d(pro)q(cess)h(groups)f(whic)o(h)189 1172 y(in)f(man)o(y)e(applications)j(has)e(to)f(b)q(e)i(done)g(along)f (pro)q(cess)g(co)q(ordinate)h(lines.)36 b(F)l(urthermore,)189 1228 y(the)14 b(de\014nition)i(of)e(collectiv)o(e)i(gather{t)o(yp)q(e)d (comm)o(unication)i(calls)g(b)q(ene\014ts)g(from)f(the)g(natural)189 1285 y(ordering)h(of)g(pro)q(cesses)h(in)g(a)e(logical)j(top)q(ology)l (.)75 1373 y(The)g(virtual)g(top)q(ology)g(handling)h(follo)o(ws)f(the) g(general)g(MPI)g(rule)g(of)g(functional)h(la)o(y)o(ering.)25 b(A)o(t)16 b(the)75 1429 y(b)q(ottom)f(it)g(is)h(based)g(on)f(the)h (general)g(pro)q(cess)g(group)f(creation)g(function)i(MPI)p 1486 1429 14 2 v 16 w(CREA)l(TE)p 1693 1429 V 17 w(GR)o(OUP)75 1486 y(plus)i(a)f(few)g(top)q(ology)g(sp)q(eci\014c)i(mapping)e (functions.)30 b(In)18 b(principle)j(this)e(is)f(su\016cien)o(t)h(for)e (de\014ning)75 1542 y(top)q(ologies)f(and)f(use)h(them)f(in)h (applications.)166 1599 y(A)f(n)o(um)o(b)q(er)g(of)f(higher{lev)o(el)j (functions)e(pro)o(vide)g(an)g(in)o(terface)g(whic)o(h)h(is)f(similar)h (to)e(what)g(users)75 1655 y(of)19 b(pac)o(k)m(ages)g(lik)o(e)h (Express)f(and)g(P)l(ARMA)o(CS)h(are)f(familiar)h(with.)32 b(This)19 b(lev)o(el)i(can)e(b)q(e)h(written)f(in)75 1712 y(a)j(p)q(ortable)h(w)o(a)o(y)e(in)i(terms)f(of)g(the)g(lo)o (w{lev)o(el)h(functions.)42 b(Implemen)o(tations)24 b(of)e(the)g (high{lev)o(el)75 1768 y(functions)e(whic)o(h)h(b)o(ypass)e(the)g(lo)o (w)h(lev)o(el)h(ma)o(y)d(b)q(e)j(adv)m(an)o(tageous)d(on)i(some)f (systems)g(in)h(terms)f(of)75 1825 y(memory)c(and)g(cpu)h(time)f (usage,)g(but)g(this)h(is)g(not)f(a)f(requiremen)o(t.)75 1967 y Fm(4.4)70 b(Ov)n(erview)21 b(of)i(the)f(prop)r(osed)j(MPI)e (functions)75 2068 y Fs(This)d(c)o(hapter)f(assumes)f(the)i(a)o(v)m (ailabilit)o(y)h(of)d(the)i(follo)o(wing)f(MPI)g(features)g(whic)o(h)h (are)f(de\014ned)h(in)75 2125 y(other)15 b(c)o(hapters:)131 2213 y(1.)22 b(An)11 b(existing)g(pro)q(cess)g(group)f(\(e.g.,)g(the)h (ALL)h(group\))e(can)g(b)q(e)i(partitioned)f(and/or)f(ren)o(um)o(b)q (ered)189 2269 y(using)189 2379 y Ft(MPI)p 298 2379 16 2 v 18 w(CREA)l(TE)p 533 2379 V 19 w(GR)o(OUP\(gid)p 835 2379 V 20 w(old,)17 b(gid)p 1025 2379 V 20 w(new,)f(group)p 1299 2379 V 19 w(k)o(ey)l(,)f(order)p 1546 2379 V 18 w(k)o(ey\))g Fs(with)189 2506 y Ft(IN)i(gid)p 339 2506 V 19 w(old)24 b Fs(group)15 b(id)h(of)e(paren)o(t)h(group,)189 2576 y Ft(OUT)j(gid)p 394 2576 V 19 w(new)23 b Fs(group)15 b(id)h(of)f(newly)h(created)f(subgroup)g(of)g(whic)o(h)h(the)f(calling) i(pro)q(cess)f(is)f(a)289 2633 y(mem)o(b)q(er,)189 2704 y Ft(IN)i(group)p 402 2704 V 18 w(k)o(ey)22 b Fs(in)o(teger)15 b(v)m(alue)i(used)e(for)g(the)g(partitioning,)p eop %%Page: 56 61 56 60 bop 75 -100 a Fs(56)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)189 45 y Ft(IN)i(order)p 393 45 16 2 v 18 w(k)o(ey)k Fs(in)o(teger)e(v)m(alue)i(used)e(for)g (determining)h(the)f(ranks)g(in)h(the)f(new)g(group)g(\(see)289 102 y(b)q(elo)o(w\),)189 193 y(This)j(function)g(is)f(called)i(b)o(y)e (all)i(pro)q(cesses)e(in)i(group)d Fi(gid)p 1265 193 14 2 v 17 w(old)i Fs(lo)q(osely)g(sync)o(hronously)l(.)39 b(It)189 250 y(creates)18 b(as)g(man)o(y)g(di\013eren)o(t)g(subgroups)h (as)f(there)h(are)f(di\013eren)o(t)g(v)m(alues)i(of)e Fi(gr)n(oup)p 1676 250 V 17 w(key)p Fs(.)30 b(The)189 306 y(ranking)16 b(in)h(the)g(newly)g(created)f(groups)g(is)h (determined)h(b)o(y)e(the)h Fi(or)n(der)p 1464 306 V 17 w(key)f Fs(v)m(alues,)h(with)g(the)189 363 y(pro)q(cess)e(ranks)g (in)h(the)f(paren)o(t)g(group)g(for)g(tie)g(breaking.)131 455 y(2.)22 b(There)14 b(is)g(an)g(in)o(ternal)h(information)f(cac)o (heing)h(facilit)o(y)l(,)g(whic)o(h)g(can)f(b)q(e)g(used)h(b)o(y)f(MPI) g(routines)189 511 y(to)c(store)h(and)g(retriev)o(e)g(information)h (related)f(to)g(a)g(pro)q(cess)g(group.)18 b(This)12 b(facilit)o(y)g(w)o(as)f(prop)q(osed)189 568 y(in)19 b(the)f(Con)o(text)f(Sub)q(committee.)30 b(High{lev)o(el)20 b(top)q(ology)e(functions)h(use)g(this)f(p)q(ossibilit)o(y)j(to)189 624 y(store)14 b(structure)h(information)h(for)f(a)g(pro)q(cess)g (group,)g(whic)o(h)h(is)g(then)g(later)f(used)h(b)o(y)f(top)q(ology)189 680 y(inquiry)21 b(functions.)35 b(Also,)22 b(sp)q(ecial)g(shift)e (functions)h(whic)o(h)g(are)f(based)g(on)g(the)g(top)q(ological)189 737 y(structure)15 b(can)g(tak)o(e)f(adv)m(an)o(tage)h(of)g(this)g (information.)75 908 y Fl(Discussion:)g Fk(If)e(the)h(information)d (cac)o(heing)j(will)d(not)j(b)q(e)g(part)f(of)g(MPI,)g(passing)h(the)g (top)q(ology)e(information)75 965 y(for)g(a)h(group)f(from)f(one)i(top) q(ology)f(function)g(to)h(the)g(other)h(can)f(b)q(e)g(handled)g(in)f (sev)o(eral)h(other)g(w)o(a)o(ys)g(whic)o(h)f(will)75 1021 y(b)q(e)j(discussed)g(later.)166 1161 y Fs(There)i(has)g(b)q(een)h (a)e(general)i(decision)g(in)g(MPI)f(to)f(start)f(all)j(n)o(um)o(b)q (ering)g(with)f(0)g(instead)g(of)g(1.)75 1217 y(Th)o(us,)f(co)q (ordinates)h(in)h(a)e(cartesian)g(structure)h(are)f(0,1,)f(...,)h (dim-1.)24 b(In)17 b(the)g(same)f(w)o(a)o(y)l(,)g(no)q(des)h(in)g(a)75 1273 y(general)f(graph)f(are)f(n)o(um)o(b)q(ered)i(zero)g(up.)75 1394 y Fj(4.4.1)55 b(Lo)n(w{lev)n(el)18 b(top)r(ology)f(functions)75 1480 y Fs(On)k(the)g(lo)o(w)o(est)e(lev)o(el)j(groups)e(are)g(created)h (b)o(y)f(the)h(general)g(function)g(MPI)p 1477 1480 V 16 w(CREA)l(TE)p 1684 1480 V 17 w(GR)o(OUP)l(.)75 1537 y(Using)13 b(the)g(input)g(argumen)o(t)f Fi(newr)n(ank)g Fs(the)g(caller)i(can)e(c)o(hange)h(the)g(pro)q(cess)f(ranks)h(in)g (the)f(new)h(group)75 1593 y(as)h(compared)g(to)g(the)g(paren)o(t)g (group.)19 b(This)c(feature)f(can)h(b)q(e)g(used)g(to)e(e\016cien)o (tly)j(map)e(the)h(pro)q(cesses)75 1649 y(in)i(the)e(group)h(to)f(the)h (p)q(ositions)h(in)f(the)g(top)q(ology)l(.)21 b(F)l(or)15 b(eac)o(h)h(class)g(of)g(top)q(ologies,)g(therefore,)f(there)75 1706 y(is)f(a)f(function)h(whic)o(h)h(computes)e(the)h(rank)f(of)g(the) h(calling)h(pro)q(cess)f(in)g(the)g(new)f(group,)h(based)f(on)h(the)75 1762 y(size)i(and)g(structure)g(of)f(the)g(top)q(ology)l(.)21 b(Note)16 b(that)e(it)i(is)g(a)g(v)m(alid)h(implemen)o(tation)g(if)f (the)f(new)h(ranks)75 1819 y(are)f(iden)o(tical)j(to)c(the)i(old)g (ranks.)k(So,)c(if)f(for)g(a)h(giv)o(en)g(mac)o(hine)g(the)f(placemen)o (t)i(of)e(pro)q(cesses)h(mak)o(es)75 1875 y(no)f(di\013erence,)h(the)f (implemen)o(tation)i(of)e(the)g(rank)g(computation)g(functions)h(is)g (trivial.)166 1932 y(F)l(or)e(cartesian)i(structures)f(the)g(function)h (is)166 2024 y Ft(newrank)h(=)g(MPI)p 545 2024 16 2 v 18 w(MAP)p 689 2024 V 17 w(CAR)l(T\(gid)p 944 2024 V 20 w(old,)g(ndims,)g(dims,)g(p)q(erio)q(d\))f Fs(with)75 2148 y Ft(IN)h(gid)p 225 2148 V 20 w(old)23 b Fs(group)15 b(id)h(of)f(paren)o(t)g(group,)75 2240 y Ft(IN)i(ndims)23 b Fs(n)o(um)o(b)q(er)15 b(of)g(dimensions)i(of)e(cartesian)g (structure,)75 2332 y Ft(IN)i(dims\(*\))23 b Fs(n)o(um)o(b)q(er)16 b(of)f(pro)q(cesses)g(in)h(eac)o(h)g(co)q(ordinate)f(direction,)75 2423 y Ft(IN)i(p)q(erio)q(d\(*\))25 b Fs(p)q(erio)q(dicit)o(y)17 b(sp)q(eci\014cation)g(in)f(eac)o(h)g(co)q(ordinate)f(direction.)75 2512 y(The)g(return)h(v)m(alue)g(is)g(the)f(reordered)g(rank)g(of)g (the)g(calling)i(pro)q(cess.)166 2704 y(The)e(corresp)q(onding)i (function)f(for)e(general)i(graph)f(structures)g(is)p eop %%Page: 57 62 57 61 bop 75 -100 a Fq(4.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b Fs(57)166 45 y Ft(newrank)17 b(=)g(MPI)p 545 45 16 2 v 18 w(MAP)p 689 45 V 17 w(GRAPH\(gid)p 992 45 V 20 w(old,)g(no)q(des,)h(index,)f (nglist\))f Fs(with)75 164 y Ft(IN)h(gid)p 225 164 V 20 w(old)23 b Fs(group)15 b(id)h(of)f(paren)o(t)g(group,)75 253 y Ft(IN)i(no)q(des)23 b Fs(n)o(um)o(b)q(er)16 b(of)e(graph)h(no)q (des,)75 343 y Ft(IN)i(index\(*\),)h(nglist\(*\))25 b Fs(de\014nition)13 b(of)e(graph)g(structure:)17 b(F)l(or)11 b(eac)o(h)g(graph)g(no)q(de)h(its)g(set)f(of)g(neigh-)189 399 y(b)q(ors)i(is)h(en)o(tered,)g(i.)f(e.,)g(the)h(set)f(of)g(pro)q (cesses)h(with)f(whic)o(h)i(the)e(no)q(de)h(pro)q(cess)g(is)g(going)f (to)g(com-)189 455 y(m)o(unicate)k(most)f(frequen)o(tly)l(.)25 b(Since)19 b(there)d(is)i(no)e(w)o(eigh)o(ting)h(of)g(no)q(de)g (connections,)h(it)f(migh)o(t)189 512 y(b)q(e)d(sensible)j(to)c(sp)q (ecify)i(only)g(those)f(neigh)o(b)q(ors)h(with)f(whic)o(h)h(there)f(is) h(a)f(considerable)i(amoun)o(t)189 568 y(of)e(comm)o(unication.)189 641 y(F)l(or)20 b(eac)o(h)h(no)q(de)h Fr(i)35 b Fs(\(0)22 b Fo(\024)h Fr(i)e Fo(\024)i Fr(nodes)14 b Fo(\000)h Fs(1\))20 b(the)h(list)h(of)e(the)h(neigh)o(b)q(or)h(no)q(des)g(is)f (stored)g(at)189 698 y Fr(ng)r(l)q(ist)p Fs(\()p Fr(j)s Fs(\))p Fr(;)30 b(index)p Fs(\()p Fr(i)p Fs(\))13 b Fo(\024)i Fr(j)i Fo(\024)e Fr(index)p Fs(\()p Fr(i)10 b Fs(+)h(1\))f Fo(\000)h Fs(1.)23 b(F)l(or)16 b(the)g(last)g(pro)q(cess)h Fr(index)p Fs(\()p Fr(i)10 b Fs(+)h(1\))16 b(p)q(oin)o(ts)189 754 y(to)g(the)g(\014rst)g(free)h(p)q(osition)h(in)f(v)o(ector)f (nglist.)25 b(Note)16 b(that)g(the)g(matrix)h(is)g(symmetric,)f(i.)h (e.,)f(if)189 811 y(no)q(de)f Fi(i)f Fs(is)h(a)f(neigh)o(b)q(or)h(of)f (no)q(de)h Fi(j)p Fs(,)f(than)g(no)q(de)h Fi(j)f Fs(is)h(also)f(a)g (neigh)o(b)q(or)h(of)f(no)q(de)h Fi(i)p Fs(.)k(This)c(prop)q(ert)o(y) 189 867 y(can)g(b)q(e)h(used)g(for)e(an)h(in)o(ternal)h(c)o(hec)o(king) g(of)f(the)h(input)g(argumen)o(ts.)75 950 y(Again,)f(the)g(function)i (returns)e(the)g(reordered)g(rank)g(of)g(calling)i(pro)q(cess.)75 1046 y(The)11 b(de\014nitions)h(of)e(the)h(argumen)o(ts)f Fh(nodes)p Fs(,)g Fh(index)p Fs(,)h(and)g Fh(nglist)e Fs(are)i(illustrated)h(with)f(the)f(follo)o(wing)75 1102 y(simple)17 b(example:)166 1159 y(Assume)e(there)h(are)f(four)f(pro)q (cesses)i(0,)e(1,)h(2,)f(3)h(with)h(the)f(follo)o(wing)h(adjacency)g (matrix:)100 1211 y(pro)q(cess)50 b(neigh)o(b)q(ors)100 1268 y(0)170 b(1,)15 b(3)100 1324 y(1)170 b(0)100 1381 y(2)g(3)100 1437 y(3)g(0,)15 b(2)166 1490 y(Then,)g(the)h(input)g (argumen)o(ts)e(are:)100 1542 y(no)q(des)i(=)50 b(4)100 1599 y(index)16 b(=)56 b(0,)14 b(2,)h(3,)f(4,)h(6)100 1655 y(nglist)h(=)54 b(1,)14 b(3,)h(0,)f(3,)h(0,)f(2)166 1708 y(If)i(the)g(top)q(ology)g(con)o(tains)g(few)o(er)f(pro)q(cesses)i (than)e(there)h(are)g(pro)q(cesses)g(in)h(the)f(paren)o(t)g(group,)75 1764 y(the)g(newrank)g(v)m(alues)h(for)e(the)g(remaining)i(pro)q (cesses)f(are)g(set)f(to)h(MPI)p 1329 1764 14 2 v 16 w(NORANK.)h(It)e(is)i(an)e(error)g(if)75 1821 y(the)g(top)q(ology)g (de\014nes)h(more)f(pro)q(cesses)h(than)f(are)g(con)o(tained)h(in)g (the)f(paren)o(t)g(group.)75 1940 y Fj(4.4.2)55 b(High{lev)n(el)17 b(top)r(ology)g(functions)75 2026 y Fs(In)i(principle,)i(all)e(that)e (is)i(needed)g(to)f(create)g(an)g(MPI)g(pro)q(cess)g(group)g(with)h (top)q(ological)g(ordering)75 2083 y(are)13 b(the)h(t)o(w)o(o)f (mapping)h(functions)g(and)g(the)g(general)g(group)g(creation)f (routine.)20 b(Ho)o(w)o(ev)o(er,)13 b(this)h(is)g(not)75 2139 y(what)19 b(the)g(applications)i(programmer)d(exp)q(ects)i(to)f (see)h(on)f(his)h(lev)o(el)h(of)e(abstraction.)31 b(Therefore,)75 2195 y(another)13 b(la)o(y)o(er)h(of)f(functions)h(is)g(de\014ned)i(on) d(top)g(of)h(the)f(lo)o(w{lev)o(el)i(functions.)20 b(This)14 b(higher)h(lev)o(el)g(can)75 2252 y(b)q(e)g(p)q(ortably)f(implemen)o (ted)i(in)f(terms)f(of)g(the)g(lev)o(el)i(b)q(elo)o(w.)k(Of)14 b(course)h(MPI)f(implemen)o(tors)h(are)f(free)75 2308 y(to)h(b)o(ypass)g(the)g(lo)o(w)o(er)g(lev)o(el)h(for)f(the)g(sak)o(e)g (of)g(b)q(etter)g(p)q(erformance)g(or)g(resource)g(usage.)166 2365 y(F)l(or)e(cartesian)h(and)g(graph)f(structures)g(the)h(functions) h(MPI)p 1226 2365 V 16 w(CAR)l(T)f(and)g(MPI)p 1561 2365 V 16 w(GRAPH)g(create)75 2421 y(a)g(pro)q(cess)g(group)g(with)h(top)q (ological)g(structure.)k(They)14 b(com)o(bine)h(the)f(call)i(to)d(the)h (mapping)h(function)75 2478 y(with)f(the)g(call)h(to)e(the)g(group)h (creation)g(routine.)20 b(The)14 b(user)f(can)h(decide)i(whether)e(the) f(ranking)h(in)h(the)75 2534 y(new)i(group)g(ma)o(y)f(b)q(e)i (reordered)f(\(for)f(a)g(more)h(e\016cien)o(t)g(mapping\))h(or)e(not.) 25 b(In)17 b(the)g(latter)g(case)g(the)75 2591 y(rank)e(of)g(the)g (calling)i(pro)q(cess)e(is)h(copied)g(from)f(the)g(paren)o(t)g(group.) 166 2647 y(The)h(option)g(of)f(not)g(reordering)h(the)g(pro)q(cess)f (ranks)h(can)f(b)q(e)i(useful)f(in)h(the)e(case)h(of)f(an)h(o)o(v)o (erla)o(y)75 2704 y(of)f(di\013eren)o(t)h(top)q(ologies.)21 b(The)16 b(user)f(can)h(then)g(fully)h(con)o(trol)e(whic)o(h)h(lo)q (cation)g(in)h(the)e(\014rst)g(top)q(ology)p eop %%Page: 58 63 58 62 bop 75 -100 a Fs(58)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fs(coincides)i(with)e(whic)o(h)h (lo)q(cation)g(in)f(the)g(second)h(one.)k(The)15 b(price)h(whic)o(h)f (one)h(ob)o(viously)f(has)g(to)f(pa)o(y)75 102 y(in)i(this)g(case)f(is) h(that)e(for)h(only)g(one)h(top)q(ology)f(the)g(pro)q(cess)g (arrangemen)o(t)f(can)i(b)q(e)g(optimized.)166 158 y(As)11 b(sp)q(eci\014ed)j(for)d(the)h(lo)o(w{lev)o(el)g(functions)h(it)f(is)g (an)f(error)g(if)h(one)g(attempts)e(to)h(de\014ne)i(a)e(top)q(ology)75 215 y(with)j(more)g(lo)q(cations)g(than)g(there)g(are)g(pro)q(cesses)g (in)h(the)f(paren)o(t)f(pro)q(cess)h(group.)20 b(If)14 b(it)g(do)q(es)g(not)g(use)75 271 y(all)j(a)o(v)m(ailable)h(pro)q (cesses,)f(the)f(un)o(used)h(pro)q(cess)f(will)i(return)e(MPI)p 1249 271 14 2 v 17 w(NORANK)h(as)f(their)h(rank)f(in)h(the)75 327 y(new)e(group.)166 384 y(MPI)p 258 384 V 16 w(CAR)l(T)h(is)g (de\014ned)g(as)166 476 y Ft(newrank)c(=)g(MPI)p 535 476 16 2 v 18 w(CAR)l(T\(gid)p 791 476 V 19 w(old,)i(gid)p 977 476 V 19 w(cart,)g(ndims,)f(dims,)g(p)q(erio)q(d,)h(reorder\))c Fs(with)75 605 y Ft(IN)17 b(gid)p 225 605 V 20 w(old)23 b Fs(group)15 b(id)h(of)f(paren)o(t)g(group,)75 699 y Ft(OUT)j(gid)p 280 699 V 20 w(cart)23 b Fs(group)17 b(id)h(of)f (cartesian)g(group)g(\(if)g(calling)i(pro)q(cess)e(is)h(part)e(of)h (it,)h(else)g(id)g(of)e(rest)189 756 y(group\),)75 850 y Ft(IN)h(ndims,)g(dims\(*\),)g(p)q(erio)q(d\(*\))25 b Fs(de\014nition)17 b(of)d(cartesian)i(structure,)e(\(for)g(details)i (see)f(the)h(def-)189 906 y(inition)h(of)d(MPI)p 476 906 14 2 v 17 w(MAP)p 600 906 V 16 w(CAR)l(T\))75 1000 y Ft(IN)j(reorder)22 b Fs(ranking)15 b(ma)o(y)g(b)q(e)g(reordered)h(or) f(not.)75 1094 y(The)10 b(return)g(v)m(alue)i(is)f(the)f(rank)g(of)f (the)i(calling)g(pro)q(cess)g(in)g(the)f(cartesian)g(group)g(\(or)f (MPI)p 1628 1094 V 17 w(NORANK\).)166 1151 y(Apart)15 b(from)f(some)h(lo)q(cal)h(computations,)f(MPI)p 1015 1151 V 16 w(CAR)l(T)h(is)g(based)f(on)g(the)h(t)o(w)o(o)d(function)j (calls)166 1243 y Ft(newrank)h(=)g(MPI)p 545 1243 16 2 v 18 w(MAP)p 689 1243 V 17 w(CAR)l(T)g(\(gid)p 961 1243 V 20 w(old,)h(ndims,)f(dims,)f(p)q(erio)q(d\))166 1335 y Fs(and)166 1427 y Ft(MPI)p 275 1427 V 18 w(CREA)l(TE)p 510 1427 V 19 w(GR)o(OUP)h(\(gid)p 829 1427 V 20 w(old,)h(gid)p 1020 1427 V 19 w(cart,)g(0,)f(newrank\))e Fs(.)166 1577 y(F)l(or)f(graph)h(structures)g(the)h(corresp)q(onding)g(routine)f(is) 166 1669 y Ft(newrank)h(=)g(MPI)p 543 1669 V 17 w(GRAPH\(gid)p 846 1669 V 20 w(old,)h(gid)p 1036 1669 V 19 w(graph,)f(no)q(des,)h (index,)f(nglist,)h(reorder\))75 1726 y Fs(with)75 1855 y Ft(IN)g(gid)p 225 1855 V 20 w(old)23 b Fs(group)15 b(id)h(of)f(paren)o(t)g(group,)75 1949 y Ft(OUT)j(gid)p 280 1949 V 20 w(graph)23 b Fs(group)18 b(id)h(of)g(graph)f(group)g (\(if)h(calling)i(pro)q(cess)d(is)i(part)e(of)g(it,)h(else)h(id)f(of)f (rest)189 2005 y(group\),)75 2099 y Ft(IN)f(no)q(des,)g(index\(*\),)h (nglist\(*\))25 b Fs(de\014nition)12 b(of)e(graph)g(structure,)h(\(for) e(details)i(see)g(the)f(de\014nition)189 2156 y(of)k(MPI)p 332 2156 14 2 v 17 w(MAP)p 456 2156 V 16 w(GRAPH\))75 2250 y Ft(IN)j(reorder)22 b Fs(ranking)15 b(ma)o(y)g(b)q(e)g(reordered) h(or)f(not.)166 2403 y(In)f(the)g(con)o(text)f(sub)q(committee)h(a)g (mec)o(hanism)g(has)f(b)q(een)i(prop)q(osed)f(for)f(cac)o(heing)i (information)75 2459 y(whic)o(h)f(is)g(asso)q(ciated)g(with)g(a)f(pro)q (cess)h(group.)19 b(If)14 b(that)e(feature)i(will)h(b)q(e)f(part)f(of)g (MPI,)g(it)h(can)f(b)q(e)h(used)75 2516 y(b)o(y)h(the)g(high{lev)o(el)j (top)q(ology)c(functions)i(to)f(store)f(the)i(follo)o(wing)g (information:)131 2610 y(1.)22 b(T)o(yp)q(e)15 b(of)g(top)q(ology)g (\(cartesian/graph\))131 2704 y(2.)22 b(F)l(or)14 b(a)h(cartesian)g (top)q(ology:)p eop %%Page: 59 64 59 63 bop 75 -100 a Fq(4.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b Fs(59)243 45 y Fo(\017)23 b Fs(ndims)16 b(\(n)o(um)o(b)q(er)f(of)g(dimensions\),) 243 121 y Fo(\017)23 b Fs(dims\(*\))15 b(\(n)o(um)o(b)q(ers)g(of)f(pro) q(cesses)i(p)q(er)g(co)q(ordinate)f(direction\),)243 197 y Fo(\017)23 b Fs(p)q(erio)q(d\(*\))15 b(\(p)q(erio)q(dicit)o(y)i (information\),)243 272 y Fo(\017)23 b Fs(o)o(wn)p 372 272 14 2 v 16 w(p)q(osition\(*\))c(\(o)o(wn)f(p)q(osition)i(in)g(grid,) h(could)f(also)f(b)q(e)h(computed)f(from)g(rank)g(and)289 329 y(dims\(*\)\))75 428 y(F)l(or)d(a)g(graph)g(structure)g(no)g (information)h(has)f(to)g(b)q(e)h(stored,)f(b)q(ecause)h(the)g(graph)f (no)q(de)h(n)o(um)o(b)q(er)f(is)75 484 y(the)h(rank)f(in)i(the)f(graph) f(group,)h(and)g(the)g(size)g(of)g(the)g(graph)f(is)i(the)e(n)o(um)o(b) q(er)i(of)e(pro)q(cesses)h(in)h(that)75 541 y(group.)166 599 y(In)12 b(case)f(the)g(cac)o(heing)h(mec)o(hanism)f(do)q(es)h(not)e (b)q(ecome)i(part)f(of)f(MPI,)h(there)g(are)g(t)o(w)o(o)e(other)i(p)q (ossi-)75 655 y(bilities:)21 b(since)13 b(the)e(storage)g(and)h (retriev)m(al)h(of)e(the)h(ab)q(o)o(v)o(e)f(information)h(is)g(only)h (done)f(b)o(y)f(the)h(top)q(ology)75 711 y(functions,)17 b(it)g(can)f(in)o(ternally)i(b)q(e)f(stored)f(with)g(other)g(group)g (information)h(in)g(an)f(implemen)o(tation{)75 768 y(dep)q(enden)o(t)h (w)o(a)o(y)l(,)d(whic)o(h)i(eac)o(h)f(v)o(endor)g(can)g(do)h(as)e(is)i (most)e(con)o(v)o(enien)o(t.)166 826 y(As)k(an)h(alternativ)o(e,)g(the) g(information)f(could)i(b)q(e)f(stored)f(in)h(an)g(opaque)f(ob)s(ject,) h(a)f(handle)i(to)75 882 y(whic)o(h)g(w)o(ould)g(ha)o(v)o(e)f(to)g(b)q (e)h(included)h(in)g(the)e(argumen)o(t)g(lists)h(of)f(all)h(related)g (top)q(ology)f(functions.)75 939 y(Ho)o(w)o(ev)o(er,)14 b(this)h(alternativ)o(e)h(w)o(ould)f(b)q(e)h(m)o(uc)o(h)g(less)f (elegan)o(t.)75 1068 y Fj(4.4.3)55 b(T)-5 b(op)r(ology)18 b(inquiry)f(functions)75 1156 y Fs(If)j(a)g(top)q(ology)g(has)f(b)q (een)j(de\014ned)f(with)f(one)g(of)g(the)g(ab)q(o)o(v)o(e)g(high{lev)o (el)i(functions,)f(the)f(top)q(ology)75 1213 y(information)c(can)h(b)q (e)f(lo)q(ok)o(ed)h(up)g(using)f(inquiry)i(functions.)23 b(They)17 b(all)g(are)f(lo)q(cal)h(calls,)g(and)f(do)g(not)75 1269 y(require)g(an)o(y)g(comm)o(unication.)22 b(In)16 b(the)g(follo)o(wing)g(the)g(assumption)f(is)i(made)e(that)g(the)h (information)75 1326 y(cac)o(heing)21 b(is)f(a)o(v)m(ailable.)36 b(Otherwise)21 b(the)f(handle)h(to)e(the)h(opaque)g(information)g(ob)s (ject)f(has)h(to)f(b)q(e)75 1382 y(added)d(to)e(the)i(argumen)o(t)e (lists)i(\(or)e(replace)j(the)e(group)g(iden)o(ti\014er)i(argumen)o (t\).)166 1440 y(The)c(function)h(MPI)p 525 1440 V 17 w(INQMAP)f(returns)g(the)g(t)o(yp)q(e)h(of)e(mapping)i(whic)o(h)g(is)g (assigned)f(to)g(a)g(group.)166 1533 y Ft(map)k(=)g(MPI)p 448 1533 16 2 v 18 w(INQMAP\(gid\).)166 1626 y Fs(The)e(return)g(v)m (alue)i(is)f(one)f(of)g(the)g(follo)o(wing:)143 1725 y Fo(\017)23 b Fs(MPI)p 281 1725 14 2 v 16 w(CAR)l(TTOP:)15 b(cartesian)h(top)q(ology)l(,)143 1825 y Fo(\017)23 b Fs(MPI)p 281 1825 V 16 w(GRAPHTOP:)15 b(graph)g(top)q(ology)l(,)143 1924 y Fo(\017)23 b Fs(MPI)p 281 1924 V 16 w(NOTOP:)15 b(no)g(top)q(ology)l(.)166 2023 y(F)l(or)j(a)g(pro)q(cess)g(group)h (with)f(cartesian)h(ordering)g(whic)o(h)g(has)f(b)q(een)i(created)e(b)o (y)h(MPI)p 1720 2023 V 16 w(CAR)l(T,)75 2079 y(MPI)p 167 2079 V 16 w(INQCAR)l(T)e(lo)q(oks)e(up)h(the)f(follo)o(wing)h (info:)166 2173 y Ft(dim)h(=)h(MPI)p 438 2173 16 2 v 18 w(INQCAR)l(T\(gid,)f(maxdim,)g(size,)g(p)q(erio)q(d,)h(p)q(os\))e Fs(with)75 2307 y Ft(IN)h(gid)24 b Fs(group)14 b(id)j(of)d(group)h (with)h(cartesian)f(structure,)75 2406 y Ft(IN)i(maxdim)23 b Fs(length)15 b(of)g(v)o(ectors)g(size\(*\),)f(p)q(erio)q(d\(*\),)i (and)f(p)q(os\(*\))f(in)j(the)e(calling)i(program,)75 2505 y Ft(OUT)h(size\(*\))24 b Fs(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(for)f(eac)o(h)g(co)q(ordinate)g(direction,)75 2604 y Ft(OUT)j(p)q(erio)q(d\(*\))25 b Fs(p)q(erio)q(dicit)o(y)17 b(y)o(es/no)e(for)g(eac)o(h)g(co)q(ordinate)g(direction,)75 2704 y Ft(OUT)j(p)q(os\(*\))24 b Fs(p)q(osition)16 b(of)f(calling)i (pro)q(cess)e(in)h(cartesian)g(structure.)p eop %%Page: 60 65 60 64 bop 75 -100 a Fs(60)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fs(The)g(return)h(v)m(alue)g(is)g (the)f(dimension)i(of)e(the)g(cartesian)g(structure.)166 102 y(F)l(or)e(a)g(pro)q(cess)h(group)g(with)g(cartesian)f(structure,)h (the)f(function)i(MPI)p 1411 102 14 2 v 16 w(INQPID)g(translates)e(the) 75 158 y(logical)j(pro)q(cess)g(co)q(ordinates)f(to)g(pro)q(cess)g(ids) h(as)f(they)g(are)g(used)h(b)o(y)f(the)h(p)q(oin)o(t{to{p)q(oin)o(t)f (routines:)166 250 y Ft(pid)j(=)f(MPI)p 423 250 16 2 v 18 w(INQPID\(gid,)h(lo)q(cation\))g Fs(with)75 385 y Ft(IN)f(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with)h(cartesian)f (structure,)75 477 y Ft(IN)i(lo)q(cation)26 b Fs(co)q(ordinate)15 b(v)o(ector)g(of)g(pro)q(cess.)166 577 y(In)h(a)f(sense,)g(the)g(in)o (v)o(erse)h(op)q(eration)f(to)g(MPI)p 970 577 14 2 v 16 w(INQPID)h(is)g(pro)o(vided)g(b)o(y)f(the)g(function:)166 669 y Ft(MPI)p 275 669 16 2 v 18 w(INQLOC\(gid,)j(rank,)f(lo)q(cation,) j(maxdim\))15 b Fs(with)75 804 y Ft(IN)i(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with)h(cartesian)f(structure,)75 896 y Ft(IN)i(rank)22 b Fs(rank)15 b(of)g(a)g(pro)q(cess)g(within)i (group)e Fh(gid)p Fs(,)75 987 y Ft(OUT)j(lo)q(cation\(*)q(\))26 b Fs(co)q(ordinate)15 b(v)o(ector)g(of)f(pro)q(cess,)75 1079 y Ft(IN)j(maxdim)23 b Fs(length)15 b(of)g(v)o(ector)g Fh(location)f Fs(in)i(the)f(calling)i(program)d(\(only)i(for)e(error)h (c)o(hec)o(king\).)75 1179 y(MPI)p 167 1179 14 2 v 16 w(INQLOC)i(translates)e(a)g(rank)f(in)j(the)e(group)g(in)o(to)g(a)g (top)q(ology)g(lo)q(cation.)166 1318 y Fl(Discussion:)22 b Fk(Dep)q(ending)17 b(on)g(the)h(decisions)g(tak)o(en)f(in)g(the)h (PT2PT)f(sub)q(committee,)f(MPI)p 1708 1318 13 2 v 16 w(INQPID)75 1375 y(migh)o(t)c(b)q(e)i(c)o(hanged)h(to)e(return)i(the)g (rank)f(in)f(the)i(group)e(rather)i(than)f(a)g(pid.)75 1578 y Fj(4.4.4)55 b(P)n(artitioning)18 b(of)h(cartesian)f(structures) 75 1664 y Fs(If)h(a)g(group)f(has)h(b)q(een)h(created)f(with)g(MPI)p 856 1664 14 2 v 16 w(CAR)l(T,)g(the)g(function)h(MPI)p 1386 1664 V 16 w(P)l(AR)l(TC)g(can)f(b)q(e)g(used)h(to)75 1720 y(partition)12 b(the)g(group)g(in)o(to)g(subgroups)g(whic)o(h)h (form)e(h)o(yp)q(erplanes)j(of)d(the)h(original)h(pro)q(cess)g (structure.)75 1777 y(As)k(an)f(example,)i(assume)e(that)g(MPI)p 751 1777 V 17 w(CAR)l(T)h(has)f(de\014ned)i(a)e(t)o(w)o(o{dimensional)i (grid)f(structure)f(on)75 1833 y(a)g(pro)q(cess)g(group.)22 b(With)16 b(a)g(call)h(to)e(MPI)p 811 1833 V 16 w(P)l(AR)l(TC)i(this)f (group)g(can)g(then)g(b)q(e)h(split)g(up)f(in)o(to)g(disjoin)o(t)75 1890 y(subgroups)f(whic)o(h)h(represen)o(t)g(ro)o(ws)e(or)g(columns)i (of)f(the)g(original)i(group.)i(The)d(function)g(syn)o(tax)e(is:)166 1982 y Ft(gid)p 238 1982 16 2 v 19 w(new)k(=)f(MPI)p 530 1982 V 18 w(P)l(AR)l(TC\(gid,)g(lostdim\))f Fs(with)75 2117 y Ft(IN)h(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with)h (cartesian)f(structure,)75 2209 y Ft(IN)i(lostdim)24 b Fs(direction)16 b(p)q(erp)q(endicul)q(ar)h(to)e(resulting)h(h)o(yp)q (erplanes.)75 2308 y(The)22 b(function)h(returns)f(the)h(group)f(iden)o (ti\014er)h(of)f(the)g(h)o(yp)q(erplane)i(group)e(in)h(whic)o(h)g(the)f (calling)75 2365 y(pro)q(cess)15 b(is)h(con)o(tained.)166 2421 y(MPI)p 258 2421 14 2 v 16 w(P)l(AR)l(TC)f(uses)f(the)g(function)h (MPI)p 883 2421 V 17 w(CREA)l(TE)p 1091 2421 V 16 w(GR)o(OUP)g(to)e (partition)i(the)f(paren)o(t)g(group,)75 2478 y(using)22 b(the)f(pro)q(cess)g(co)q(ordinate)h(in)f(direction)i Fh(lostdim)d Fs(as)g(the)i(k)o(ey)e(v)m(alue.)39 b(After)21 b(a)f(group)h(has)75 2534 y(b)q(een)14 b(partitioned)f(b)o(y)g(MPI)p 565 2534 V 16 w(P)l(AR)l(TC,)g(the)f(inquiry)i(functions)g(MPI)p 1275 2534 V 16 w(INQCAR)l(T)g(and)f(MPI)p 1694 2534 V 16 w(INQPID)75 2591 y(consisten)o(tly)j(refer)f(to)g(the)g(grid)g (sections)h(if)g(applied)h(to)d(a)h(subgroup.)166 2647 y(If)g(the)h(pro)q(cess)f(top)q(ology)g(is)h(a)e(cartesian)i (structure,)e(a)h(collectiv)o(e)i(shift)e(op)q(eration)h(most)e(lik)o (ely)75 2704 y(is)19 b(used)f(along)h(a)f(co)q(ordinate)g(direction.)30 b(The)19 b(function)g(MPI)p 1208 2704 V 16 w(SHIFTGR)g(partitions)f(a)g (cartesian)p eop %%Page: 61 66 61 65 bop 75 -100 a Fq(4.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b Fs(61)75 45 y(group)17 b(in)o(to)f(one{dimensional)j(subgroups,)e(whic)o(h)h (are)e(all)i(orien)o(ted)f(in)h(one)f(co)q(ordinate)g(direction.)75 102 y(Ev)o(ery)e(pro)q(cess)g(is)h(con)o(tained)g(in)g(exactly)f(one)h (subgroup:)166 197 y Ft(gid)p 238 197 16 2 v 19 w(new)i(=)f(MPI)p 530 197 V 18 w(SHIFTGR\(gid,)h(direction\))f Fs(with)75 354 y Ft(IN)g(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with)h (cartesian)f(structure,)75 461 y Ft(IN)i(direction)24 b Fs(co)q(ordinate)16 b(direction)g(of)f(one{dimensional)i(subgroups.) 166 583 y(MPI)p 258 583 14 2 v 16 w(SHIFTGR)11 b(uses)f(the)g(function) h(MPI)p 918 583 V 17 w(CREA)l(TE)p 1126 583 V 16 w(GR)o(OUP)g(to)e (partition)i(the)f(paren)o(t)g(group.)75 639 y(The)15 b(return)f(v)m(alue)i(is)f(the)g(group)f(iden)o(ti\014er)i(of)e(the)h (one{dimensional)h(subgroup)f(to)f(whic)o(h)h(the)g(call-)75 696 y(ing)i(pro)q(cess)g(b)q(elongs.)24 b(The)17 b(subgroup)g(has)f(a)g (one{dimensional)j(cartesian)d(structure.)24 b(It)16 b(is)h(either)75 752 y(p)q(erio)q(dic)23 b(\(ring\))e(or)f(not)g (\(linear\),)j(dep)q(ending)g(on)e(the)g(p)q(erio)q(dicit)o(y)i(of)d (the)h(paren)o(t)g(group)f(in)i(the)75 809 y(corresp)q(onding)16 b(direction.)166 951 y Fl(Discussion:)f Fk(Instead)f(of)f(creating)g (one{dimensional)e(subgroups)j(and)f(then)h(use)g(the)g(standard)g (collec-)75 1007 y(tiv)o(e)e(comm)o(unicatio)o(n)d(\(circular)k(or)f (end{o\013)s(\))h(shift)e(functions,)h(one)h(could)e(also)h(de\014ne)h (a)f(top)q(ology)f(based)h(shift)75 1064 y(function.)29 b(It)18 b(w)o(ould)e(lo)q(ok)h(v)o(ery)h(m)o(uc)o(h)e(lik)o(e)h(the)h (standard)g(shift,)g(with)f(an)h(additional)e(argumen)o(t)g(de\014ning) 75 1120 y(the)i(co)q(ordinate)g(direction)g(for)f(the)h(shift.)29 b(Dep)q(ending)18 b(on)g(the)g(p)q(erio)q(dicit)o(y)f(information)e(of) i(the)h(cartesian)75 1177 y(structure,)e(the)e(shift)g(function)f (could)h(c)o(ho)q(ose)h(the)f(circular)g(or)g(end{o\013)g(v)o(ersion)g (automatically)m(.)166 1319 y Fs(In)22 b(a)g(p)q(ortable)g(program)f (the)h(n)o(um)o(b)q(er)g(of)f(a)o(v)m(ailable)j(pro)q(cessors)d (usually)i(is)g(not)e(kno)o(wn)h(at)75 1376 y(compile)g(time.)35 b(In)21 b(the)g(case)f(of)g(a)g(cartesian)g(top)q(ology)g(the)h (function)g(MPI)p 1478 1376 V 16 w(MAKDIM)g(helps)g(in)75 1432 y(setting)i(the)f(n)o(um)o(b)q(ers)h(of)f(pro)q(cesses)h(p)q(er)g (co)q(o)q(dinate)g(direction,)j(dep)q(ending)e(on)f(the)g(n)o(um)o(b)q (er)f(of)75 1488 y(pro)q(cesses)16 b(in)g(the)f(group)g(and)g(optional) h(constrain)o(ts)f(whic)o(h)h(can)f(b)q(e)h(sp)q(eci\014ed)h(b)o(y)e (the)h(user:)166 1584 y Ft(MPI)p 275 1584 16 2 v 18 w(MAKDIM\(gid,)h (ndims,)f(dims\))f Fs(with)75 1741 y Ft(IN)i(gid)24 b Fs(group)14 b(id)j(of)d(group)h(whic)o(h)h(is)g(to)f(b)q(e)g (structured,)75 1848 y Ft(IN)i(ndims)23 b Fs(n)o(um)o(b)q(er)15 b(of)g(cartesian)g(dimensions,)75 1954 y Ft(IN/OUT)j(dims)23 b Fs(n)o(um)o(b)q(er)15 b(of)g(pro)q(cesses)g(in)i(eac)o(h)e(co)q (ordinate)g(direction.)166 2076 y(The)d(pro)q(cesses)g(of)f(the)h (group)f(with)i(ID)e(n)o(um)o(b)q(er)h Fh(gid)g Fs(are)f(divided)j(as)d (ev)o(enly)i(as)e(p)q(ossible)j(among)75 2133 y Fh(ndims)i Fs(dimensions,)i(sub)s(ject)e(to)g(the)g(constrain)o(t)g(that)g(all)i (pro)q(cesses)e(in)i(the)e(group)g(are)g(used.)25 b(The)75 2189 y(caller)19 b(ma)o(y)e(further)h(constrain)g(the)g(op)q(eration)g (of)g(this)g(routine)g(b)o(y)g(sp)q(ecifying)i(elemen)o(ts)f(of)e(arra) o(y)75 2246 y Fh(dims)p Fs(.)23 b(If)17 b Fh(dims\(i\))f Fs(is)h(set)f(to)g(a)h(p)q(ositiv)o(e)g(n)o(um)o(b)q(er,)g(the)g (routine)g(will)h(\014x)f(the)f(n)o(um)o(b)q(er)h(of)g(pro)q(cesses)75 2302 y(in)g(co)q(ordinate)g(direction)g Fh(i)f Fs(to)g Fh(dims\(i\))p Fs(.)22 b(Pro)q(cesses)16 b(will)i(then)f(b)q(e)f (distributed)i(among)e(directions)75 2359 y(where)f Fh(dims\(i\))g Fs(is)g(set)g(to)g(0,)f(sub)s(ject)h(to)g(the)g(ab)q(o)o(v)o(e)g (constrain)o(ts.)166 2418 y(Note)k(that)h(an)f(error)g(will)j(o)q(ccur) e(if)h Fh(dims\(i\))d Fs(is)j(p)q(ositiv)o(e)g(and)f(do)q(es)g(not)f (ev)o(enly)i(divide)h(the)75 2475 y(n)o(um)o(b)q(er)13 b(of)f(pro)q(cesses)g(in)h(the)g(group.)19 b(Negativ)o(e)12 b(input)h(v)m(alues)h(of)e Fh(dims\(i\))f Fs(will)j(also)e(cause)h(an)f (error.)75 2531 y(All)k(elemen)o(ts)g(of)f Fh(dims\(*\))f Fs(m)o(ust)h(b)q(e)h(set)f(to)f(0)h(for)g(completely)h(automatic)f(pro) q(cess)g(distribution.)166 2591 y(When)c(the)g(function)h(returns,)f Fh(dims\(i\))f Fs(will)j(b)q(e)e(set)g(to)f(the)h(n)o(um)o(b)q(er)g(of) g(pro)q(cesses)g(in)h(co)q(ordinate)75 2647 y(direction)20 b Fh(i)p Fs(.)29 b(F)l(or)18 b Fh(dims\(i\))g Fs(not)g(set)g(b)o(y)h (the)f(user,)h Fh(dims\(i\))f Fs(will)i(b)q(e)f(ordered)g(\(MAXIMUM,)e (...,)75 2704 y(MINIMUM\).)d(Arra)o(y)h Fh(dims\(*\))f Fs(is)i(suitable)g(for)f(use)g(as)g(input)h(to)f(routine)g(MPI)p 1503 2704 14 2 v 17 w(CAR)l(T.)p eop %%Page: 62 67 62 66 bop 75 -100 a Fs(62)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fm(4.5)70 b(A)22 b(simple)f(example)75 147 y Fs(The)12 b(example)h(in)g(\014gure)f(4.1)f (sho)o(ws)g(ho)o(w)h(the)g(grid)g(de\014nition)i(and)e(inquiry)i (functions)f(can)f(b)q(e)h(used)f(in)75 203 y(an)k(application)h (program.)k(A)16 b(partial)g(di\013eren)o(tial)h(equation,)f(for)f (example)h(the)g(P)o(oisson)g(equation,)75 259 y(is)g(to)f(b)q(e)h (solv)o(ed)f(on)h(a)f(rectangular)g(domain.)21 b(First)15 b(the)g(pro)q(cesses)h(organize)f(themselv)o(es)h(in)h(a)e(t)o(w)o(o{) 75 316 y(dimensional)j(structure.)i(Eac)o(h)15 b(pro)q(cess)h(then)g (inquires)h(for)e(the)h(PID)f(n)o(um)o(b)q(ers)h(of)f(its)h(neigh)o(b)q (ors)g(in)75 372 y(the)f(four)g(directions.)21 b(The)15 b(n)o(umerical)h(problem)g(is)g(solv)o(ed)f(b)o(y)g(an)g(iterativ)o(e)g (metho)q(d,)g(the)g(details)h(of)75 429 y(whic)o(h)g(are)f(hidden)i(in) f(the)f(subroutine)h Fh(relax)p Fs(.)166 485 y(In)e(eac)o(h)f (relaxation)g(step)g(eac)o(h)g(pro)q(cess)h(computes)f(new)g(v)m(alues) h(for)f(the)g(solution)h(grid)f(function)75 542 y(at)19 b(all)i(p)q(oin)o(ts)f(o)o(wned)f(b)o(y)h(the)g(pro)q(cess.)33 b(Then)21 b(the)e(v)m(alues)i(at)e(in)o(terpro)q(cess)h(b)q(oundaries)h (ha)o(v)o(e)f(to)75 598 y(b)q(e)h(exc)o(hanged)h(with)f(neigh)o(b)q (oring)h(pro)q(cesses.)37 b(F)l(or)20 b(example,)j(the)d(exc)o(hange)h (subroutine)h(migh)o(t)75 655 y(con)o(tain)13 b(a)g(call)i(lik)o(e)f (MPI)p 524 655 14 2 v 16 w(SENDC\(...,neigh)p 873 655 V 16 w(pid\(1\),...\))k(to)12 b(send)i(up)q(dated)g(v)m(alues)h(to)d (the)h(left{hand)75 711 y(neigh)o(b)q(or)j(\(i-1,j\).)j(Note)c(that)g (there)g(is)h(only)f(lo)q(cal)i(comm)o(unication)f(in)g(this)f(exc)o (hange)h(op)q(eration.)p eop %%Page: 63 68 63 67 bop 75 -100 a Fq(4.5.)34 b(A)15 b(SIMPLE)h(EXAMPLE)1180 b Fs(63)252 163 y(in)o(teger)15 b(ndims,)h(mx)p 614 163 14 2 v 16 w(neigh)252 219 y(parameter)e(\(ndims=2,)i(mx)p 757 219 V 16 w(neigh=4\))252 276 y(in)o(teger)f(gid,)h(gid)p 554 276 V 17 w(cart,)e(dims\(ndims\),)h(neigh)p 1057 276 V 18 w(def\(2\),)f(neigh)p 1325 276 V 17 w(pid\(mx)p 1484 276 V 18 w(neigh\))252 332 y(in)o(teger)h(o)o(wn)p 485 332 V 16 w(p)q(osition\(ndims\),)h(p)q(erio)q(d\(ndims\),)h(i,)e (j,)g(n)o(um)p 1319 332 V 16 w(neigh,)h(ret,)f(rank)252 388 y(real)p Fo(\003)p Fs(8)g(u\(0:101,0:101)o(\),)d(f\(0:101,0:10)o (1\))252 445 y(data)i(dims)i(/)f(ndims)h Fo(\003)f Fs(0)g(/)75 501 y(C)89 b(Create)15 b(a)g(grid)g(structure)g(in)h(ALL)g(group)f(and) h(inquire)g(ab)q(out)g(o)o(wn)e(p)q(osition)252 558 y(gid)i(=)f(ALL)75 614 y(C)89 b(Set)15 b(pro)q(cess)h(grid)f(size)i(and)e(p)q(erio)q (dicit)o(y)252 671 y(ret)g(=)g(MPI)p 465 671 V 16 w(MAKDIM\(gid,)g (ndims,)h(dims\))252 727 y(if)f(\(ret)g(.lt.)20 b(0\))14 b(call)j(error)p 697 727 V 15 w(handler\(ret\))252 784 y(p)q(erio)q(d\(1\))e(=)h(MPI)p 594 784 V 16 w(PER)252 840 y(p)q(erio)q(d\(2\))f(=)h(MPI)p 594 840 V 16 w(PER)252 897 y(rank)f(=)g(MPI)p 499 897 V 17 w(CAR)l(T)g(\(gid,)g(gid)p 826 897 V 17 w(cart,)f(ndims,)i(dims,)g(p)q(erio)q(d\))252 953 y(ret)f(=)g(MPI)p 465 953 V 16 w(INQCAR)l(T)i(\(gid)p 790 953 V 17 w(cart,)d(ndims,)h(dims,)h(p)q(erio)q(d,)g(o)o(wn)p 1415 953 V 16 w(p)q(osition\))75 1009 y(C)89 b(Lo)q(ok)15 b(up)h(the)f(PID)h(n)o(um)o(b)q(ers)f(for)g(the)g(neigh)o(b)q(ors.)21 b(Own)15 b(pro)q(cess)h(co)q(ordinates)f(are)g(\(i,j\).)75 1066 y(C)89 b(Neigh)o(b)q(ors)16 b(are)f(\(i{1,j\),)f(\(i+1,j\),)g (\(i,j{1\),)g(\(i,j+1\))252 1122 y(i)i(=)f(o)o(wn)p 413 1122 V 16 w(p)q(osition\(1\))252 1179 y(j)g(=)g(o)o(wn)p 414 1179 V 16 w(p)q(osition\(2\))252 1235 y(neigh)p 360 1235 V 17 w(def\(1\))g(=)g(i{1)252 1292 y(neigh)p 360 1292 V 17 w(def\(2\))g(=)g(j)252 1348 y(neigh)p 360 1348 V 17 w(pid\(1\))h(=)f(MPI)p 653 1348 V 16 w(INQPID)i(\(gid,)e(neigh)p 1064 1348 V 17 w(def)t(\))252 1405 y(neigh)p 360 1405 V 17 w(def\(1\))g(=)g(i+1)252 1461 y(neigh)p 360 1461 V 17 w(def\(2\))g(=)g(j)252 1518 y(neigh)p 360 1518 V 17 w(pid\(2\))h(=)f(MPI)p 653 1518 V 16 w(INQPID)i(\(gid,)e(neigh)p 1064 1518 V 17 w(def)t(\))252 1574 y(neigh)p 360 1574 V 17 w(def\(1\))g(=)g(i)252 1630 y(neigh)p 360 1630 V 17 w(def\(2\))g(=)g(j{1)252 1687 y(neigh)p 360 1687 V 17 w(pid\(3\))h(=)f(MPI)p 653 1687 V 16 w(INQPID)i(\(gid,)e(neigh)p 1064 1687 V 17 w(def)t(\))252 1743 y(neigh)p 360 1743 V 17 w(def\(1\))g(=)g(i)252 1800 y(neigh)p 360 1800 V 17 w(def\(2\))g(=)g(j+1)252 1856 y(neigh)p 360 1856 V 17 w(pid\(4\))h(=)f(MPI)p 653 1856 V 16 w(INQPID)i(\(gid,)e(neigh)p 1064 1856 V 17 w(def)t(\))252 1913 y(n)o(um)p 342 1913 V 16 w(neigh)h(=)g(mx)p 592 1913 V 16 w(neigh)75 1969 y(C)89 b(Initialize)18 b(the)e(grid)f(functions)h(and)g(start)d(the)j (iteration)252 2026 y(call)g(init)g(\(u,)f(f)t(\))252 2082 y(do)g(10)g(it=1,100)322 2139 y(call)i(relax)e(\(u,)g(f)t(\))75 2195 y(C)89 b(Exc)o(hange)15 b(data)g(with)g(neigh)o(b)q(or)h(pro)q (cesses)322 2251 y(call)h(exc)o(hange)e(\(u,)g(neigh)p 779 2251 V 17 w(pid\))75 2308 y(10)131 b(con)o(tin)o(ue)252 2364 y(call)16 b(output)f(\(u\))252 2421 y(end)147 2625 y(Figure)g(4.1:)k(Set{up)d(of)e(pro)q(cess)i(structure)f(for)f(t)o(w)o (o{dimensional)i(parallel)h(P)o(oisson)e(solv)o(er.)p eop %%Page: 64 69 64 68 bop 75 356 a Fn(Chapter)34 b(5)75 564 y Fu(Language)41 b(Binding)952 2828 y Fs(64)p eop %%Page: 65 70 65 69 bop 75 356 a Fn(Chapter)34 b(6)75 564 y Fu(Correctness)857 786 y Fs(Ewing)16 b(Lusk)166 880 y(This)k(c)o(hapter)g(will)i(ev)o(olv) o(e)e(to)f(include)j(discussions)g(and)e(examples)h(related)f(to)g(the) g(issue)g(of)75 937 y(correct)e(and)h(safe)g(MPI)g(programs.)29 b(It)19 b(ma)o(y)f(ev)o(en)o(tually)i(include)h(a)e(formal)f(sp)q (eci\014cation)j(for)d(the)75 993 y(seman)o(tics)f(of)g(MPI)g (routines.)26 b(F)l(or)16 b(the)h(time)h(b)q(eing,)g(it)g(includes)h (discussions)g(and)e(examples)h(that)75 1050 y(ha)o(v)o(e)d(up)g(to)g (no)o(w)g(app)q(eared)h(in)g(other)e(c)o(hapters.)75 1170 y Fj(6.0.1)55 b(Order)75 1256 y Fs(MPI)17 b(preserv)o(es)f(the)h (order)g(of)f(messages)g(b)q(et)o(w)o(een)h(an)o(y)g(\014xed)g(pair)g (of)g(pro)q(cesses.)24 b(In)18 b(other)e(w)o(ords,)75 1312 y(if)22 b(pro)q(cess)f(A)h(executes)f(t)o(w)o(o)f(successiv)o(e)j (send)f Fh(start)e Fs(sub)q(op)q(erations,)j(pro)q(cess)f(B)f(executes) h(t)o(w)o(o)75 1369 y(successiv)o(e)e(receiv)o(e)g Fh(start)e Fs(op)q(erations,)i(and)f(b)q(oth)h(receiv)o(es)f(matc)o(h)g(either)h (sends,)g(then)f(the)g(\014rst)75 1425 y(receiv)o(e)f(will)h(receiv)o (e)g(the)e(message)g(sen)o(t)g(b)o(y)h(the)f(\014rst)g(send,)i(and)e (the)h(second)g(receiv)o(e)g(will)h(receiv)o(e)75 1482 y(the)f(message)f(sen)o(t)h(b)o(y)g(the)g(second)g(send.)29 b(Th)o(us,)18 b(if)h(a)e(t)o(w)o(o)g(messages)g(from)g(the)h(same)g (source)g(can)75 1538 y(satisfy)g(a)g(p)q(ending)i(receiv)o(e,)g(the)e (\014rst)g(message)g(sen)o(t)g(is)g(accepted;)i(if)f(a)f(message)g(can) g(satisfy)g(t)o(w)o(o)75 1594 y(p)q(ending)f(receiv)o(es,)f(the)f (\014rst)g(receiv)o(e)h(p)q(osted)f(is)h(satis\014ed.)166 1651 y(The)c(last)f(paragraph)g(assumes)g(that)g(the)h(send)g Fh(start)e Fs(op)q(erations)i(are)f(ordered)h(b)o(y)f(the)h(program)75 1707 y(order)20 b(at)f(pro)q(cess)h(A,)g(and)g(the)g(receiv)o(e)h Fh(start)e Fs(op)q(erations)h(are)f(ordered)i(b)o(y)e(the)h(program)f (order)75 1764 y(at)h(pro)q(cess)h(B.)g(If)g(a)g(pro)q(cess)g(is)g(m)o (ultithreaded)h(and)f(the)g(op)q(erations)h(are)e(executed)i(b)o(y)f (distinct)75 1820 y(threads,)d(then)g(the)g(seman)o(tics)f(of)h(the)f (threaded)h(system)g(ma)o(y)f(not)g(de\014ne)i(an)e(order)h(b)q(et)o(w) o(een)g(the)75 1877 y(t)o(w)o(o)c(op)q(erations,)h(in)h(whic)o(h)g (case)f(the)g(condition)i(is)f(v)o(oid.)75 1997 y Fj(6.0.2)55 b(Progress)19 b(and)g(F)-5 b(airness)75 2083 y Fs(W)l(e)20 b(can)g(mo)q(del)g(the)g(execution)h(of)e(MPI)h(programs)e(as)h(an)h (in)o(teraction)g(b)q(et)o(w)o(een)g(executing)h(pro-)75 2139 y(cesses)d(that)f(execute)h(eac)o(h)g(their)g(o)o(wn)f(program,)g (and)g(the)h Ft(comm)o(unication)j(subsystem)p Fs(.)26 b(The)75 2195 y(comm)o(unication)16 b(subsystem)g(ma)o(y)f(ha)o(v)o(e)g (v)m(arious)h(constrain)o(ts)g(on)f(the)h(amoun)o(t)f(of)g(resources)h (it)g(can)75 2252 y(use.)k(E.g.:)166 2308 y(Bounds)e(on)f(the)g(n)o(um) o(b)q(er)h(and)f(total)g(sizes)h(of)f(activ)o(e)h(comm)o(unication)g (ob)s(jects.)25 b(Suc)o(h)18 b(b)q(ound)75 2365 y(can)d(b)q(e)h (global,)g(p)q(er)f(no)q(de,)h(or)e(p)q(er)i(pair)g(of)e(comm)o (unicating)i(no)q(des.)166 2421 y(Bounds)e(on)g(the)g(n)o(um)o(b)q(er)g (and)h(total)e(sizes)i(of)e(messages)g(bu\013ered)i(in)f(the)g(system.) 19 b(Suc)o(h)c(b)q(ound)75 2478 y(can,)k(again,)g(b)q(e)g(global,)g(p)q (er)g(no)q(de,)g(or)f(p)q(er)h(pair)f(of)g(comm)o(unicating)h(no)q(de.) 30 b(In)19 b(addition,)h(a)e(mes-)75 2534 y(sage)f(ma)o(y)g(b)q(e)h (bu\013ered)g(at)f(the)g(sender,)i(at)d(the)i(receiv)o(er,)g(at)f(b)q (oth,)h(or)f(p)q(erhaps)h(at)f(another)g(place)75 2591 y(altogether.)166 2647 y(Th)o(us,)i(it)g(will)i(b)q(e)e(di\016cult)h (to)e(set)h(rules)g(on)g(resource)g(managemen)o(t)f(of)g(the)h(comm)o (unication)75 2704 y(subsystem.)29 b(Ho)o(w)o(ev)o(er,)17 b(it)i(is)g(generally)g(exp)q(ected)g(that)f(implemen)o(ters)h(will)h (pro)o(vide)f(information)952 2828 y(65)p eop %%Page: 66 71 66 70 bop 75 -100 a Fs(66)1097 b Fq(CHAPTER)15 b(6.)30 b(CORRECTNESS)75 45 y Fs(on)15 b(the)g(mec)o(hanism)g(used)h(for)e (resource)h(allo)q(cation,)h(and)f(that)f(query)i(and)f(set)f (functions)i(will)h(allo)o(w)75 102 y(to)e(query)g(and)g(p)q(ossibly)i (con)o(trol)e(the)g(amoun)o(t)g(of)f(a)o(v)m(ailable)j(resources.)166 158 y(W)l(e)g(pro)o(vide)h(in)g(this)f(section)h(a)f(set)g(of)g (minimal)h(requiremen)o(ts)g(on)f(the)h(comm)o(unication)f(sub-)75 214 y(system.)26 b(Programs)15 b(that)i(execute)h(on)f(an)o(y)g (subsystem)g(that)f(ful\014ls)j(these)f(minimal)g(requiremen)o(ts)75 271 y(are)12 b Ft(safe)h Fs(and)g(will)h(p)q(ort)f(to)f(an)o(y)g(MPI)h (implemen)o(tation.)21 b Ft(Unsafe)13 b Fs(programs)e(ma)o(y)h(execute) i(on)e(some)75 327 y(MPI)k(implemen)o(tations,)h(dep)q(ending)h(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(and)g(the)g (implemen-)75 384 y(tation)k(used)g(for)g(the)g(MPI)g(comm)o(unication) h(subsystem.)34 b(Finally)22 b Ft(erroneous)d Fs(programs)g(nev)o(er)75 440 y(execute)e(correctly)l(.)22 b(\(While)17 b(it)f(is)g(desirable)i (to)d(detect)h(erroneous)g(programs,)e(it)i(is)h(not)e(p)q(ossible)j (to)75 497 y(do)c(so)h(at)e(compile)j(time,)f(and)g(often)f(prohibitiv) o(e)i(to)e(do)h(so)f(a)g(run)h(time.)20 b(Th)o(us,)14 b(the)h(do)q(cumen)o(t)g(do)q(es)75 553 y(not)f(sp)q(ecify)h(a)f(b)q (eha)o(vior)h(for)f(erroneous)g(programs,)f(although)h(the)g(desired)i (b)q(eha)o(vior)f(is)f(to)g(return)g(a)75 610 y(useful)i(error)f (message.\))131 716 y(1.)22 b(If)15 b(a)g(pro)q(cess)g(executes)h(an)f Fh(INIT)f Fs(op)q(eration,)h(then)g(the)h(op)q(eration)f(ev)o(en)o (tually)h(succeeds,)g(or)e(a)189 773 y Fi(r)n(esour)n(c)n(e)f(exc)n (eption)f Fs(o)q(ccurs.)19 b(The)13 b(standard)f(do)q(es)g(not)h(sp)q (ecify)g(when)g(a)f(resource)h(exception)g(is)189 829 y(allo)o(w)o(ed)h(to)g(o)q(ccur.)20 b(It)14 b(is)h(exp)q(ected)h(that)d (an)h(op)q(erational)h(de\014nition)h(will)g(b)q(e)f(made)g(a)o(v)m (ailable,)189 885 y(in)20 b(the)g(form)e(of)h(test)g(programs)g(that)f (ha)o(v)o(e)h(to)g(execute)h(with)g(no)g(resource)f(exceptions.)34 b(It)189 942 y(is)21 b(highly)i(desirable)f(to)f(ha)o(v)o(e)f(generous) h(b)q(ounds)h(on)f(the)g(n)o(um)o(b)q(er)h(of)e(concurren)o(tly)i (activ)o(e)189 998 y(comm)o(unication)17 b(ob)s(jects)f(eac)o(h)g(pro)q (cess)h(ma)o(y)f(ha)o(v)o(e,)g(so)g(that,)g(in)i(practice,)f Fh(INIT)f Fs(op)q(erations)189 1055 y(will)g(alw)o(a)o(ys)f(b)q(e)h (guaran)o(teed)f(to)f(succeed.)131 1149 y(2.)22 b(Eac)o(h)16 b(pro)q(cess)g(can)h(initiate)h(a)e(comm)o(unication)h(op)q(eration)g (for)e(eac)o(h)i(activ)o(e)f(comm)o(unication)189 1205 y(ob)s(ject.)j(I.e.)h(correct)15 b Fh(START)f Fs(op)q(erations)i(alw)o (a)o(ys)e(succeed)i(\(ev)o(en)o(tually\).)131 1299 y(3.)22 b(A)e(send)h(op)q(eration)f(is)h Ft(enabled)h Fs(if)e(the)h(sending)g (pro)q(cess)g(has)f(issued)i(a)e Fh(COMPLETE)f Fs(op)q(era-)189 1356 y(tion)f(and)g(the)f(receiving)j(pro)q(cess)e(has)g(issued)g(a)g Fh(START)f Fs(op)q(eration)h(for)f(a)h(matc)o(hing)f(receiv)o(e.)189 1412 y(Symmetrically)l(,)k(a)d(receiv)o(e)i(op)q(eration)f(is)g Ft(enabled)g Fs(if)h(the)e(receiving)j(pro)q(cess)e(has)f(issued)i(a) 189 1468 y Fh(COMPLETE)10 b Fs(op)q(eration)i(and)f(the)h(sending)g (pro)q(cess)g(has)f(issued)i(a)e Fh(START)g Fs(op)q(eration)g(for)g(a)g (matc)o(h-)189 1525 y(ing)19 b(send.)31 b(An)19 b(enabled)h(op)q (eration)e(ma)o(y)g(b)q(ecome)i Ft(disabled)f Fs(either)h(b)q(ecause)f (it)g(completes)189 1581 y(successfully)e(or,)e(in)h(the)f(case)h(of)f (a)g(receiv)o(e,)h(b)q(ecause)g(the)g(matc)o(hing)f(message)g(is)h (successfully)189 1638 y(receiv)o(ed)g(b)o(y)f(another)g(receiv)o(e)g (op)q(eration)h(or,)e(in)i(the)f(case)g(of)g(a)f(send,)i(b)q(ecause)g (the)f(matc)o(hing)189 1694 y(receiv)o(e)h(successfully)h(receiv)o(es)f (another)f(message.)189 1770 y Ft(An)24 b(enabled)j(op)q(eration)f (either)g(completes)f(successfully)h(or)e(b)q(ecomes)i(p)q(erma-)189 1826 y(nen)o(tly)17 b(disabled.)189 1901 y Fs(I.e.,)d(an)h(enabled)i (op)q(eration)e(either)h(ev)o(en)o(tually)g(succeeds,)g(or)e(b)q (ecomes)i(disabled)h(\(progress\);)189 1958 y(and)i(an)g(op)q(eration)g (cannot)g(b)q(e)h(enabled)h(in\014nitely)g(man)o(y)e(times)g(without)g (ev)o(er)g(succeeding)189 2014 y(\(fairness\).)131 2108 y(4.)j(A)15 b Fh(FREE)f Fs(op)q(eration)i(alw)o(a)o(ys)e(succeeds)j (\(ev)o(en)o(tually\).)166 2214 y(The)i(four)f(conditions)i(guaran)o (tee)d(progress)h(in)i(the)e(comm)o(unication)i(subsystem.)30 b(The)18 b(third)75 2271 y(condition)e(guaran)o(tee)f(\(w)o(eak\))f (fairness)h(among)g(comp)q(eting)h(comm)o(unication)g(requests.)166 2327 y(Examples)g(\(in)o(v)o(olving)g(t)o(w)o(o)e(pro)q(cessors)h(with) g(ranks)g(0)g(and)g(1\))166 2384 y(The)g(follo)o(wing)h(program)e(is)i (safe,)f(and)g(should)h(alw)o(a)o(ys)f(succeed.)75 2534 y Fh(CALL)23 b(MPI_RANK\(rank,)f(context\))75 2591 y(IF)i (\(rank.EQ.0\))e(THEN)147 2647 y(CALL)h(MPI_SENDB\(sendbuf,)e(len,)j (1,)f(tag,)h(context\))147 2704 y(CALL)f(MPI_RECVB\(recvbuf,)e(len,)j (1,)f(tag,)h(context\))p eop %%Page: 67 72 67 71 bop 1830 -100 a Fs(67)75 45 y Fh(ELSE)95 b(!)24 b(rank.EQ.1)147 102 y(CALL)f(MPI_RECVB\(recvbuf,)e(len,)j(0,)f(tag,)h (context\))147 158 y(CALL)f(MPI_SENDB\(sendbuf,)e(len,)j(0,)f(tag,)h (context\))75 214 y(END)f(IF)166 308 y Fs(The)15 b(follo)o(wing)h (program)e(is)i(erroneous,)f(and)g(should)h(alw)o(a)o(ys)f(fail.)75 471 y Fh(CALL)23 b(MPI_RANK\(rank,)f(context\))75 527 y(IF)i(\(rank.EQ.0\))e(THEN)147 584 y(CALL)h(MPI_RECVB\(recvbuf,)e (len,)j(1,)f(tag,)h(context\))147 640 y(CALL)f(MPI_SENDB\(sendbuf,)e (len,)j(1,)f(tag,)h(context\))75 697 y(ELSE)95 b(!)24 b(rank.EQ.1)147 753 y(CALL)f(MPI_RECVB\(recvbuf,)e(len,)j(0,)f(tag,)h (context\))147 810 y(CALL)f(MPI_SENDB\(sendbuf,)e(len,)j(0,)f(tag,)h (context\))75 866 y(END)f(IF)166 972 y Fs(The)15 b(receiv)o(e)g(op)q (eration)g(of)f(the)h(\014rst)f(pro)q(cess)h(m)o(ust)f(complete)i(b)q (efore)f(its)f(send,)h(and)g(can)g(com-)75 1029 y(plete)j(only)g(if)g (the)f(matc)o(hing)g(send)h(of)f(the)g(second)h(pro)q(cessor)f(is)h (executed;)g(the)g(receiv)o(e)g(op)q(eration)75 1085 y(of)c(the)h(second)g(pro)q(cess)g(m)o(ust)f(complete)h(b)q(efore)g (its)f(send)i(and)e(can)h(complete)g(only)g(if)g(the)g(matc)o(hing)75 1142 y(send)h(of)f(the)g(\014rst)g(pro)q(cess)g(is)h(executed.)21 b(This)15 b(program)f(will)j(deadlo)q(c)o(k.)166 1198 y(The)10 b(follo)o(wing)h(program)f(is)g(unsafe,)h(and)g(ma)o(y)e (succeed)j(or)e(fail,)h(dep)q(ending)i(on)d(implemen)o(tation.)75 1361 y Fh(CALL)23 b(MPI_RANK\(rank,)f(context\))75 1417 y(IF)i(\(rank.EQ.0\))e(THEN)147 1474 y(CALL)h(MPI_SENDB\(sendbuf,)e (len,)j(1,)f(tag,)h(context\))147 1530 y(CALL)f(MPI_RECVB\(recvbuf,)e (len,)j(1,)f(tag,)h(context\))75 1587 y(ELSE)95 b(!)24 b(rank.EQ.1)147 1643 y(CALL)f(MPI_SENDB\(sendbuf,)e(len,)j(0,)f(tag,)h (context\))147 1700 y(CALL)f(MPI_RECVB\(recvbuf,)e(len,)j(0,)f(tag,)h (context\))75 1756 y(END)f(IF)166 1919 y Fs(The)11 b(message)g(sen)o(t) g(b)o(y)g(eac)o(h)g(pro)q(cess)g(has)g(to)f(b)q(e)i(copied)g(out)f(b)q (efore)g(the)g(send)h(op)q(eration)f(returns)75 1975 y(and)k(the)g(receiv)o(e)h(op)q(eration)f(starts.)k(F)l(or)14 b(the)h(program)f(to)g(complete,)h(it)h(is)f(necessary)g(that)g(at)f (least)75 2032 y(one)i(of)g(the)h(t)o(w)o(o)e(messages)g(sen)o(t)h(is)h (bu\013ered)g(out)f(of)f(either)i(pro)q(cesses')g(address)f(space.)23 b(Th)o(us,)17 b(this)75 2088 y(program)c(can)i(succeed)g(only)g(if)g (the)f(comm)o(unication)h(system)f(has)g(su\016cien)o(t)h(bu\013er)g (space)f(to)g(bu\013er)75 2145 y Fh(len)h Fs(w)o(ords)f(of)h(data.)166 2201 y(If)g(additional)h(requiremen)o(ts)f(will)i(b)q(ecome)e(part)f (of)h(the)g(standard)f(\(e.g.,)f(b)q(ounds)j(on)f(the)f(mini-)75 2258 y(mal)i(n)o(um)o(b)q(er)f(of)g(concurren)o(tly)h(activ)o(e)f (handles)i(that)e(need)h(b)q(e)g(supp)q(orted,)f(then)h(further)f (programs)75 2314 y(b)q(ecome)h(safe.)p eop %%Page: 68 73 68 72 bop 75 356 a Fn(Chapter)34 b(7)75 564 y Fu(En)m(vironmen)m(tal)41 b(Inquiry)952 2828 y Fs(68)p eop %%Page: 69 74 69 73 bop 75 359 a Fn(Chapter)34 b(8)75 570 y Fu(Pro\014ling)75 814 y Fm(8.1)70 b(Requiremen)n(ts)75 921 y Fs(T)l(o)15 b(meet)g(the)g(MPI)g(pro\014ling)i(in)o(terface,)e(an)g(implemen)o (tation)i(of)d(the)i(MPI)f(functions)h Fi(must)131 1027 y Fs(1.)22 b(pro)o(vide)14 b(a)f(mec)o(hanism)h(through)f(whic)o(h)h (all)g(of)f(the)g(MPI)h(de\014ned)h(functions)f(ma)o(y)e(b)q(e)i (accessed)189 1083 y(with)e(a)g(name)f(shift.)20 b(Th)o(us)11 b(all)i(of)f(the)g(MPI)g(functions)h(\(whic)o(h)f(normally)g(start)f (with)h(the)g(pre\014x)189 1140 y(\\)p Fh(MPI)p 287 1140 15 2 v 16 w Fs("\))j(should)h(also)f(b)q(e)h(accessable)g(with)g(the)f (pre\014x)h(\\)p Fh(PMPI)p 1300 1140 V 16 w Fs(".)131 1246 y(2.)22 b(ensure)c(that)f(those)h(MPI)g(functions)h(whic)o(h)f (are)g(not)f(replaced)i(ma)o(y)f(still)h(b)q(e)g(link)o(ed)g(in)o(to)f (an)189 1302 y(executable)e(image)f(without)h(causing)f(name)h (clashes.)131 1408 y(3.)22 b(do)q(cumen)o(t)c(the)f(implemen)o(tation)i (of)e(di\013eren)o(t)h(language)g(bindings)h(of)e(the)h(MPI)g(in)o (terface)f(if)189 1464 y(they)g(are)h(la)o(y)o(ered)g(on)f(top)g(of)h (eac)o(h)f(other,)h(so)f(that)g(the)h(pro\014ler)g(dev)o(elop)q(er)h (kno)o(ws)e(whether)189 1521 y(she)d(m)o(ust)f(implemen)o(t)i(the)e (pro\014le)i(in)o(terface)f(for)f(eac)o(h)h(binding,)h(or)e(can)h (economise)h(b)o(y)e(imple-)189 1577 y(men)o(ting)i(it)h(only)f(for)g (the)g(lo)o(w)o(est)g(lev)o(el)h(routines.)75 1737 y Fm(8.2)70 b(Discussion)75 1845 y Fs(The)13 b(ob)s(jectiv)o(e)g(of)g (the)g(MPI)g(pro\014ling)h(in)o(terface)g(is)f(to)f(ensure)i(that)e(it) i(is)f(relativ)o(ely)h(easy)f(for)g(authors)75 1901 y(of)18 b(pro\014ling)i(\(and)f(other)f(similar\))h(to)q(ols)g(to)f(in)o (terface)h(their)g(co)q(des)g(to)f(MPI)h(implemen)o(tations)h(on)75 1958 y(di\013eren)o(t)15 b(mac)o(hines.)166 2017 y(Since)20 b(MPI)e(is)h(a)f(mac)o(hine)h(indep)q(enden)o(t)h(standard)e(with)h (man)o(y)f(di\013eren)o(t)g(implemen)o(tations,)75 2074 y(it)e(is)h(unreasonable)g(to)e(exp)q(ect)i(that)f(the)g(authors)f(of)h (pro\014ling)h(to)q(ols)f(for)g(MPI)g(will)i(ha)o(v)o(e)d(access)i(to) 75 2130 y(the)c(source)h(co)q(de)f(whic)o(h)h(implemen)o(ts)h(MPI)e(on) g(an)o(y)g(particular)h(mac)o(hine.)20 b(It)13 b(is)h(therefore)f (necessary)75 2187 y(to)21 b(pro)o(vide)i(a)f(mec)o(hanism)g(b)o(y)g (whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o(h)g(to)q(ols)g(can)g (collect)i(whatev)o(er)75 2243 y(p)q(erformance)15 b(information)h (they)f(wish)h Fi(without)g Fs(access)f(to)g(the)g(underlying)i (implemen)o(tation.)166 2302 y(W)l(e)d(b)q(eliev)o(e)i(that)e(ha)o (ving)g(suc)o(h)g(an)h(in)o(terface)f(is)h(imp)q(ortan)o(t)e(if)i(MPI)f (is)g(to)g(b)q(e)h(attractiv)o(e)e(to)g(end)75 2359 y(users,)i(since)i (the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g(di\013eren)o(t)h(to)q (ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h(factor)f(in)h(attracting) 75 2415 y(users)f(to)g(the)g(MPI)g(standard.)166 2475 y(The)e(pro\014ling)h(in)o(terface)e(is)h(just)g(that,)f(an)g(in)o (terface.)20 b(It)12 b(sa)o(ys)g Fi(nothing)g Fs(ab)q(out)g(the)h(w)o (a)o(y)f(in)h(whic)o(h)75 2531 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f(to)g(la)o(y)h(do)o(wn)g(what)g (information)g(is)h(collected)g(through)75 2588 y(the)g(in)o(terface,)g (or)g(ho)o(w)g(the)g(collected)i(information)e(is)h(sa)o(v)o(ed,)e (\014ltered,)i(or)f(displa)o(y)o(ed.)166 2647 y(While)g(the)g(initial)h (imp)q(etus)f(for)f(the)g(dev)o(elopmen)o(t)h(of)e(this)i(in)o(terface) f(arose)g(from)f(the)h(desire)i(to)75 2704 y(p)q(ermit)g(the)f (implemen)o(tation)i(of)d(pro\014ling)j(to)q(ols,)d(it)i(is)f(clear)h (that)f(an)g(in)o(terface)g(lik)o(e)h(that)f(sp)q(eci\014ed)952 2828 y(69)p eop %%Page: 70 75 70 74 bop 75 -100 a Fs(70)1180 b Fq(CHAPTER)15 b(8.)35 b(PR)o(OFILING)75 45 y Fs(ma)o(y)17 b(also)h(pro)o(v)o(e)f(useful)i (for)e(other)h(purp)q(oses,)g(suc)o(h)g(as)g(\\in)o(ternet)o(w)o (orking")f(m)o(ultiple)i(MPI)f(imple-)75 102 y(men)o(tations.)j(Since)c (all)f(that)f(is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there)h(is)g (no)f(ob)s(jection)h(to)f(its)h(b)q(eing)g(used)75 158 y(wherev)o(er)f(it)h(is)f(useful.)166 214 y(As)j(the)g(issues)h(b)q (eing)g(addressed)f(here)h(are)f(in)o(timately)h(tied)f(up)h(with)f (the)g(w)o(a)o(y)f(in)i(whic)o(h)g(ex-)75 271 y(ecutable)j(images)e (are)g(built,)j(whic)o(h)e(ma)o(y)f(di\013er)h(greatly)f(on)g (di\013eren)o(t)h(mac)o(hines,)h(the)e(examples)75 327 y(giv)o(en)c(b)q(elo)o(w)g(should)h(b)q(e)f(treated)g(solely)g(as)f (one)h(w)o(a)o(y)f(of)g(implemen)o(ting)j(the)d(ob)s(jectiv)o(e)h(of)f (the)h(MPI)75 384 y(pro\014ling)h(in)o(terface.)k(The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o(tation)h(are)e (those)g(detailed)75 440 y(in)i(the)f(Requiremen)o(ts)i(section)e(ab)q (o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this)i(c)o(hapter)f (is)g(only)h(presen)o(t)f(as)75 497 y(justi\014cation)g(and)f (discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.)166 553 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f(in)h (whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed)75 610 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 666 y(equally)d(v)m(alid\).)75 806 y Fm(8.3)70 b(Logic)22 b(of)i(the)e(design)75 907 y Fs(Pro)o(vided)16 b(that)f(an)g(MPI)g (implemen)o(tation)i(meets)e(the)h(requiremen)o(ts)g(ab)q(o)o(v)o(e,)e (it)i(is)g(p)q(ossible)h(for)e(the)75 964 y(implemen)o(tor)20 b(of)e(the)h(pro\014ling)h(system)f(to)f(in)o(tercept)i(all)f(of)g(the) g(MPI)g(calls)h(whic)o(h)f(are)g(made)g(b)o(y)75 1020 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h(collect)g(whatev)o (er)f(information)g(she)h(requires)g(b)q(efore)f(calling)75 1076 y(the)c(underlying)i(MPI)d(implemen)o(tation)i(\(through)f(its)g (name)f(shifted)i(en)o(try)e(p)q(oin)o(ts\))h(to)f(ac)o(hiev)o(e)i(the) 75 1133 y(desired)g(e\013ects.)75 1272 y Fm(8.4)70 b(Examples)75 1376 y Fj(8.4.1)55 b(Pro\014ler)18 b(implem)o(en)n(tation)75 1461 y Fs(Supp)q(ose)25 b(that)e(the)h(pro\014ler)h(wishes)g(to)e (accum)o(ulate)i(the)f(total)f(amoun)o(t)g(of)h(data)f(sen)o(t)h(b)o(y) g(the)75 1518 y(MPI)p 167 1518 14 2 v 16 w(SEND)18 b(function,)g(along) f(with)h(the)f(total)g(elapsed)h(time)f(sp)q(en)o(t)h(in)g(the)f (function.)27 b(This)18 b(could)75 1574 y(trivially)f(b)q(e)f(ac)o (hiev)o(ed)g(th)o(us)f(:{)166 1631 y Fj(W)-6 b(ARNING)18 b(:)d Fs(syn)o(tax)f(correction)h(needed)i(in)f(examples)g(when)g(MPI)f (syn)o(tax)f(is)i(de\014ned.)75 1709 y Fh(static)23 b(int)g (totalBytes;)75 1766 y(static)g(int)g(totalTime;)75 1822 y(extern)g(int)g(now\(void\);)71 b(/*)23 b(Some)h(suitable)e(timer)i (function)e(!)i(*/)75 1935 y(int)f(MPI_SEND\(const)f(int)i(tag,)f (const)g(int)h(bytes,)f(void)g(*)h(buffer,)f(const)g(int)g(who\))75 1992 y({)147 2048 y(int)g(tstart)g(=)h(now\(\);)147 2105 y(int)f(result)g(=)h(PMPI_SEND\(tag,bytes,buffe)o(r,who\);)68 b(/*)24 b(Pass)f(on)h(all)f(the)g(arguments)g(*/)147 2217 y(totalBytes)f(+=)i(bytes;)405 b(/*)23 b(Accumulate)g(byte)g (count)g(*/)147 2274 y(totalTime)46 b(+=)24 b(now\(\))f(-)h(tstart;)190 b(/*)23 b(and)h(time)333 b(*/)147 2387 y(return)23 b(result;)75 2443 y(})75 2561 y Fj(8.4.2)55 b(MPI)19 b(library)f(implem)o(en)n (tation)75 2647 y Fs(On)g(a)f(Unix)h(system,)f(in)h(whic)o(h)g(the)g (MPI)f(library)h(is)g(implemen)o(ted)h(in)f(C,)f(then)g(there)h(are)e (v)m(arious)75 2704 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i(t)o (w)o(o)e(of)g(the)h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.)35 b(Whic)o(h)20 b(is)h(b)q(etter)p eop %%Page: 71 76 71 75 bop 75 -100 a Fq(8.4.)34 b(EXAMPLES)1393 b Fs(71)75 45 y(dep)q(ends)17 b(on)e(whether)g(the)h(link)o(er)g(and)f(compiler)i (supp)q(ort)e(w)o(eak)g(sym)o(b)q(ols.)75 165 y Ft(Systems)h(with)i(w)o (eak)f(sym)o(b)q(ols)75 250 y Fs(If)i(the)f(compiler)i(and)f(link)o(er) h(supp)q(ort)f(w)o(eak)f(external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g(system)75 307 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h(library)g(is)f(required)h (through)f(the)g(use)g(of)g Fh(#pragma)23 b(weak)13 b Fs(th)o(us)75 363 y(:{)75 465 y Fh(#pragma)23 b(weak)g(MPI_EXAMPLE)g(=) g(PMPI_EXAMPLE)75 578 y(int)g(PMPI_EXAMPLE\(/*)f(appropriate)h(args)g (*/\))75 634 y({)170 691 y(/*)h(Useful)f(content)g(*/)75 747 y(})166 849 y Fs(The)d(e\013ect)g(of)g(this)g Fh(#pragma)f Fs(is)i(to)e(de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Fh(MPI)p 1472 849 15 2 v 17 w(EXAMPLE)f Fs(as)h(a)g(w)o(eak)75 906 y(de\014nition,)d(this)f(means)g(that)f(the)g(link)o(er)i(will)g (not)e(complain)i(if)f(there)g(is)g(another)f(de\014nition)j(of)d(the) 75 962 y(sym)o(b)q(ol)g(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1019 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1138 y Ft(Systems)h(without)j(w)o(eak)d(sym)o(b)q (ols)75 1224 y Fs(In)j(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h (then)g(one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h (the)f(C)h(macro)75 1280 y(pre-pro)q(cessor)c(th)o(us)g(:{)75 1382 y Fh(#ifdef)23 b(PROFILELIB)75 1438 y(#)95 b(ifdef)23 b(__STDC__)75 1495 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1551 y(#)95 b(else)75 1608 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 1664 y(#)95 b(endif)75 1721 y(#else)75 1777 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 1834 y(#endif)166 1935 y Fs(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2037 y Fh(int)23 b(FUNCTION\(MPI_EXAMPLE\)\(/*)e (appropriate)h(args)i(*/\))75 2094 y({)170 2150 y(/*)g(Useful)f (content)g(*/)75 2207 y(})166 2308 y Fs(The)d(same)g(source)g(\014le)h (can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q(duce)i(b)q(oth)f(v)o (ersions)g(of)g(the)g(library)l(,)75 2365 y(dep)q(ending)d(on)f(the)f (state)f(of)h(the)g Fh(PROFILELIB)f Fs(macro)h(sym)o(b)q(ol.)166 2421 y(It)e(is)g(required)h(that)e(the)h(standard)f(MPI)h(library)g(b)q (e)h(built)g(in)f(suc)o(h)g(a)g(w)o(a)o(y)e(that)h(the)h(inclusion)i (of)75 2478 y(MPI)g(functions)g(can)g(b)q(e)h(ac)o(hiev)o(ed)f(one)g (at)g(a)f(time.)20 b(This)c(is)f(a)f(somewhat)g(unpleasan)o(t)i (requiremen)o(t,)75 2534 y(since)22 b(it)f(ma)o(y)f(mean)h(that)f(eac)o (h)g(external)i(function)f(has)g(to)f(b)q(e)h(compiled)i(from)d(a)g (separate)g(\014le.)75 2591 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g (necessary)g(so)f(that)g(the)g(author)g(of)g(the)h(pro\014ling)g (library)g(need)h(only)f(de\014ne)g(those)75 2647 y(MPI)f(functions)g (whic)o(h)g(she)g(wishes)h(to)e(in)o(tercept,)h(references)g(to)f(an)o (y)g(others)g(b)q(eing)i(ful\014lled)h(b)o(y)e(the)75 2704 y(normal)g(MPI)g(library)l(.)21 b(Therefore)15 b(the)h(link)g (step)f(can)h(lo)q(ok)f(something)h(lik)o(e)g(this)g(:{)p eop %%Page: 72 77 72 76 bop 75 -100 a Fs(72)1180 b Fq(CHAPTER)15 b(8.)35 b(PR)o(OFILING)75 45 y Fh(\045)24 b(cc)f(...)h(-lmyprof)f(-lpmpi)g (-lmpi)166 151 y Fs(Here)e Fh(libmyprof.a)e Fs(con)o(tains)i(the)g (pro\014ler)g(functions)h(whic)o(h)f(in)o(tercept)g(some)g(of)f(the)h (MPI)75 208 y(functions.)j Fh(libpmpi.a)15 b Fs(con)o(tains)i(the)f (\\name)h(shifted")g(MPI)f(functions,)h(and)g Fh(libmpi.a)e Fs(con)o(tains)75 264 y(the)g(normal)g(de\014nitions)i(of)e(the)g(MPI)h (functions.)75 407 y Fm(8.5)70 b(Multiple)20 b(lev)n(els)g(of)k(in)n (terception)75 509 y Fs(The)15 b(sc)o(heme)h(giv)o(en)g(here)f(do)q(es) h(not)f(directly)h(supp)q(ort)f(the)h(nesting)g(of)e(pro\014ling)j (functions,)e(since)i(it)75 565 y(pro)o(vides)f(only)f(a)g(single)h (alternativ)o(e)g(name)f(for)f(eac)o(h)h(MPI)g(function.)21 b(Consideration)16 b(w)o(as)e(giv)o(en)i(to)75 622 y(an)f(implemen)o (tation)i(whic)o(h)f(w)o(ould)g(allo)o(w)g(m)o(ultiple)h(lev)o(els)g (of)e(call)h(in)o(terception,)h(ho)o(w)o(ev)o(er)d(w)o(e)h(w)o(ere)75 678 y(unable)j(to)e(construct)g(an)g(implemen)o(tation)i(of)e(this)h (whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o(wing)i(disadv)m(an-) 75 735 y(tages)143 829 y Fo(\017)23 b Fs(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 922 y Fo(\017)23 b Fs(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o(en) g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 1016 y(Since)i(one)e(of)g(the)h(ob)s(jectiv)o(es)f(of)g(MPI)h(is)f(to)g (p)q(ermit)h(e\016cien)o(t,)g(lo)o(w)f(latency)h(implemen)o(tations,)h (and)75 1073 y(it)j(is)f(not)g(the)h(business)g(of)f(a)g(standard)g(to) g(require)h(a)f(particular)h(implemen)o(tation)g(language,)g(w)o(e)75 1129 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h(outlined)h(ab)q(o)o(v) o(e.)166 1186 y(Note,)f(ho)o(w)o(ev)o(er,)f(that)h(it)h(is)f(p)q (ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o(v)o(e)g(to)g(implemen) o(t)i(a)e(m)o(ulti-lev)o(el)75 1242 y(system,)k(since)g(the)g(function) g(called)i(b)o(y)d(the)h(user)f(ma)o(y)g(call)i(man)o(y)e(di\013eren)o (t)g(pro\014ling)i(functions)75 1299 y(b)q(efore)15 b(calling)i(the)f (underlying)h(MPI)e(function.)166 1355 y(Unfortunately)22 b(suc)o(h)g(an)g(implemen)o(tation)h(ma)o(y)e(require)h(more)g(co-op)q (eration)g(b)q(et)o(w)o(een)g(the)75 1411 y(di\013eren)o(t)g (pro\014ling)i(libraries)g(than)e(is)h(required)g(for)e(the)i(single)g (lev)o(el)h(implemen)o(tation)f(detailed)75 1468 y(ab)q(o)o(v)o(e.)p eop %%Page: 73 78 73 77 bop 75 359 a Fn(Chapter)34 b(9)75 569 y Fu(Maps,)39 b(Groups)h(and)f(Con)m(texts)431 794 y Fs(Lyndon)16 b(Clark)o(e,)e (Mark)h(Sears,)f(T)l(on)o(y)h(Skjellum,)i(Marc)d(Snir)75 952 y Fm(9.1)70 b(In)n(tro)r(duction)75 1059 y Fs(MPI)20 b(pro)o(vides)h(supp)q(ort)g(for)f(the)g(execution)i(of)e Ft(parallel)25 b(pro)q(cedures)p Fs(.)36 b(A)20 b(parallel)i(pro)q (cedure)75 1115 y(is)f(executed)g(collectiv)o(ely)h(b)o(y)e(a)g(set)g (of)g(comm)o(unicating)h(pro)q(cesses.)35 b(T)l(ransfer)20 b(of)f(con)o(trol)h(to)g(the)75 1172 y(parallel)d(pro)q(cedure,)e(and)h (bac)o(k,)e(is)i(ac)o(hiev)o(ed)g(b)o(y)f(ha)o(ving)g(eac)o(h)g (executing)i(pro)q(cess)e(transfer)f(con)o(trol)75 1228 y(to)h(the)i(lo)q(cal)g(pro)q(cedure)g(co)q(de,)f(and)h(return)f(from)f (it.)23 b(Not)15 b(all)i(pro)q(cesses)g(need)g(transfer)e(con)o(trol)h (to)75 1284 y(the)f(same)g(parallel)h(pro)q(cedure;)g(the)f(parallel)h (pro)q(cedure)g(ma)o(y)f(in)o(v)o(olv)o(e)g(only)h(a)e(subset)h(of)g (pro)q(cesses,)75 1341 y(or)g(di\013eren)o(t)i(parallel)g(calls)g(ma)o (y)f(b)q(e)g(executed)h(b)o(y)f(di\013eren)o(t)g(subsets)h(of)e(pro)q (cesses.)23 b(The)16 b(collectiv)o(e)75 1397 y(comm)o(unication)h (calls)h(pro)o(vided)f(b)o(y)g(MPI)f(are)g(examples)i(of)e(suc)o(h)h (parallel)h(pro)q(cedures.)25 b(Libraries)75 1454 y(suc)o(h)16 b(as)e(parallel)j(scien)o(ti\014c)g(libraries)g(will)g(b)q(e)e(another) g(example.)166 1513 y(It)d(is)h(highly)h(desirable)g(to)e(allo)o(w)g (the)h(pro)q(cesses)g(that)e(execute)j(a)e(parallel)i(pro)q(cedure)f (use)g(a)f(\\vir-)75 1569 y(tual)k(pro)q(cess)g(name)f(space")h(lo)q (cal)g(to)f(the)h(in)o(v)o(ok)m(ation.)21 b(Th)o(us,)15 b(the)h(co)q(de)g(of)f(the)h(parallel)h(pro)q(cedure)75 1626 y(will)j(lo)q(ok)e(iden)o(tical,)i(irresp)q(ectiv)o(e)f(of)f(the)g (absolute)g(addresses)g(of)f(the)h(executing)h(pro)q(cesses.)29 b(It)17 b(is)75 1682 y(often)e(the)f(case)h(that)f(parallel)j (application)f(co)q(de)g(is)f(built)h(b)o(y)f(comp)q(osing)g(sev)o (eral)g(parallel)i(mo)q(dules)75 1739 y(\(e.g.,)e(a)h(n)o(umerical)i (solv)o(er,)e(and)h(a)f(graphic)h(displa)o(y)g(mo)q(dule\).)24 b(Supp)q(ort)17 b(of)f(a)g(virtual)h(name)g(space)75 1795 y(for)j(eac)o(h)g(mo)q(dule)i(will)g(allo)o(w)f(to)e(comp)q(ose)i (mo)q(dules)g(that)f(w)o(ere)g(dev)o(elop)q(ed)i(separately)f(without) 75 1852 y(c)o(hanging)d(all)g(message)f(passing)g(calls)h(within)h(eac) o(h)e(mo)q(dule.)27 b(The)17 b(set)g(of)g(pro)q(cesses)h(that)e (execute)75 1908 y(a)i(parallel)i(pro)q(cedure)f(ma)o(y)f(b)q(e)h (\014xed,)g(or)f(ma)o(y)g(b)q(e)h(determined)g(dynamically)i(b)q(efore) d(the)h(in)o(v)o(ok)m(a-)75 1965 y(tion.)g(Th)o(us,)13 b(MPI)h(has)e(to)h(pro)o(vide)g(a)g(mec)o(hanism)h(for)e(creating)i (dynamically)g(sets)f(of)g(lo)q(cally)i(named)75 2021 y(pro)q(cesses.)35 b(W)l(e)20 b(alw)o(a)o(ys)f(n)o(um)o(b)q(er)h(pro)q (cesses)h(that)e(execute)h(a)g(parallel)i(pro)q(cedure)f(consecutiv)o (ely)l(,)75 2077 y(starting)15 b(form)g(zero.)21 b(Th)o(us,)16 b(a)f Ft(group)h Fs(is)g(an)f(ordered)h(set)g(of)f(pro)q(cesses.)22 b(Eac)o(h)15 b(pro)q(cess)h(in)h(a)e(group)75 2134 y(is)h(asso)q (ciated)f(with)h(a)e Ft(rank)p Fs(,)h(starting)f(from)h(zero.)20 b(Pro)q(cesses)15 b(are)g(iden)o(ti\014ed)i(b)o(y)e(their)h(ranks)e (when)75 2190 y(comm)o(unication)i(o)q(ccurs)f(within)i(the)e(group.) 166 2249 y(Another)g(imp)q(ortan)o(t)g(goal)g(is)g(the)h(supp)q(ort)f (of)g(a)f Ft(lo)q(osely)19 b(sync)o(hronous)13 b Fs(transfer)i(of)g (con)o(trol:)75 2306 y(no)f(sync)o(hronization)g(o)q(ccurs)h(either)f (b)q(efore)g(or)f(after)h(the)f(call.)21 b(Th)o(us,)14 b(a)f(pro)q(cess)h(ma)o(y)f(start)g(sending)75 2362 y(messages)g(that)g (p)q(ertain)h(to)f(the)g(execution)i(of)e(a)g(parallel)i(pro)q(cedure)g (b)q(efore)e(all)i(other)e(participating)75 2419 y(pro)q(cesses)k(ha)o (v)o(e)f(joined)i(the)e(execution;)i(a)e(pro)q(cess)h(ma)o(y)f(receiv)o (e)i(a)e(message)g(that)g(p)q(ertains)h(to)f(the)75 2475 y(execution)e(of)e(a)g(parallel)i(pro)q(cedure,)g(while)g (participating)f(in)h(another)e(parallel)i(execution.)20 b(A)13 b Ft(com-)75 2532 y(m)o(unication)20 b(con)o(text)c Fs(mec)o(hanism)h(is)f(needed)i(to)d(distinguish)j(comm)o(unication)f (that)f(b)q(elongs)h(to)75 2588 y(distinct)f(parallel)h(pro)q(cedure)f (executions.)21 b(\(Ev)o(en)15 b(if)g(parallel)i(transfer)d(of)h(con)o (trol)g(is)h(executed)g(syn-)75 2645 y(c)o(hroneously)l(,)f(one)g (still)i(needs)e(a)g(con)o(text)f(mec)o(hanism)h(for)g(the)f(sync)o (hronization)i(calls)g(themselv)o(es.\))166 2704 y(Normally)l(,)24 b(a)e(parallel)h(pro)q(cedure)g(is)g(written)f(so)g(that)f(all)i (messages)f(pro)q(duced)h(during)g(its)952 2828 y(73)p eop %%Page: 74 79 74 78 bop 75 -100 a Fs(74)694 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y Fs(execution)i(are)e (also)h(consumed)g(b)o(y)g(the)g(pro)q(cesses)g(that)f(execute)h(the)g (pro)q(cedure.)22 b(Ho)o(w)o(ev)o(er,)15 b(if)h(one)75 102 y(parallel)k(pro)q(cedure)g(calls)f(another,)g(then)g(it)g(migh)o (t)f(b)q(e)i(desirable)g(to)e(allo)o(w)g(suc)o(h)h(call)h(to)e(pro)q (ceed)75 158 y(while)k(messages)e(are)h(p)q(ending)h(\(the)e(messages)g (will)j(b)q(e)e(consumed)g(b)o(y)g(the)f(pro)q(cedure)i(after)e(the)75 214 y(call)e(returns\).)24 b(In)18 b(suc)o(h)f(case,)f(a)h(new)g(comm)o (unication)g(con)o(text)g(is)g(needed)h(for)e(the)h(called)h(parallel) 75 271 y(pro)q(cedure,)e(ev)o(en)f(if)h(the)f(transfer)g(of)f(con)o (trol)h(is)h(sync)o(hronized.)166 327 y(A)d Ft(con)o(text)h Fs(is)g(the)f(MPI)g(mec)o(hanism)h(for)f(partitioning)h(comm)o (unication)g(space.)20 b(A)13 b(send)h(made)75 384 y(in)k(a)f(con)o (text)g(cannot)g(b)q(e)i(receiv)o(ed)f(in)g(another)g(con)o(text.)26 b(Con)o(texts)16 b(are)h(iden)o(ti\014ed)j(in)e(MPI)f(using)75 440 y(in)o(teger-v)m(alued)g Ft(con)o(text)p 540 440 16 2 v 19 w(id)p Fs('s.)166 497 y(The)d(comm)o(unication)i(domain)e (used)h(b)o(y)f(a)g(parallel)i(pro)q(cedure)g(is)e(iden)o(ti\014ed)j(b) o(y)d(a)g Ft(comm)o(uni-)75 553 y(cator)p Fs(.)29 b(Comm)o(unicators)17 b(bring)h(together)f(the)h(concepts)h(of)e(pro)q(cess)h(group)g(and)g (comm)o(unication)75 610 y(con)o(text.)29 b(A)18 b(comm)o(unicator)g (is)h(an)f(explicit)j(parameter)c(in)j(eac)o(h)e(p)q(oin)o(t-to-p)q (oin)o(t)h(comm)o(unication)75 666 y(op)q(eration.)31 b(The)19 b(comm)o(unicator)f(iden)o(ti\014es)j(the)d(comm)o(unication)i (con)o(text)e(of)g(that)g(op)q(eration;)j(it)75 723 y(iden)o(ti\014es)f (the)f(group)f(of)g(pro)q(cesses)g(that)g(can)g(b)q(e)h(in)o(v)o(olv)o (ed)h(in)f(this)g(comm)o(unication;)h(and)e(it)h(pro-)75 779 y(vides)g(the)g(translation)f(from)g(virtual)h(pro)q(cess)g(names,) g(whic)o(h)g(are)f(ranks)g(within)i(the)e(group,)h(in)o(to)75 835 y(absolute)13 b(addresses.)20 b(Collectiv)o(e)14 b(comm)o(unication)g(calls)g(also)f(tak)o(e)f(a)h(comm)o(unicator)f(as) h(parameter;)75 892 y(it)k(is)h(exp)q(ected)g(that)e(parallel)j (libraries)g(will)f(b)q(e)g(built)g(to)f(accept)g(a)g(comm)o(unicator)f (as)h(parameter.)75 948 y(Comm)o(unicators)d(are)h(represen)o(ted)h(b)o (y)f(opaque)g(MPI)h(ob)s(jects.)166 1005 y(MPI)g(do)q(es)h(not)e(pro)o (vide)i(for)f(absolute)g(pro)q(cess)h(names.)22 b(Rather,)17 b(pro)q(cesses)f(are)g(alw)o(a)o(ys)g(iden-)75 1061 y(ti\014ed)h(b)o(y) g(their)g(rank)f(inside)i(a)e(group.)23 b(A)16 b(univ)o(ersal)i(group)e (that)g(includes)i(all)g(pro)q(cesses)e(a)o(v)m(ailable)75 1118 y(when)k(computation)f(starts)f(is)i(the)g(nearest)f(equiv)m(alen) o(t)i(to)e(an)g(absolute)h(pro)q(cess)g(name)f(space)h(in)75 1174 y(MPI.)i(Relativ)o(e)i(naming)e(is)h(consisten)o(t)g(with)f(MPI)h (implemen)o(tations)g(where)g(pro)q(cesses)g(can)f(b)q(e)75 1231 y(added)16 b(dynamically)l(.)166 1287 y(New)j(pro)q(cess)g(groups) f(are)h(built)h(b)o(y)f(subsetting)g(and)g(reordering)g(pro)q(cesses)g (within)h(existing)75 1344 y(groups)i(\(as)f(de\014ned)j(b)o(y)e(comm)o (unicators\).)40 b(A)23 b(new)f(group)g(is)h(de\014ned)h(from)d(an)h (old)h(group)f(b)o(y)75 1400 y(sp)q(ecifying)d(the)f(corresp)q(ondence) h(b)q(et)o(w)o(een)e(the)h(rank)f(of)g(eac)o(h)h(pro)q(cess)f(in)i(the) e(new)h(group)f(and)h(its)75 1456 y(rank)f(in)h(the)f(old)h(group.)26 b(Suc)o(h)18 b(corresp)q(ondence)g(is)g(called)h(a)e Ft(map)p Fs(;)h(it)f(is)h(represen)o(ted)g(in)g(MPI)f(b)o(y)75 1513 y(an)e(opaque)g(ob)s(ject.)75 1656 y Fm(9.2)70 b(Maps)75 1757 y Fs(A)19 b Ft(map)g Fs(is)g(a)g(one-to)f(one-mapping)i(from)e(0)p Fr(:::m)11 b Fo(\000)i Fs(1)19 b(in)o(to)g(the)g(set)f(of)h(non)g (negativ)o(e)g(in)o(tegers;)h Fr(m)75 1814 y Fs(is)e(the)g Ft(size)g Fs(of)f(map)g Fr(f)5 b Fs(.)28 b(A)17 b(p)q(ossible)j (represen)o(tation)d(for)g(suc)o(h)h(map)g(is)g(a)f(list)h(of)g Fr(m)f Fs(in)o(tegers;)i(it)e(is)75 1870 y(often)d(con)o(v)o(enien)o(t) h(to)e(think)j(of)d(a)h(map)g(as)g(b)q(eing)i(suc)o(h)f(list.)20 b(Another)14 b(p)q(ossible)i(represen)o(tation)f(is)f(as)75 1927 y(an)j(algorithmic)g(sp)q(eci\014cation)i(of)d(the)h(map)f (\(e.g.,)g(hash)h(function\).)25 b(A)16 b(map)h(is)g(represen)o(ted)g (b)o(y)g(an)75 1983 y(opaque)e(map)g(ob)s(ject.)166 2040 y(Maps)e(are)g(used)h(in)h(order)e(to)g(represen)o(t)h(groups.)19 b(If)14 b(pro)q(cesses)g(are)f(n)o(um)o(b)q(ered)h(from)f(0)g(to)g Fr(n)7 b Fo(\000)g Fs(1,)75 2096 y(then)15 b(a)f(map)g Fr(f)k Fs(:)12 b(0)p Fr(:::m)7 b Fo(\000)i Fs(1)j Fo(!)h Fs(0)p Fr(:::n)7 b Fo(\000)i Fs(1)14 b(represen)o(ts)h(a)f(subset)g (group)g(of)g Fr(m)g Fs(pro)q(cesses:)20 b(pro)q(cess)15 b Fr(f)5 b Fs(\()p Fr(i)p Fs(\))75 2153 y(has)16 b(rank)f Fr(i)h Fs(in)h(the)f(new)g(group.)22 b(A)16 b(map,)f(b)o(y)h(itself,)h (do)q(es)f(not)f(represen)o(t)h(a)g(group;)f(it)i(do)q(es)f(so)f(only) 75 2209 y(relativ)o(e)g(to)e(a)h(pre-de\014ned)i(con)o(taining)f (group,)f(i.e.,)g(relativ)o(e)h(to)e(another)h(ranking)h(of)f(the)g (pro)q(cesses.)166 2342 y Fl(Discussion:)51 b Fk(Should)17 b(decide)i(if)f(opaque)g(ob)r(jects)h(are)g(justi\014ed,)g(or)f (whether)h(an)f(arra)o(y)g(of)g(indices)75 2391 y(is)f(OK.)h(On)g(the)g (\\opaque)f(ob)r(ject")h(side:)26 b(Ma)o(y)17 b(ha)o(v)o(e)g (algorithmic)e(map)h(de\014nitions)h(and)g(ma)o(y)f(spread)i(the)75 2441 y(de\014nition)f(o)o(v)o(er)g(more)g(than)g(one)h(no)q(de,)g(on)f (a)g(v)o(ery)g(large)h(system.)28 b(On)17 b(the)h(explicit)f(represen)o (tation)i(side:)75 2491 y(The)g(represen)o(tation)h(is)e(simple,)g(w)o (e)h(a)o(v)o(oid)e(an)h(additional)f(ob)r(ject,)i(and)g(there)h(is)e (no)g(m)o(uc)o(h)f(w)o(aste.)33 b(Also,)75 2541 y(with)13 b(an)g(explicit)f(represen)o(tation,)j(the)f(user)g(can)f(easily)g(add) g(his/her)g(o)o(wn)g(map)f(constructors,)i(and)f(building)75 2591 y(comm)o(unicators)e(is)j(faster.)19 b(Need)c(to)f(understand)h (what)e(an)h(opaque)g(map)e(w)o(ould)h(b)q(e)i(in)e(F)m(ortran.)166 2647 y(If)j(maps)f(are)i(explictly)e(represen)o(ted)20 b(as)c(arra)o(ys,)h(then)g(one)f(needs)i(map)d(size)i(to)f(b)q(e)h(an)f (additional)e(pa-)75 2704 y(rameter.)p eop %%Page: 75 80 75 79 bop 75 -100 a Fq(9.2.)34 b(MAPS)1517 b Fs(75)166 45 y(F)l(or)16 b(eac)o(h)g Fh(m)p Fs(,)h(an)f(iden)o(tit)o(y)i(map)e Fh(MPI)p 817 45 15 2 v 17 w(IDENT\(m\))f Fs(of)h(size)i Fh(m)e Fs(is)h(prede\014ned:)24 b Fh(MPI)p 1581 45 V 17 w(IDENT)p Fs(\()p Fh(m)p Fs(\))o(\()p Fr(i)p Fs(\))14 b(=)75 102 y Fr(i)h(;)8 b(i)j Fs(=)i(0)p Fr(;)8 b(:::;)g(m)f Fo(\000)k Fs(1.)166 241 y Fl(Discussion:)43 b Fk(If)16 b(maps)f(are)h(non)g(opaque)g(then)h(the)g(iden)o(t)f(map)e(is)i(just)h (a)f(long)f(arra)o(y)h(with)g Fb(i)g Fk(stored)75 297 y(at)h(the)g Fb(i)p Fk(-th)g(en)o(try)m(,)g(and)f(it)h(is)f(just)h Fg(MPI)p 733 297 14 2 v 15 w(IDENT)p Fk(.)e(If)h(maps)g(are)h(opaque,)g (w)o(e)g(need)g(a)f(di\013eren)o(t)i(map)d(for)h(eac)o(h)75 354 y(domain)c(size)i(\(a)g(function)g(from)e(in)o(tegers)i(to)g (maps?\))j(Need)e(to)f(think)f(of)h(F)m(ortran)f(implemen)o(tation.)75 555 y Fj(9.2.1)55 b(Op)r(erations)18 b(on)h(maps)75 641 y Fs(The)12 b(follo)o(wing)h(op)q(erations)f(are)g(de\014ned)h(on)f (maps.)19 b(Eac)o(h)12 b(function)h(is)f(in)o(v)o(ok)o(ed)h(lo)q(cally) g(b)o(y)f(a)g(pro)q(cess.)166 733 y Ft(MPI)p 275 733 16 2 v 18 w(MAP)p 419 733 V 18 w(SIZE\(map,)17 b(size\))75 856 y(IN)g(map)23 b Fs(handle)16 b(to)f(map)g(ob)s(ject)75 945 y Ft(OUT)j(size)23 b Fs(size)16 b(of)f(map)g(\(in)o(teger\))166 1068 y Ft(MPI)p 275 1068 V 18 w(MAP)p 419 1068 V 18 w(APPL)l(Y\()h (map,)h(argumen)o(t,)f(v)m(alue\))75 1191 y(IN)h(map)23 b Fs(handle)16 b(to)f(map)g(ob)s(ject)75 1279 y Ft(IN)i(argumen)o(t)22 b Fs(map)15 b(argumen)o(t)g(\(in)o(teger\))75 1368 y Ft(OUT)j(v)m(alue)24 b Fs(image)15 b(of)g Fh(argument)f Fs(under)i Fh(map)e Fs(\(in)o(teger\))166 1491 y Ft(MPI)p 275 1491 V 18 w(MAP)p 419 1491 V 18 w(INVERSE\()k(map,)e(argumen)o(t,)h (v)m(alue\))75 1614 y(IN)g(map)23 b Fs(handle)16 b(to)f(map)g(ob)s (ject)75 1702 y Ft(OUT)j(argumen)o(t)23 b Fs(in)o(v)o(erse)14 b(image)g(of)g Fh(value)f Fs(under)i Fh(map)p Fs(;)e(-1)h(if)h Fh(value)e Fs(is)i(not)e(in)i(the)f(range)g(of)g Fh(map)p Fs(.)75 1791 y Ft(IN)j(v)m(alue)23 b Fs(map)16 b(v)m(alue)g(\(in)o (teger\))166 1914 y Ft(MPI)p 275 1914 V 18 w(MAP)p 419 1914 V 18 w(LIST\(map,)i(len,)f(arra)o(y)p 924 1914 V 18 w(of)p 984 1914 V 19 w(in)o(teger,)g(size\))166 2006 y Fs(Creates)d(an)i(explicit)h(list)f(represen)o(tation)f(of)g(a)g (map.)75 2086 y Ft(IN)i(map)23 b Fs(handle)16 b(to)f(map)g(ob)s(ject)75 2175 y Ft(IN)i(LEN)24 b Fs(length)15 b(of)g(arra)o(y)f(\(in)o(teger\)) 75 2263 y Ft(OUT)k(arra)o(y)p 332 2263 V 18 w(of)p 392 2263 V 19 w(in)o(teger)23 b Fs(list)16 b(of)f(v)m(alues)h(in)g(the)f (range)g(of)g Fh(map)75 2351 y Ft(OUT)j(size)23 b Fs(length)16 b(of)f(returned)h(list)g({)e(size)j(of)d(map)h(\(in)o(teger\).)166 2508 y Fl(Discussion:)166 2564 y Fk(I)i(c)o(hanged)h(terminology)c (from)i(\\elemen)o(t,)h(rank")f(to)h(\\argumen)o(t,)g(v)n(alue")f (since)i(b)q(oth)f(argumen)o(t)f(and)75 2621 y(v)n(alue)d(are)i(ranks.) p eop %%Page: 76 81 76 80 bop 75 -100 a Fs(76)694 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y Fj(9.2.2)55 b(Map)19 b(constructors)75 132 y Fs(The)c(map)g(constructors)g(ma)o(y)f (b)q(e)i(either)g(lo)q(cal)g(or)f(ma)o(y)g(require)h(comm)o(unication.) 166 224 y Ft(MPI)p 275 224 16 2 v 18 w(MAP)p 419 224 V 18 w(BUILD\(map,)i(arra)o(y)p 877 224 V 17 w(of)p 936 224 V 19 w(in)o(teger,)f(size\))166 317 y Fs(Build)g(a)d(map)g(from)g (an)h(explicit)i(list)f(represen)o(tation.)j(This)d(function)f(is)g (called)i(lo)q(cally)f(b)o(y)f(one)75 373 y(pro)q(cess.)75 469 y Ft(OUT)j(map)23 b Fs(handle)16 b(to)f(map)g(ob)s(ject)75 565 y Ft(IN)i(arra)o(y)p 277 565 V 18 w(of)p 337 565 V 19 w(in)o(teger)23 b Fs(list)16 b(of)e(v)m(alues)j(in)f(the)f(range)g (of)g Fh(map)75 660 y Ft(IN)i(size)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(arra)o(y)e({)h(map)g(size)h(\(in)o(teger\))166 792 y Ft(MPI)p 275 792 V 18 w(MAP)p 419 792 V 18 w(SPLIT\(comm,)h(map,) g(k)o(ey)l(,)e(index,)i(newmap\))166 884 y Fs(This)22 b(is)g(a)f(collectiv)o(e)j(function)e(that)f(is)h(called)h(b)o(y)f(all) g(pro)q(cesses)g(in)h(the)e(group)h(asso)q(ciated)75 941 y(with)17 b Fh(\(comm,)23 b(map\))p Fs(.)g(All)18 b(calling)g(pro)q(cesses)f(pro)o(vide)g(the)g(same)f(v)m(alues)h(for)f (the)h(parameters)f Fh(comm)75 997 y Fs(and)h Fh(map)p Fs(.)25 b(A)17 b(separate)f(group)g(of)h(pro)q(cesses)g(is)h(formed)e (for)h(eac)o(h)f(distinct)j(v)m(alue)f(of)e Fh(key)p Fs(,)h(with)g(the)75 1053 y(pro)q(cesses)f(ordered)g(according)g(to)f (the)h(v)m(alue)h(of)e Fh(index)p Fs(.)21 b(Eac)o(h)15 b(pro)q(cess)h(in)h(suc)o(h)f(group)f(is)i(returned)75 1110 y(in)f Fh(newmap)e Fs(the)i(map)f(of)f(this)i(group)f(within)h Fh(comm)p Fs(.)75 1219 y Ft(IN)h(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1314 y Ft(IN)i(map)23 b Fs(map)15 b(ob)s(ject)g(handle)75 1410 y Ft(IN)i(k)o(ey)22 b Fs(\(in)o(teger\))75 1506 y Ft(IN)17 b(index)23 b Fs(\(in)o(teger\))75 1602 y Ft(OUT)18 b(newmap)23 b Fs(map)15 b(ob)s(ject)f(handle)166 1711 y(Additional)k(map)f(creation)g(functions)g(ma)o(y)f(b)q(e)h(pro)o (vided.)25 b(F)l(ollo)o(ws)17 b(a)f(p)q(ossible)i(list)g(construc-)75 1767 y(tors.)166 1907 y Fl(Discussion:)d Fk(Do)f(w)o(e)g(w)o(an)o(t)f (to)h(mandate)f(them)g(in)g(MPI?)166 2082 y Ft(MPI)p 275 2082 V 18 w(MAP)p 419 2082 V 18 w(UNION\()k(map1,)h(map2,)f (newmap\))166 2174 y Fh(map1)g Fs(and)h Fh(map2)f Fs(are)h(maps)f(with) i(disjoin)o(t)f(ranges.)27 b(The)18 b(resulting)h(concatenation)f(is)h (a)e(map)75 2231 y(with)d(a)g(range)g(whic)o(h)h(is)f(the)g(union)h(of) f(the)g(ranges)g(of)f Fh(map1)h Fs(and)g Fh(map2)p Fs(,)f(with)i (elemen)o(ts)f(in)h(the)f(range)75 2287 y(of)h Fh(map1)f Fs(listed)i(\014rst,)e(and)h(original)h(order)f(preserv)o(ed)g(within)i (eac)o(h)e(range.)k(The)c(union)h(op)q(eration)f(is)75 2343 y(asso)q(ciativ)o(e)h(but)f(not)g(comm)o(utativ)o(e.)415 2469 y Fh(newmap)o Fs(\()p Fr(i)p Fs(\))d(=)670 2397 y Fd(\()724 2441 y Fh(map1)o Fs(\()p Fr(i)p Fs(\))308 b(if)16 b Fr(i)c(<)h(siz)r(e)p Fs(\()p Fh(map1)p Fs(\))724 2497 y Fh(map2)o Fs(\()p Fr(i)d Fo(\000)g Fr(siz)r(e)p Fs(\()p Fh(map1)p Fs(\)\))40 b(otherwise)75 2608 y Ft(IN)17 b(map1)23 b Fs(map)15 b(ob)s(ject)g(handle)75 2704 y Ft(IN)i(map2)23 b Fs(map)15 b(ob)s(ject)g(handle)p eop %%Page: 77 82 77 81 bop 75 -100 a Fq(9.3.)34 b(CONTEXT)p 417 -100 14 2 v 16 w(ID)1349 b Fs(77)75 45 y Ft(OUT)18 b(newmap)23 b Fs(map)15 b(ob)s(ject)f(handle)166 194 y Ft(MPI)p 275 194 16 2 v 18 w(MAP)p 419 194 V 18 w(PR)o(ODUCT\()k(map1,)f(map2,)g (range,)g(newmap\))166 287 y Fh(map1)12 b Fs(is)h(a)g(map)f(with)i(v)m (alues)g(in)f(the)g(range)g(0)p Fr(;)8 b(:::;)g Fh(r)o(ange)r Fo(\000)d Fs(1.)20 b Fh(newmap)11 b Fs(is)j(the)f(cartesian)g(pro)q (duct)75 343 y(of)j Fh(map1)f Fs(and)h Fh(map2)p Fs(,)g(whic)o(h)g (maps)g(the)g(pair)h(\()p Fr(i;)8 b(j)s Fs(\))13 b(in)o(to)j(the)g (pair)h(\()p Fh(map1)o Fs(\()p Fr(i)p Fs(\))p Fr(;)8 b Fh(map2)n Fs(\()p Fr(j)s Fs(\)\).)20 b(P)o(airs)c(in)h(the)75 400 y(domain)f(and)g(in)g(the)f(range)g(of)g(this)h(mapping)g(are)g(n)o (um)o(b)q(ered)g(in)g(ro)o(w)f(ma)s(jor)f(order,)g(i.e.,)i(pair)f(\()p Fr(i;)8 b(j)s Fs(\))75 456 y(has)15 b(n)o(um)o(b)q(er)h Fr(i)9 b Fo(\001)h Fh(range)f Fs(+)i Fr(j)s Fs(.)19 b(The)d(pro)q(duct) f(op)q(eration)h(is)f(asso)q(ciativ)o(e)h(but)f(not)g(comm)o(utativ)o (e.)480 574 y Fh(newmap)o Fs(\()p Fr(i)10 b Fo(\001)f Fh(range)h Fs(+)g Fr(j)s Fs(\))i(=)h Fh(map1)o Fs(\()p Fr(i)p Fs(\))c Fo(\001)h Fh(range)f Fs(+)i Fh(map2)o Fs(\()p Fr(j)s Fs(\))75 687 y Ft(IN)17 b(map1)23 b Fs(map)15 b(ob)s(ject)g(handle)75 786 y Ft(IN)i(map2)23 b Fs(map)15 b(ob)s(ject)g(handle)75 885 y Ft(OUT)j(newmap)23 b Fs(map)15 b(ob)s(ject)f(handle)75 1036 y Fm(9.3)70 b(Con)n(text)p 478 1036 21 2 v 24 w(id)75 1140 y Fs(A)18 b Ft(con)o(text)p 299 1140 16 2 v 19 w(id)g Fs(is)h(an)f(in)o(teger.)28 b(The)18 b(range)g(of)f(v)m(alid)j(v)m(alues)f(for)e Fh(context)p 1438 1140 15 2 v 17 w(id)g Fs(is)i(implemen)o(tation)75 1197 y(dep)q(enden)o(t,)d(and)g(can)f(b)q(e)h(found)f(b)o(y)h(calling)h (a)d(suitable)j(query)e(function,)h(as)f(describ)q(ed)i(in)f(7.)75 1326 y Fj(9.3.1)55 b(Op)r(erations)18 b(on)h(con)n(text)p 797 1326 17 2 v 19 w(id's)75 1450 y Ft(MPI)p 184 1450 16 2 v 18 w(CONTEXT)p 466 1450 V 19 w(ALLOCA)l(TE\(comm,)e(map,)g(arra) o(y)p 1210 1450 V 18 w(of)p 1270 1450 V 19 w(con)o(textids,)g(len\))166 1543 y Fs(Allo)q(cates)i(an)f(arra)o(y)f(of)h(con)o(text)p 759 1543 14 2 v 16 w(id's.)29 b(This)19 b(is)g(a)f(collectiv)o(e)i(op)q (eration)f(that)e(is)i(executed)g(b)o(y)75 1600 y(all)e(pro)q(cesses)g (in)g(the)g(group)f(de\014ned)i(b)o(y)e Fh(\(comm,)23 b(map\))p Fs(.)g(The)17 b(con)o(text)p 1367 1600 V 16 w(ids)g(that)e(are)h(returned)h(are)75 1656 y(unique)g(within)g(the)f (group)f(asso)q(ciated)h(with)g Fh(\(comm,)23 b(map\))p Fs(.)d(The)c(arra)o(y)e(returned)i(is)g(the)g(same)f(on)75 1712 y(all)j(pro)q(cesses)f(that)f(call)i(the)f(function)h(\(same)e (order,)g(same)h(n)o(um)o(b)q(er)g(of)g(elemen)o(ts\).)25 b(The)17 b(call)h(ma)o(y)75 1769 y(blo)q(c)o(k)e(un)o(til)g(all)g(pro)q (cesses)g(within)g(the)g(executing)g(group)f(ha)o(v)o(e)g(in)o(v)o(ok)o (ed)g(the)g(call.)75 1882 y Ft(IN)i(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1981 y Ft(IN)i(map)23 b Fs(map)15 b(ob)s(ject)g(handle)75 2080 y Ft(OUT)j(arra)o(y)p 332 2080 16 2 v 18 w(of)p 392 2080 V 19 w(con)o(text)p 580 2080 V 19 w(ids)23 b Fs(\(arra)o(y)14 b(of)g(in)o(tegers\))75 2180 y Ft(IN)j(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(con)o(text)p 611 2180 14 2 v 16 w(id's)g(to)g(allo)q(cate)h(\(in)o(teger\))166 2328 y Ft(MPI)p 275 2328 16 2 v 18 w(CONTEXT)p 557 2328 V 19 w(RESER)-6 b(VE\(con)o(text)p 1008 2328 V 21 w(id\))166 2421 y Fs(Reserv)o(e)13 b(a)f(con)o(text)p 516 2421 14 2 v 16 w(id)h(v)m(alue.)21 b(This)13 b(is)g(a)f(lo)q(cal)i(call)g(that) d(is)j(in)o(v)o(ok)o(ed)e(b)o(y)h(a)f(single)i(pro)q(cess.)19 b(A)13 b(re-)75 2478 y(serv)o(ed)g(con)o(text)p 362 2478 V 16 w(id)g(v)m(alue)h(will)g(not)e(b)q(e)i(allo)q(cated)f(b)o(y)g(a)f (subsequen)o(t)h(call)h(to)e Fh(MPI)p 1486 2478 15 2 v 17 w(CONTEXT)p 1671 2478 V 16 w(ALLOCATE)75 2534 y Fs(b)o(y)j(the)h(same)f(pro)q(cess.)20 b(It)15 b(is)h(erroneous)f(to)g (reserv)o(e)g(a)g(con)o(text)p 1203 2534 14 2 v 16 w(id)h(that)f(has)g (b)q(een)h(already)g(b)q(een)g(al-)75 2591 y(lo)q(cated)g(b)o(y)f Fh(MPI)p 371 2591 15 2 v 17 w(CONTEXT)p 556 2591 V 16 w(ALLOCATE)p Fs(.)75 2704 y Ft(IN)i(con)o(text)p 325 2704 16 2 v 19 w(id)23 b Fs(\(in)o(teger\))p eop %%Page: 78 83 78 82 bop 75 -100 a Fs(78)694 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)166 45 y Fl(Discussion:)166 95 y Fk(Ma)o(y)e(w)o(an)o(t)h(to)g(reserv)o(e)h(an)f(arra)o(y)g(of)f(v) n(alues,)g(rather)i(than)f(one)g(v)n(alue.)166 152 y(Implemen)o (tations)6 b(ma)o(y)i(ha)o(v)o(e)h(there)h(o)o(wn)f(reserv)o(ed)j(v)n (alues.)k(The)10 b(e\013ect)g(is)g(as)f(if)f(calls)h(to)g Fg(MPI)p 1638 152 14 2 v 15 w(CONTEXT)p 1807 152 V 15 w(RESERVE)75 209 y Fk(where)15 b(executed)h(in)d(a)h(pream)o(ble.)166 384 y Ft(MPI)p 275 384 16 2 v 18 w(CONTEXT)p 557 384 V 19 w(FREE\(con)o(text)p 905 384 V 20 w(id\))166 476 y Fs(F)l(ree)i(a)g(reserv)o(ed)h(con)o(text)p 634 476 14 2 v 16 w(id)g(v)m(alue.)24 b(The)17 b(v)m(alue)g(b)q(ecomes)g(a)o(v) m(ailable)h(for)e(a)g(subsequen)o(t)h(reser-)75 532 y(v)m(ation)g(b)o (y)g Fh(MPI)p 354 532 15 2 v 16 w(CONTEXT)p 538 532 V 17 w(RESERVE)e Fs(or)h(subsequen)o(t)h(allo)q(cation)h(b)o(y)f Fh(MPI)p 1376 532 V 16 w(CONTEXT)p 1560 532 V 17 w(ALLOCATE)p Fs(.)e(It)h(is)75 589 y(erroneous)f(to)g(free)g(a)g(v)m(alue)h(that)f (is)g(asso)q(ciated)h(with)f(an)g(activ)o(e)h(comm)o(unicator.)75 697 y Ft(IN)h(con)o(text)p 325 697 16 2 v 19 w(id)23 b Fs(\(in)o(teger\))166 881 y Fl(Discussion:)166 931 y Fk(Here,)15 b(to)q(o,)e(w)o(e)h(migh)o(t)e(prefer)j(to)e(free)i(an)f (arra)o(y)f(of)h(v)n(alues,)f(rather)h(than)g(one.)166 981 y(Comm)o(uni)o(cators,)c(lik)o(e)h(all)g(opaque)h(ob)r(jects,)h (are)g(freed)g(b)o(y)e(a)h(call)g(to)f Fg(MPI)p 1346 981 14 2 v 16 w(FREE)p Fk(.)f(W)m(e)i(ma)o(y)e(prefer)j(to)f(ha)o(v)o (e)75 1031 y(that)i(same)g(call)f(free)i(also)f(the)h(asso)q(ciated)g (con)o(text)p 915 1031 13 2 v 16 w(id.)j(With)c(the)h(curren)o(t)h (design,)e(if)f(w)o(e)i(w)o(an)o(t)f(to)g(free)h(b)q(oth)75 1081 y(comm)o(unicator)f(and)j(con)o(text)p 568 1081 V 15 w(id)g(w)o(e)g(need)h(to)e(store)i(the)g(con)o(text)p 1165 1081 V 15 w(id)f(v)n(alue,)f(free)i(the)f(comm)o(unicator,)d(next) 75 1131 y(free)h(the)f(con)o(text)p 361 1131 V 16 w(id.)75 1359 y Fm(9.4)70 b(Comm)n(unicators)75 1461 y Fs(A)21 b Ft(comm)o(unicator)g Fs(is)h(an)f(opaque)g(ob)s(ject)g(that)f(iden)o (ti\014es)j(a)e(group)g(of)f(pro)q(cesses)i(and)f(a)g(com-)75 1518 y(m)o(unication)e(con)o(text)e(for)g(that)h(group.)27 b(Lik)o(e)19 b(other)f(opaque)g(ob)s(jects,)f(comm)o(unicators)h (cannot)f(b)q(e)75 1574 y(transfered)e(b)q(et)o(w)o(een)g(pro)q (cesses;)h(con)o(text)p 824 1574 14 2 v 15 w(id's)g(are)f(used)h(to)e (transfer)h(information)g(on)g(con)o(text.)166 1631 y(As)g(a)g (short-hand,)g(w)o(e)g(shall)i(iden)o(ti\014y)g(the)e(group)g(of)g(pro) q(cesses)h(asso)q(ciated)g(with)f(a)g(comm)o(uni-)75 1687 y(cator)h Fh(comm)g Fs(with)h(the)g(comm)o(unicator)f(itself.)26 b(Th)o(us,)16 b(\\the)h(pro)q(cess)g(with)g(rank)f Fr(i)h Fs(in)g Fh(comm)p Fs(")f(should)75 1744 y(b)q(e)e(understo)q(o)q(d)g (as)g(meaning)g(\\)f(the)h(pro)q(cess)g(with)g(rank)f Fr(i)g Fs(in)i(the)f(group)f(asso)q(ciated)h(with)g Fh(comm)p Fs(".)k(In)75 1800 y(the)e(same)g(manner,)g(\\the)g(comm)o(unication)h (con)o(text)f Fh(comm)p Fs(")f(should)i(b)q(e)g(understo)q(o)q(d)f(to)g (mean)g(\\the)75 1857 y(comm)o(unication)g(con)o(text)f(asso)q(ciated)g (with)g Fh(comm)p Fs(".)166 1913 y(An)h(initial)h(comm)o(unicator)f Fh(MPI)p 745 1913 15 2 v 16 w(COMM)p 857 1913 V 17 w(INIT)f Fs(is)h(de\014ned)h(when)f(the)g(program)e(starts.)20 b(Its)c(asso-)75 1970 y(ciated)j(group)e(con)o(tains)h(all)h(pro)q (cesses)g(that)e(start)g(the)h(computation.)28 b(Applications)20 b(that)d(do)h(not)75 2026 y(need)e(m)o(ultiple)h(pro)q(cess)f(groups)e (or)h(m)o(ultiple)i(con)o(texts,)d(will)j(only)f(use)f(this)h(comm)o (unicator.)166 2083 y(Let)i Fh(comm)f Fs(b)q(e)h(a)g(comm)o(unicator)f (that)g(is)i(asso)q(ciated)f(with)g(a)f(group)h(of)f(size)i Fr(n)p Fs(,)f(and)g(let)g Fh(map)e Fs(:)75 2140 y(0)p Fr(::m)8 b Fo(\000)h Fs(1)k Fo(!)g Fs(0)p Fr(::n)8 b Fo(\000)h Fs(1)15 b(b)q(e)g(a)f(map.)20 b(Then)15 b(the)g(pair)g Fh(\(comm,)23 b(map\))14 b Fs(de\014nes)i(a)e(subgroup,)h(namely)g(the) 75 2196 y(subgroup)g(of)g(pro)q(cesses)h(with)f(ranks)g Fh(map)p Fs(\(0\))p Fr(;)8 b(:::;)f Fh(map)m Fs(\()p Fr(m)i Fo(\000)i Fs(1\))j(in)i Fh(comm)p Fs(.)75 2320 y Fj(9.4.1)55 b(Op)r(erations)18 b(on)h(comm)n(unicators)75 2442 y Ft(MPI)p 184 2442 16 2 v 18 w(COMM)p 379 2442 V 18 w(MAP\(comm,)d(sub)q(comm,)g(map\))166 2534 y Fs(Returns)g(a)e (map)h(suc)o(h)h(that)e Fh(\(comm,)23 b(map\))15 b Fs(is)h(the)f(group) g(asso)q(ciated)g(with)h Fh(subcomm)p Fs(;)e(i.e.,)g(if)i Fh(i)75 2591 y Fs(is)h(the)f(rank)g(of)g(a)h(pro)q(cess)f(in)h(the)g (group)f(asso)q(ciated)h(with)f Fh(subcomm)p Fs(,)g(then)g Fh(map\(i\))g Fs(is)h(the)f(rank)g(of)75 2647 y(that)f(same)g(pro)q (cess)g(in)i(the)e(group)g(asso)q(ciated)h(with)g Fh(comm)p Fs(.)j(The)d(call)h(is)f(erroneous)f(if)h Fh(subcomm)e Fs(has)75 2704 y(a)h(pro)q(cess)g(that)g(is)h(not)e(mem)o(b)q(er)i(of)e Fh(comm)p Fs(.)p eop %%Page: 79 84 79 83 bop 75 -100 a Fq(9.4.)34 b(COMMUNICA)l(TORS)1224 b Fs(79)75 45 y Ft(IN)17 b(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 141 y Ft(IN)i(sub)q(comm)23 b Fs(comm)o(unicator)14 b(ob)s(ject)h(handle)75 237 y Ft(OUT)j(map)23 b Fs(map)15 b(ob)s(ject)g(handle)166 346 y(The)i Fh(MPI)p 336 346 15 2 v 17 w(COMM)p 449 346 V 17 w(MAP)g Fs(function)h(retriev)o (es)f(the)g(group)g(of)g(pro)q(cesses)h(that)e(is)i(asso)q(ciated)g (with)f(a)75 403 y(comm)o(unicator.)h(Ho)o(w)o(ev)o(er,)12 b(since)g(absolute)h(pro)q(cess)f(names)f(are)h(not)f(visible)j(in)f (MPI,)e(the)h(group)g(can)75 459 y(only)j(b)q(e)g(de\014ned)h(relativ)o (e)f(to)f(another)g(encompassing)h(group.)20 b(The)14 b(\\absolute")h(pro)q(cess)g(n)o(um)o(b)q(er)f(is)75 516 y(obtained)i(b)o(y)f(using)h Fh(MPI)p 519 516 V 17 w(COMM)p 632 516 V 16 w(INIT)f Fs(as)g(a)g(reference)h(comm)o (unicator.)166 608 y Ft(MPI)p 275 608 16 2 v 18 w(COMM)p 470 608 V 18 w(SIZE\(comm,)h(size\))166 701 y Fs(Returns)f(the)f(size)h (of)f(the)g(group)g(asso)q(ciated)g(with)h Fh(comm)p Fs(.)75 810 y Ft(IN)h(comm)22 b Fs(handle)17 b(to)d(comm)o(unicator)75 906 y Ft(OUT)k(size)23 b Fs(group)15 b(size)h(\(in)o(teger\))166 1051 y Ft(MPI)p 275 1051 V 18 w(COMM)p 470 1051 V 18 w(CONTEXTID\(comm,)h(con)o(text)p 1170 1051 V 19 w(id\))166 1143 y Fs(Returns)f(the)f(con)o(text)p 567 1143 14 2 v 16 w(id)h(asso)q(ciated)f(with)h(the)f(comm)o(unicator)g Fh(comm)p Fs(.)75 1252 y Ft(IN)i(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1348 y Ft(OUT)j(con)o(text)p 380 1348 16 2 v 19 w(id)24 b Fs(con)o(text)p 613 1348 14 2 v 15 w(id)75 1473 y Fj(9.4.2)55 b(Comm)n(unicator)17 b(constructors)75 1596 y Ft(MPI)p 184 1596 16 2 v 18 w(COMM)p 379 1596 V 18 w(MAKE\(comm,)f(map,)h(con)o(text)p 1049 1596 V 19 w(id,)h(new)o(comm\))166 1688 y Fs(Creates)h(a)h(new)g (comm)o(unicator)g(ob)s(ject,)g(whic)o(h)h(is)f(asso)q(ciated)h(with)f (the)g(group)g(de\014ned)h(b)o(y)75 1745 y Fh(\(comm,)i(map\))p Fs(,)14 b(and)h(the)g(con)o(text)p 681 1745 14 2 v 15 w(id)h Fh(context)p 917 1745 15 2 v 16 w(id)p Fs(.)j(This)d(is)f(a)f (lo)q(cal)i(call)g(executed)g(b)o(y)e(one)h(pro)q(cess.)75 1801 y(Ho)o(w)o(ev)o(er,)d(the)h(new)h(comm)o(unicator)e(ob)s(ject)h (should)h(not)e(b)q(e)i(used)g(for)e(comm)o(unication)i(b)q(et)o(w)o (een)f(t)o(w)o(o)75 1858 y(pro)q(cesses)j(unless)g(they)f(b)q(oth)h(ha) o(v)o(e)e(called)j(the)e(function.)166 1915 y(It)d(is)h(erroneous)f(to) g(create)g(on)h(the)f(same)g(pro)q(cess)h(t)o(w)o(o)e(distinct)i(comm)o (unicators)f(with)h(the)f(same)75 1971 y(con)o(text)p 225 1971 14 2 v 16 w(id.)75 2080 y Ft(IN)17 b(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 2176 y Ft(IN)i(map)23 b Fs(map)15 b(ob)s(ject)g(handle)75 2272 y Ft(IN)i(con)o(text)p 325 2272 16 2 v 19 w(id)23 b Fs(con)o(text)p 557 2272 14 2 v 16 w(id)75 2369 y Ft(OUT)18 b(new)o(comm)k Fs(comm)o(unicator)15 b(ob)s(ject)f(handle)166 2478 y(The)23 b(con)o(tect)p 413 2478 V 16 w(id)g(used)g(in)h(this)f (call)g(ma)o(y)f(b)q(e)i(an)e(in)o(teger)h(returned)g(b)o(y)f(a)h (previous)g(call)h(to)75 2534 y Fh(MPI)p 150 2534 15 2 v 17 w(ALLOC)p 287 2534 V 16 w(CONTEXT)p Fs(,)17 b(executed)h(within) h(the)f(same)f(group.)27 b(In)18 b(suc)o(h)g(case,)g(it)g(is)g(guaran)o (teed)g(that)75 2591 y(the)f(con)o(text)p 305 2591 14 2 v 16 w(id)h(is)g(unique,)h(and)e(has)g(not)g(b)q(een)h(used)g (already)g(to)e(create)h(a)g(comm)o(unicator.)25 b(Ho)o(w-)75 2647 y(ev)o(er,)17 b(the)g(user)g(is)h(free)f(to)f(manage)g(on)h(it)g (o)o(wn)g(the)g(con)o(text)p 1165 2647 V 16 w(id's,)g(and)g(use)g (other)g(mec)o(hanisms)h(for)75 2704 y(their)e(allo)q(cation.)22 b(F)l(or)14 b(example,)i(one)g(could)g(ha)o(v)o(e)f(a)g(single)i Ft(con)o(text)p 1330 2704 16 2 v 19 w(id)h(serv)o(er)13 b Fs(generate)i(unique)p eop %%Page: 80 85 80 84 bop 75 -100 a Fs(80)694 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y Fs(con)o(text)p 225 45 14 2 v 16 w(id's)f(for)g(the)g(en)o(tire)h(system;)f(one)g(can)g (preallo)q(cate)i(statically)f(some)f(con)o(text)p 1609 45 V 15 w(id)h(v)m(alues,)h(for)75 102 y(the)f(use)h(of)f(libraries;)h (etc.)166 234 y Fl(Discussion:)166 284 y Fk(Do)d(w)o(e)h(w)o(an)o(t)f (to)h(prohibit)f(implem)o(en)o(tations)e(that)j(sync)o(hronize)h(the)f (creation)g(of)f(comm)o(unicators)e(\(i.e.,)75 334 y(where)k(a)f(call)f (to)g Fg(MPI)p 425 334 V 16 w(MAKE)p 529 334 V 14 w(COMM)g Fk(blo)q(c)o(ks)h(un)o(til)f(all)g(mem)o(b)q(ers)g(of)g(the)h(group)g (ha)o(v)o(e)g(made)f(the)h(call\)?)166 383 y(Is)g(it)g(preferable)g(to) g(ha)o(v)o(e)g(a)f Fg(array)p 739 383 V 15 w(of)p 798 383 V 15 w(comm)g Fk(as)h(parameter,)f(rather)i(than)f(one)g(comm)o (unicator?)166 440 y(W)m(e)i(don't)g(ha)o(v)o(e)g(no)o(w)g(the)h (abilit)o(y)e(to)h(ha)o(v)o(e)g(t)o(w)o(o)g(di\013eren)o(t)i(groups)e (with)g(the)h(same)f(con)o(text;)i(e.g.,)d(t)o(w)o(o)75 496 y(di\013eren)o(t)d(n)o(um)o(b)q(erings)e(of)g(the)i(pro)q(cesses,)i (used)d(with)g(the)h(same)e(comm)o(unicati)o(on)e(con)o(text.)18 b(The)11 b(alternativ)o(e)g(is)75 553 y(to)h(remo)o(v)o(e)f(the)i (restriction)g(that)f(t)o(w)o(o)g(distinct)g(comm)o(unicators)e(cannot) i(use)h(the)g(same)e(con)o(text)p 1641 553 13 2 v 16 w(id.)17 b(It)12 b(is)g(no)o(w)75 609 y(the)18 b(user)h(resp)q (onsibilit)o(y)e(to)g(disam)o(biguate)e(these)k(t)o(w)o(o)e(comm)o (unicators;)g(the)h(system)f(do)q(es)h(not)f(guaran)o(tee)75 666 y(that)e(a)g(message)g(sen)o(t)h(with)e(one)i(comm)o(unicator)c (can)j(b)q(e)h(receiv)o(ed)g(only)f(with)f(this)h(same)g(comm)o (unicator,)d(if)75 722 y(they)h(use)g(the)g(same)f(con)o(text)p 545 722 V 16 w(id.)17 b(W)m(e)12 b(sa)o(v)o(e)h(on)f(con)o(text)p 972 722 V 16 w(id's)f(but)i(lo)q(ose)g(safet)o(y)f(\(the)h(sender)i (and)d(receiv)o(er)i(ma)o(y)75 779 y(b)q(e)i(using)g(di\013eren)o(t)g (pro)q(cess)i(n)o(um)o(b)q(erings,)c(if)h(t)o(w)o(o)g(di\013eren)o(t)i (comm)o(unicators)c(are)j(used)h({)e(the)i(receiv)o(er)g(ma)o(y)75 835 y(not)d(iden)o(tify)f(correctly)i(the)f(sender\).)75 1061 y Fm(9.5)70 b(W)-6 b(orking)23 b(without)f(con)n(text)p 993 1061 21 2 v 25 w(id's)75 1162 y Fs(It)12 b(is)h(p)q(ossible)g(to)f (create)g(comm)o(unicators)f(directly)l(,)j(without)e(using)h(con)o (text)p 1430 1162 14 2 v 16 w(id's.)19 b(A)12 b(single)h(function)75 1219 y(com)o(bines)f(the)f(allo)q(cation)h(of)e(a)h(new)g(con)o(text)p 865 1219 V 16 w(id)h(and)f(the)g(generation)g(of)f(a)h(new)g(comm)o (unicator.)18 b(This)75 1275 y(allo)o(ws)f(a)g(\\naiv)o(e")f(user)h (that)g(do)q(es)g(not)f(need)i(to)e(customize)i(con)o(text)p 1329 1275 V 15 w(id)g(allo)q(cation)g(to)e(ignore)i(this)75 1332 y(MPI)j(feature.)38 b(Direct)21 b(comm)o(unicator)g(creation)h(is) g(safer,)g(since)g(uniqueness)h(of)e(con)o(text)p 1742 1332 V 16 w(id's)g(is)75 1388 y(guaran)o(teed,)14 b(b)o(y)h (construction.)166 1480 y Ft(MPI)p 275 1480 16 2 v 18 w(COMM)p 470 1480 V 18 w(SAFEMAKE\(comm,)h(map,)h(new)o(comm\))166 1572 y Fs(Creates)c(a)h(new)g(comm)o(unicator)g(\(with)g(an)g(attac)o (hed)f(con)o(text)p 1270 1572 14 2 v 16 w(id\))i(that)e(is)h(asso)q (ciated)h(with)f(the)75 1628 y(group)19 b(de\014ned)i(b)o(y)e(the)g (pair)h Fh(\(comm,)j(map\))p Fs(.)31 b(This)20 b(is)g(a)f(collectiv)o (e)i(call)f(that)f(has)g(to)g(b)q(e)h(in)o(v)o(ok)o(ed)75 1685 y(b)o(y)d(all)h(mem)o(b)q(ers)f(in)g(this)h(group.)24 b(All)18 b(of)f(them)g(are)f(pro)o(viding)i(the)f(same)g(input)h (parameters)e Fh(comm)75 1741 y Fs(and)j Fh(map)p Fs(.)32 b(The)19 b(call)h(ma)o(y)f(blo)q(c)o(k)h(un)o(til)g(all)g(pro)q(cesses) g(in)g(the)f(group)g(ha)o(v)o(e)g(in)o(v)o(ok)o(ed)g(it.)32 b(The)20 b(new)75 1798 y(comm)o(unicator)c(ma)o(y)f(b)q(e)i(safely)g (used)f(for)g(comm)o(unication)h(with)f(an)o(y)g(mem)o(b)q(er)g(of)g (the)g(new)h(group,)75 1854 y(once)f(the)f(call)h(returns.)75 1958 y Ft(IN)h(comm)22 b Fs(comm)o(unicator)15 b(handle)75 2052 y Ft(IN)i(map)23 b Fs(map)15 b(handle)75 2145 y Ft(OUT)j(new)o(comm)k Fs(comm)o(unicator)15 b(handle)166 2249 y Fh(MPI)p 241 2249 15 2 v 17 w(COMM)p 354 2249 V 16 w(SAFEMAKE)f Fs(is)75 2354 y Fh(MPI_CONTEXT_ALLOC\(comm,)21 b(map,)i(context_id,)f(1\);)75 2410 y(MPI_COMM_MAKE\(comm,)f(map,)j (context_id,)e(newcomm\);)75 2467 y(``MPI_SYNCH\(comm,map\)'')f(/*)i (by)h(which)f(we)h(mean)f(a)h(synchronization)576 2523 y(operation)f(in)g(the)h(subgroup)f(defined)f(by)i(\(comm,map\))f(*/) 166 2704 y Fl(Discussion:)p eop %%Page: 81 86 81 85 bop 75 -100 a Fq(9.6.)34 b(W)o(ORKING)16 b(WITHOUT)f(MAPS)h(AND)f (CONTEXT)p 1181 -100 14 2 v 16 w(ID'S)547 b Fs(81)166 45 y Fk(Do)12 b(w)o(e)g(prefer)i(to)e(create)i(an)e(arra)o(y)g(of)f (comm)o(unicators,)f(rather)j(than)g(one?)k(\(T)m(o)12 b(b)q(e)h(consisten)o(t)g(with)f(the)75 95 y Fg(MPI)p 144 95 V 15 w(CONTEXT)p 313 95 V 14 w(ALLOC)h Fk(function\).)166 145 y Fg(MPI)p 235 145 V 15 w(SYNCH\(comm,map\))f Fk(is)i(not)h(curren) o(tly)h(de\014ned)g(in)e(the)i(collectiv)o(e)f(comm)o(unicati)o(on)d (library;)i(it)h(can)75 195 y(b)q(e)g(co)q(ded)f(using)g(p)q(oin)o (t-to-p)q(oin)o(t)e(calls,)h(but)i(seems)f(imp)q(ortan)o(t)e(enough)i (to)f(b)q(e)i(included)f(there.)166 251 y(General)i(though)o(t)h({)f (if)g(w)o(e)g(supp)q(ort)i(collectiv)o(e)e(comm)o(unication)d(calls)k (with)f(a)g(map)f(as)i(an)f(additional)75 308 y(parameter,)g(then)h(w)o (e)g(ha)o(v)o(e)f(disasso)q(ciated)h(the)g(\\con)o(text")g(of)f(a)g (collectiv)o(e)g(comm)o(unication,)e(from)g(its)j(sup-)75 364 y(p)q(orting)c(\\group".)k(W)m(e)12 b(need)i(to)f(do)f(it)h(when)g (w)o(e)g(build)f(new)i(comm)o(unicators,)c(in)i(order)i(to)f(b)q(o)q (otstrap)g({)g(if)f(w)o(e)75 421 y(w)o(an)o(t)h(the)h(call)f(that)h (generates)h(a)f(new)g(comm)o(unicator)d(to)i(in)o(v)o(olv)o(e)f(only)h (the)h(pro)q(cesses)j(in)c(the)h(new)g(group.)75 647 y Fm(9.6)70 b(W)-6 b(orking)23 b(without)f(maps)h(and)h(con)n(text)p 1305 647 21 2 v 24 w(id's)75 748 y Fs(It)16 b(is)g(p)q(ossible)h(to)e (a)o(v)o(oid)g(b)q(oth)h(the)g(use)f(of)h(map)f(ob)s(jects)g(and)h(of)f (con)o(text)p 1384 748 14 2 v 16 w(id's)h(altogether.)k(A)c(single)75 805 y(function)d(com)o(bines)g(the)f(generation)g(of)f(a)h(map,)g(the)g (allo)q(cation)i(of)d(a)h(con)o(text)p 1445 805 V 16 w(id)g(and)h(the)f(generation)75 861 y(of)17 b(a)h(new)g(comm)o (unicator.)28 b(This)18 b(allo)o(ws)g(\\naiv)o(e")g(users)g(to)f(use)h (MPI)g(without)g(b)q(eing)h(exp)q(osed)g(to)75 918 y(maps)c(or)g(con)o (text)p 400 918 V 15 w(id's.)166 1010 y Ft(MPI)p 275 1010 16 2 v 18 w(COMM)p 470 1010 V 18 w(BUILD\(comm,)i(arra)o(y)p 966 1010 V 18 w(of)p 1026 1010 V 19 w(ranks,)f(size,)i(new)o(comm\))166 1102 y Fs(Build)k(a)f(map)f(from)g(an)g(explicit)j(list)e(represen)o (tation.)36 b(This)21 b(is)g(a)f(collectiv)o(e)i(function)g(that)75 1158 y(is)f(called)i(b)o(y)d(all)i(mem)o(b)q(ers)f(of)g(the)f(new)h (group.)37 b(All)22 b(pro)o(vide)f(the)g(same)g(parameters)f(for)g Fh(comm)p Fs(,)75 1215 y Fh(array)p 198 1215 15 2 v 16 w(of)p 262 1215 V 17 w(ranks)15 b Fs(and)h Fh(size)p Fs(.)j(The)d(call)g(ma)o(y)f(blo)q(c)o(k)h(un)o(til)g(it)g(w)o(as)e(in) o(v)o(ok)o(ed)i(on)f(all)h(the)g(group)f(mem-)75 1271 y(b)q(ers.)40 b(When)22 b(it)f(returns,)i(the)f(new)g(comm)o(unicator)f (ma)o(y)g(b)q(e)h(used)g(to)f(comm)o(unicate)h(with)g(an)o(y)75 1328 y(mem)o(b)q(er)15 b(of)g(the)g(new)h(group.)75 1422 y Ft(IN)h(comm)22 b Fs(handle)17 b(to)d(comm)o(unicator)75 1516 y Ft(IN)j(arra)o(y)p 277 1516 16 2 v 18 w(of)p 337 1516 V 19 w(ranks)22 b Fs(list)16 b(of)f(v)m(alues)h(in)g(the)f(range)g (of)g Fh(map)75 1610 y Ft(IN)i(size)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(arra)o(y)e({)h(map)g(size)h(\(in)o(teger\))75 1704 y Ft(OUT)i(new)o(comm)k Fs(handle)16 b(to)f(new)g(comm)o(unicator) 166 1798 y Fh(MPI)p 241 1798 15 2 v 17 w(COMM)p 354 1798 V 16 w(BUILD)g Fs(is)75 1905 y Fh(MPI_MAP_BUILD\(comm,)21 b(array_of_ranks,)h(size,)h(newmap\);)75 1961 y (MPI_COMM_SAFEMAKE\(comm,)e(newmap,)i(newcomm\);)166 2103 y Ft(MPI)p 275 2103 16 2 v 18 w(COMM)p 470 2103 V 18 w(COPY\(comm,)16 b(new)o(comm\))166 2195 y Fs(Creates)h(a)g(new)g (comm)o(unicator)g(with)h(the)g(same)f(group)g(as)g(the)g(old)h(comm)o (unicator.)26 b(This)18 b(is)75 2252 y(a)i(collectiv)o(e)i(function)g (that)e(is)h(called)h(b)o(y)e(all)i(mem)o(b)q(ers)e(of)g(new)h(group.) 36 b(All)22 b(pro)o(vide)f(the)f(same)75 2308 y(parameters)d(for)g Fh(comm)p Fs(.)27 b(The)18 b(call)h(ma)o(y)e(blo)q(c)o(k)h(un)o(til)h (it)f(w)o(as)f(in)o(v)o(ok)o(ed)h(on)f(all)i(the)f(group)f(mem)o(b)q (ers.)75 2365 y(When)e(it)f(returns,)g(the)g(new)h(comm)o(unicator)f (ma)o(y)f(b)q(e)i(used)g(to)f(comm)o(unicate)g(with)h(an)o(y)f(mem)o(b) q(er)g(of)75 2421 y(the)h(group.)75 2515 y Ft(IN)i(comm)22 b Fs(handle)17 b(to)d(comm)o(unicator)75 2609 y Ft(OUT)k(new)o(comm)k Fs(handle)16 b(to)f(new)g(comm)o(unicator)166 2704 y Fh(MPI)p 241 2704 15 2 v 17 w(COMM)p 354 2704 V 16 w(COPY)g Fs(is)p eop %%Page: 82 87 82 86 bop 75 -100 a Fs(82)694 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y Fh (MPI_COMM_SIZE\(comm,)21 b(size\);)75 102 y(MPI_COMM_SAFEMAKE\(comm,)g (MPI_IDENT\(size\),)g(newcomm\);)166 248 y Ft(MPI)p 275 248 16 2 v 18 w(COMM)p 470 248 V 18 w(SPLIT\(comm,)c(k)o(ey)l(,)e (index,)j(new)o(comm\))166 340 y Fs(Split)c(the)e(group)h(asso)q (ciated)f(with)h Fh(comm)p Fs(;)g(creates)f(a)g(new)h(group)f(for)g (eac)o(h)h(distinct)h(v)m(alue)f(of)f Fh(key)75 397 y Fs(that)g(con)o(tains)h(the)g(pro)q(cesses)g(that)f(supplied)k(that)c (k)o(ey)g(v)m(alue;)j(the)e(pro)q(cesses)g(are)g(rank)o(ed)f(according) 75 453 y(to)g(the)h Fh(index)f Fs(v)m(alues)h(they)g(supplied.)22 b(A)12 b(new)h(comm)o(unicator)f(is)i(created)e(for)g(eac)o(h)h (subgroup.)19 b(Eac)o(h)75 510 y(pro)q(cess)c(is)f(returned)h(a)f (handle)i(to)d(the)i(comm)o(unicator)f(for)f(the)i(new)f(subgroup)h(it) f(b)q(elongs)h(to.)k(This)75 566 y(is)f(a)e(collectiv)o(e)j(call)f (executed)g(b)o(y)f(all)h(pro)q(cesses)f(in)h(the)f(group)g(asso)q (ciated)g(with)h Fh(comm)p Fs(.)25 b(They)17 b(call)75 623 y(ma)o(y)h(blo)q(c)o(k)i(un)o(til)g(all)h(pro)q(cesses)e(ha)o(v)o (e)g(in)o(v)o(ok)o(ed)g(the)g(function.)33 b(When)19 b(the)g(call)i(returns)e(the)g(new)75 679 y(comm)o(unicator)c(ma)o(y)f (b)q(e)i(safely)g(used)g(for)e(comm)o(unication)i(in)g(the)f(new)h (group.)75 790 y Ft(IN)h(comm)22 b Fs(handle)17 b(to)d(comm)o(unicator) 75 887 y Ft(IN)j(k)o(ey)22 b Fs(\(in)o(teger\))75 984 y Ft(IN)17 b(index)23 b Fs(\(in)o(teger\))75 1082 y Ft(OUT)18 b(new)o(comm)k Fs(handle)16 b(to)f(new)g(comm)o(unicator)166 1192 y Fh(MPI)p 241 1192 15 2 v 17 w(COMM)p 354 1192 V 16 w(SPLIT\(comm,)23 b(key,)g(index,)g(newcomm\))14 b Fs(is)75 1304 y Fh(MPI_COMM_SIZE\(comm,)21 b(size\);)75 1360 y(MPI_MAP_SPLIT\(comm,)g(MPI_IDENT\(size\),)h(key,)h(index,)g (newmap\);)75 1417 y(MPI_COMM_SAFEMAKE\(comm,)e(newmap,)i(newcomm\);) 166 1603 y Fl(Discussion:)166 1654 y Fk(Ma)o(y)13 b(w)o(an)o(t)g(a)h (sp)q(ecial)f(DONTCARE)h(k)o(ey)g(v)n(alue)f(that)g(indicates)h(that)g (the)g(caller)g(need)g(not)g(ha)o(v)o(e)f(a)h(new)75 1704 y(comm)o(unicator)75 1935 y Fm(9.7)70 b(Examples)75 2038 y Fs(W)l(e)19 b(sa)o(y)f(that)f(a)i(parallel)h(pro)q(cedure)f(is)g Fi(active)g Fs(at)f(a)g(pro)q(cess)h(if)g(the)g(pro)q(cess)g(b)q (elongs)g(to)f(a)g(group)75 2094 y(that)e(ma)o(y)h(collectiv)o(ely)i (execute)f(the)f(pro)q(cedure,)h(and)f(some)f(mem)o(b)q(er)h(of)g(that) f(group)h(is)g(curren)o(tly)75 2151 y(executing)c(the)f(pro)q(cedure)g (co)q(de.)20 b(If)12 b(a)f(parallel)j(pro)q(cedure)e(is)h(activ)o(e)e (at)h(a)f(pro)q(cess,)h(then)g(this)h(pro)q(cess)75 2207 y(ma)o(y)e(b)q(e)i(receiving)g(messages)e(p)q(ertaining)i(to)f(this)g (pro)q(cedure,)h(ev)o(en)f(if)g(it)g(do)q(es)h(not)e(curren)o(tly)h (execute)75 2264 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 2390 y Fj(9.7.1)55 b(Nonreen)n(tran)n(t)19 b(parallel)e(pro)r(cedures) 75 2478 y Fs(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o (y)g(p)q(oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(ok)m(ation)g (of)f(a)g(parallel)75 2534 y(pro)q(cedure)e(can)g(b)q(e)g(activ)o(e)g (at)f(an)o(y)g(pro)q(cess.)30 b(I.e.,)19 b(concurren)o(t)g(in)o(v)o(ok) m(ations)f(of)h(the)f(same)g(parallel)75 2591 y(pro)q(cedure)e(ma)o(y)e (o)q(ccur)h(only)g(within)h(disjoin)o(t)f(groups)f(of)h(pro)q(cesses.) 20 b(F)l(or)14 b(example,)h(all)h(in)o(v)o(ok)m(ations)75 2647 y(of)f(parallel)i(pro)q(cedures)f(in)o(v)o(olv)o(e)g(all)h(pro)q (cesses,)e(pro)q(cesses)h(are)f(single-threaded,)i(and)e(there)h(are)f (no)75 2704 y(recursiv)o(e)h(in)o(v)o(ok)m(ations.)p eop %%Page: 83 88 83 87 bop 75 -100 a Fq(9.7.)34 b(EXAMPLES)1393 b Fs(83)166 45 y(In)17 b(suc)o(h)g(a)g(case,)g(a)f(con)o(text)p 668 45 14 2 v 16 w(id)i(can)f(b)q(e)g(statically)h(allo)q(cated)f(to)g(eac) o(h)g(pro)q(cedure.)25 b(The)17 b(static)75 102 y(allo)q(cation)h(can)g (b)q(e)g(done)g(in)g(a)f(pream)o(ble,)h(as)f(part)g(of)g (initialization)j(co)q(de.)27 b(Or,)17 b(it)h(can)g(b)q(e)g(done)f(a)75 158 y(compile/link)j(time,)d(if)h(the)f(implemen)o(tation)i(has)e (additional)h(mec)o(hanisms)g(to)e(reserv)o(e)h(con)o(text)p 1824 158 V 16 w(id)75 214 y(v)m(alues.)22 b(Comm)o(unicators)14 b(to)h(b)q(e)h(used)g(b)o(y)f(the)h(di\013eren)o(t)f(pro)q(cedures)i (can)e(b)q(e)h(build)h(in)g(a)e(pream)o(ble,)75 271 y(if)i(the)g (executing)h(groups)e(are)h(statically)g(de\014ned;)i(if)e(the)g (executing)h(groups)e(c)o(hange)h(dynamically)l(,)75 327 y(then)f(a)g(new)h(comm)o(unicator)e(has)h(to)g(b)q(e)h(built)g (whenev)o(er)g(the)f(executing)h(group)f(c)o(hanges,)g(but)h(this)75 384 y(new)k(comm)o(unicator)g(can)h(b)q(e)g(built)g(using)g(the)g(same) e(preallo)q(cated)j(con)o(text)p 1491 384 V 16 w(id.)39 b(If)21 b(the)g(parallel)75 440 y(pro)q(cedures)d(can)f(b)q(e)g (organized)g(in)o(to)g(libraries,)h(so)f(that)f(only)h(one)g(pro)q (cedure)h(of)e(eac)o(h)h(library)h(can)75 497 y(b)q(e)i(concurren)o (tly)g(activ)o(e)f(at)f(eac)o(h)i(pro)q(cessor,)f(then)g(it)h(is)g (su\016cien)o(t)f(to)g(allo)q(cate)h(one)f(con)o(text)f(p)q(er)75 553 y(library)l(.)75 687 y Fj(9.7.2)55 b(P)n(arallel)26 b(pro)r(cedures)f(that)i(are)f(nonreen)n(tran)n(t)g(within)g(eac)n(h)h (executing)246 746 y(group)75 836 y Fs(This)21 b(co)o(v)o(ers)g(the)g (case)f(where,)j(at)d(an)o(y)g(p)q(oin)o(t)i(in)f(time,)h(for)f(eac)o (h)g(pro)q(cess)g(group,)g(there)g(can)g(b)q(e)75 892 y(at)d(most)g(one)h(activ)o(e)g(in)o(v)o(ok)m(ation)g(of)g(a)f (parallel)i(pro)q(cedure)g(b)o(y)f(a)f(pro)q(cess)h(mem)o(b)q(er.)31 b(Ho)o(w)o(ev)o(er,)19 b(it)75 948 y(migh)o(t)i(b)q(e)h(p)q(ossible)i (that)c(the)i(same)f(pro)q(cedure)h(is)g(concurren)o(tly)g(in)o(v)o(ok) o(ed)g(in)g(t)o(w)o(o)e(partially)j(\(or)75 1005 y(completely\))17 b(o)o(v)o(erlapping)f(groups.)k(F)l(or)15 b(example,)i(the)e(same)g (collectiv)o(e)j(comm)o(unication)e(function)75 1061 y(ma)o(y)e(b)q(e)i(concurren)o(tly)g(in)o(v)o(ok)o(ed)g(on)f(t)o(w)o(o) f(partially)i(o)o(v)o(erlapping)g(groups.)166 1120 y(In)i(suc)o(h)h(a)e (case,)h(a)g(con)o(text)p 674 1120 V 16 w(id)h(is)f(asso)q(ciated)g (with)g(eac)o(h)g(parallel)i(pro)q(cedure)e(and)g(eac)o(h)g(exe-)75 1176 y(cuting)f(group,)e(so)h(that)f(o)o(v)o(erlapping)i(execution)g (groups)e(ha)o(v)o(e)h(distinct)h(comm)o(unication)g(con)o(texts.)75 1233 y(\(One)c(do)q(es)g(not)g(need)h(a)e(di\013eren)o(t)h(con)o(text)p 831 1233 V 16 w(id)h(from)e(eac)o(h)h(group;)g(one)g(merely)g(needs)h (a)e(\\coloring")h(of)75 1289 y(the)j(groups,)f(so)g(that)g(One)h(can)g (generate)f(the)h(comm)o(unicators)f(for)g(eac)o(h)h(parallel)h(pro)q (cedure)f(when)75 1346 y(the)g(execution)i(groups)e(are)g(de\014ned.)25 b(Here,)17 b(again,)f(one)g(only)h(need)h(one)e(con)o(text)g(for)g(eac) o(h)g(library)l(,)75 1402 y(if)g(no)f(t)o(w)o(o)f(pro)q(cedures)i(from) e(the)h(same)g(library)h(can)g(b)q(e)g(concurren)o(tly)f(activ)o(e)h (in)g(the)f(same)g(group.)166 1461 y(Note)j(that,)h(for)f(collectiv)o (e)j(comm)o(unication)e(libraries,)i(w)o(e)d(do)h(allo)o(w)g(sev)o (eral)g(concurren)o(t)g(in-)75 1517 y(v)o(ok)m(ations)13 b(within)h(the)f(same)g(group:)18 b(a)13 b(broadcast)f(in)i(a)f(group)f (ma)o(y)h(b)q(e)g(started)f(at)h(a)f(pro)q(cess)i(b)q(efore)75 1574 y(the)h(previous)h(broadcast)f(in)h(that)e(group)h(ended)h(at)f (another)g(pro)q(cess.)20 b(In)c(suc)o(h)f(a)g(case,)g(one)g(cannot)75 1630 y(rely)e(on)f(con)o(text)g(mec)o(hanisms)h(to)f(disam)o(biguate)h (successiv)o(e)h(in)o(v)o(ok)m(ations)e(of)g(the)h(same)f(parallel)i (pro-)75 1687 y(cedure)f(within)h(the)f(same)f(group:)18 b(the)13 b(pro)q(cedure)g(need)h(b)q(e)f(implemen)o(ted)h(so)e(as)g(to) g(a)o(v)o(oid)h(confusion.)75 1743 y(E.g.,)h(for)h(broadcast,)f(one)h (ma)o(y)g(need)h(to)f(carry)f(additional)j(information)f(in)g (messages,)e(suc)o(h)i(as)f(the)75 1800 y(broadcast)h(ro)q(ot,)g(to)g (help)i(in)f(suc)o(h)g(disam)o(biguation;)i(one)d(also)h(relies)h(on)f (preserv)m(ation)g(of)f(message)75 1856 y(order)e(b)o(y)g(MPI.)g(With)g (suc)o(h)h(an)f(approac)o(h,)f(w)o(e)h(ma)o(y)g(b)q(e)h(gaining)g(p)q (erformance,)f(but)g(w)o(e)g(lo)q(ose)h(mo)q(d-)75 1913 y(ularit)o(y)l(.)28 b(It)18 b(is)g(not)f(su\016cien)o(t)i(to)e (implemen)o(t)i(the)f(parallel)h(pro)q(cedure)g(so)e(that)g(it)h(w)o (orks)f(correctly)75 1969 y(in)i(isolation,)h(when)g(in)o(v)o(ok)o(ed)e (only)i(once;)g(it)f(needs)g(to)f(b)q(e)i(implemen)o(ted)g(so)e(that)g (an)o(y)g(n)o(um)o(b)q(er)h(of)75 2026 y(successiv)o(e)d(in)o(v)o(ok)m (ations)f(will)h(execute)g(correctly)l(.)k(Of)14 b(course,)h(the)g (same)f(approac)o(h)g(can)h(b)q(e)g(used)h(for)75 2082 y(other)f(parallel)i(libraries.)75 2216 y Fj(9.7.3)55 b(W)-5 b(ell)18 b(nested)g(parallel)f(pro)r(cedures)75 2306 y Fs(Calls)e(of)g(parallel)h(pro)q(cedures)g(are)e(w)o(ell)i (nested)f(if)g(a)g(new)g(parallel)h(pro)q(cedure)g(is)f(alw)o(a)o(ys)f (in)o(v)o(ok)o(ed)h(in)75 2363 y(a)j(subset)h(of)e(a)h(group)g (executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 2419 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 2478 y(In)e(suc)o(h)f(a)g(case,)g(a)g(new)g (con)o(text)f(need)i(to)f(b)q(e)h(dynamically)g(allo)q(cated)h(for)d (eac)o(h)h(new)g(in)o(v)o(ok)m(ation)75 2534 y(of)d(a)g(parallel)i(pro) q(cedure.)20 b(Ho)o(w)o(ev)o(er,)10 b(a)g(stac)o(k)g(mec)o(hanism)h (can)f(b)q(e)i(used)f(for)f(allo)q(cating)h(new)g(con)o(texts.)75 2591 y(Th)o(us,)19 b(a)f(p)q(ossible)j(mec)o(hanism)e(is)g(to)f(allo)q (cate)h(\014rst)f(a)h(large)f(n)o(um)o(b)q(er)h(of)f(con)o(text)p 1566 2591 V 16 w(id's)h(\(up)g(to)f(the)75 2647 y(upp)q(er)h(b)q(ound)g (on)f(the)g(depth)h(of)f(nested)g(parallel)i(pro)q(cedure)f(calls\),)g (and)g(then)f(use)h(a)e(lo)q(cal)j(stac)o(k)75 2704 y(managemen)o(t)f (of)h(these)g(con)o(text)p 674 2704 V 15 w(id's)h(on)e(eac)o(h)h(pro)q (cess)h(to)e(create)g(a)h(new)g(comm)o(unicator)g(\(using)p eop %%Page: 84 89 84 88 bop 75 -100 a Fs(84)694 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y Fh(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(MAKE)p Fs(\))g(for)f(eac)o(h)i (new)f(in)o(v)o(ok)m(ation.)166 184 y Fl(Discussion:)g Fk(General)f(case)166 323 y Fs(In)h(the)g(general)g(case,)g(there)f(ma) o(y)g(b)q(e)i(m)o(ultiple)g(concurren)o(tly)f(activ)o(e)g(in)o(v)o(ok)m (ations)g(of)g(the)f(same)75 380 y(parallel)23 b(pro)q(cedure)f(within) g(the)g(same)f(group;)i(in)o(v)o(ok)m(ations)f(ma)o(y)f(not)g(b)q(e)h (w)o(ell)g(nested.)38 b(A)22 b(new)75 436 y(con)o(text)17 b(need)i(to)e(b)q(e)h(created)g(for)f(eac)o(h)g(in)o(v)o(ok)m(ation.)28 b(It)18 b(is)g(the)g(user)g(resp)q(onsibilit)o(y)i(to)d(mak)o(e)g(sure) 75 493 y(that,)k(if)h(t)o(w)o(o)d(distinct)j(parallel)h(pro)q(cedures)e (are)g(in)o(v)o(ok)o(ed)g(concurren)o(tly)h(on)e(o)o(v)o(erlapping)i (sets)e(of)75 549 y(pro)q(cesses,)15 b(then)h(con)o(text)p 538 549 14 2 v 16 w(id)g(allo)q(cation)g(or)f(comm)o(unicator)f (creation)i(is)g(prop)q(erly)g(co)q(ordinated.)75 692 y Fm(9.8)70 b(Left)75 794 y Fs(Remote)15 b(pro)q(cedure)h(call)h(and)e (C2)g(ob)s(jects.)166 850 y(Ho)o(w)f(to)h(handle)h(gro)o(wing)f(pro)q (cess)h(set.)p eop %%Page: 85 90 85 89 bop 75 379 a Fu(Biblio)q(graph)m(y)98 604 y Fs([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 661 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 717 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 820 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 876 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 933 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1035 y([3])j(R.)14 b(Butler)h(and)f(E.)f(Lusk.)18 b(User's)13 b(guide)i(to)e(the)h(P4)g(programming)f(system.)k(T)l(ec)o(hnical)f (Rep)q(ort)168 1092 y(TM-ANL{92/17,)e(Argonne)h(National)h(Lab)q (oratory)l(,)e(1992.)98 1194 y([4])21 b(S.)c(Chittor)g(and)h(R.)f(J.)g (En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d (mesh{connected)i(w)o(ormhole{)168 1251 y(routed)f(net)o(w)o(orks)e (for)i(in)o(terpro)q(cessor)f(comm)o(unication)i(in)f(m)o (ulticomputers.)25 b(In)17 b Fi(Pr)n(o)n(c)n(e)n(e)n(dings)168 1307 y(of)g(the)f(1990)i(Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p Fs(,)c(pages)j(647{656,)e(1990.)98 1410 y([5])21 b(S.)f(Chittor)g(and)g (R.)g(J.)f(En)o(b)q(o)q(dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g (of)f(mapping)i(on)e(the)h(comm)o(unica-)168 1466 y(tion)g(p)q (erformance)g(of)f(large)g(m)o(ulticomputers.)34 b(In)20 b Fi(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f(1991)h(International)168 1523 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n (essing,)f(vol.)i(II)f(\(Softwar)n(e\))p Fs(,)f(pages)h(I)q(I{1)h({)f (I)q(I{4,)g(1991.)98 1626 y([6])21 b(J.)13 b(Dongarra,)f(R.)h(Hemp)q (el,)i(A.)e(Hey)l(,)g(and)h(D.)e(W)l(alk)o(er.)k(A)e(prop)q(osal)f(for) f(a)h(user{lev)o(el,)i(message{)168 1682 y(passing)i(in)o(terface)g(in) g(a)f(distributed)i(memory)e(en)o(vironmen)o(t.)23 b(T)l(ec)o(hnical)18 b(Rep)q(ort)f(TM-12231,)168 1739 y(ORNL,)g(Octob)q(er)e(1992.)98 1841 y([7])21 b(Edin)o(burgh)g(P)o(arallel)f(Computing)g(Cen)o(tre,)g (Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fi(CHIMP)19 b(Conc)n(epts)p Fs(,)168 1898 y(June)e(1991.)98 2000 y([8])k(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f(Univ)o (ersit)o(y)h(of)g(Edin)o(burgh.)22 b Fi(CHIMP)15 b(V)m(ersion)h(1.0)168 2057 y(Interfac)n(e)p Fs(,)e(Ma)o(y)g(1992.)98 2159 y([9])21 b(G.)13 b(Geist)f(and)h(V.)g(Sunderam.)k(Net)o(w)o(ork)11 b(based)i(concurren)o(t)g(computing)h(on)e(the)h(PVM)g(system.)168 2216 y(T)l(ec)o(hnical)k(Rep)q(ort)f(TM-11760,)d(Oak)i(Ridge)i (National)e(Lab)q(oratory)l(,)f(June)j(1991.)75 2319 y([10])k(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o(eyton,) f(and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f (PICL:)g(a)168 2375 y(p)q(ortable)f(instrumen)o(ted)f(comm)o(unication) h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e(TM-11616,)f(Oak)h(Ridge) 168 2432 y(National)k(Lab)q(oratory)l(,)e(Octob)q(er)i(1990.)75 2534 y([11])21 b(R.)15 b(Hemp)q(el.)20 b(The)14 b(ANL/GMD)g(macros)f (\(P)l(ARMA)o(CS\))h(in)i(fortran)d(for)g(p)q(ortable)i(parallel)h (pro-)168 2591 y(gramming)f(using)g(the)g(message)f(passing)h (programming)f(mo)q(del)i({)e(users')h(guide)g(and)g(reference)168 2647 y(man)o(ual.)23 b(T)l(ec)o(hnical)18 b(rep)q(ort,)d(GMD,)g(P)o (ostfac)o(h)f(1316,)h(D-5205)g(Sankt)h(Augustin)g(1,)g(German)o(y)l(,) 168 2704 y(No)o(v)o(em)o(b)q(er)f(1991.)952 2828 y(85)p eop %%Page: 86 91 86 90 bop 75 -100 a Fs(86)1391 b Fq(BIBLIOGRAPHY)75 45 y Fs([12])21 b(R.)15 b(Hemp)q(el,)h(H.-C.)e(Hopp)q(e,)h(and)f(A.)h (Supalo)o(v.)k(P)o(armacs{6.0)12 b(library)k(in)o(terface)e(sp)q (eci\014cation.)168 102 y(T)l(ec)o(hnical)h(rep)q(ort,)d(GMD,)g(P)o (ostfac)o(h)f(1316,)h(D-5205)f(Sankt)h(Augustin)i(1,)e(German)o(y)l(,)h (Decem)o(b)q(er)168 158 y(1992.)75 252 y([13])21 b(nCUBE)16 b(Corp)q(oration.)j Fi(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i (r2.0)p Fs(,)f(Decem)o(b)q(er)f(1990.)75 346 y([14])21 b(H.)c(M)q(\177)-24 b(uhlen)o(b)q(ein)19 b(O.)d(Kr\177)-23 b(amer.)24 b(Mapping)16 b(strategies)g(in)i(message{based)e(m)o (ultipro)q(cessor)h(sys-)168 402 y(tems.)j Fi(Par)n(al)r(lel)15 b(Computing)p Fs(,)g(9:213{225,)d(1989.)75 496 y([15])21 b(P)o(arasoft)14 b(Corp)q(oration.)22 b Fi(Expr)n(ess)16 b(V)m(ersion)g(1.0:)22 b(A)17 b(Communic)n(ation)g(Envir)n(onment)e (for)j(Par-)168 552 y(al)r(lel)e(Computers)p Fs(,)f(1988.)75 646 y([16])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g (system.)22 b(In)17 b Fi(Pr)n(o)n(c)n(e)n(e)n(dings)e(of)i(the)g(Thir)n (d)g(Confer)n(enc)n(e)e(on)168 703 y(Hyp)n(er)n(cub)n(e)g(Concurr)n (ent)f(Computers)i(and)f(Applic)n(ations)p Fs(,)e(pages)h(384{390.)d(A) o(CM)j(Press,)f(1988.)75 796 y([17])21 b(A.)14 b(Skjellum)i(and)e(A.)g (Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f (comm)o(unication)h(library)168 853 y(atop)g(the)f(reactiv)o(e)h(k)o (ernel.)20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g (editors,)g Fi(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 909 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g (Confer)n(enc)n(e)p Fs(,)e(pages)i(767{776.)d(IEEE)168 966 y(Press,)d(1990.)75 1060 y([18])21 b(A.)12 b(Skjellum,)i(S.)e (Smith,)h(C.)e(Still,)j(A.)d(Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q (co)q(de)g(message)f(passing)168 1116 y(system.)20 b(T)l(ec)o(hnical)d (rep)q(ort,)d(La)o(wrence)i(Liv)o(ermore)f(National)h(Lab)q(oratory)l (,)e(Septem)o(b)q(er)i(1992.)75 1210 y([19])21 b(V.)e(Sunderam.)30 b(PVM:)18 b(a)g(framew)o(ork)f(for)h(parallel)i(distributed)h (computing.)30 b Fi(Concurr)n(ency:)168 1266 y(Pr)n(actic)n(e)16 b(and)g(Exp)n(erienc)n(e)p Fs(,)e(2\(4\):315{339)o(,)e(1990.)75 1360 y([20])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g (passing)g(in)i(a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec) o(h-)168 1417 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National) g(Lab)q(oratory)l(,)e(August)h(1992.)p eop %%Page: 87 92 87 91 bop 75 356 a Fn(App)s(endix)34 b(A)75 564 y Fu(Pro)s(cess)40 b(T)-10 b(op)s(ology)42 b(Routines)75 786 y Fs(In)16 b(this)f(app)q(endix)i(F)l(ortran)d(sp)q(eci\014cations)j(for)e(the)g (prop)q(osed)h(top)q(ology)f(functions)h(are)f(giv)o(en.)191 895 y Fo(\017)g Ft(MPI)p 338 895 16 2 v 18 w(CAR)l(T)149 b Fs(Map)15 b(a)g(cartesian)g(structure)g(\(grid/torus\))191 952 y Fo(\017)g Ft(MPI)p 338 952 V 18 w(MAKDIM)57 b Fs(Divide)17 b(a)d(pro)q(cess)i(group)f(among)f(n)i(cartesian)f(dimensions)191 1008 y Fo(\017)g Ft(MPI)p 338 1008 V 18 w(P)l(AR)l(TC)117 b Fs(P)o(artition)15 b(a)g(cartesian)g(structure)191 1065 y Fo(\017)g Ft(MPI)p 338 1065 V 18 w(GRAPH)101 b Fs(Map)15 b(a)g(graph)191 1121 y Fo(\017)g Ft(MPI)p 338 1121 V 18 w(INQMAP)72 b Fs(Chec)o(k)15 b(mapping)h(t)o(yp)q(e)191 1178 y Fo(\017)f Ft(MPI)p 338 1178 V 18 w(INQCAR)l(T)49 b Fs(Find)16 b(size)g(of)f(cartesian)g(structure)g(and)g(o)o(wn)g(p)q (osition)191 1234 y Fo(\017)g Ft(MPI)p 338 1234 V 18 w(INQGR)118 b Fs(Find)16 b(o)o(wn)f(no)q(de)g(n)o(um)o(b)q(er)h(in)g (graph)191 1291 y Fo(\017)f Ft(MPI)p 338 1291 V 18 w(INQPID)102 b Fs(Lo)q(ok)15 b(up)h(PID)f(n)o(um)o(b)q(ers)952 2828 y(87)p eop %%Page: 88 93 88 92 bop 75 -100 a Fs(88)663 b Fq(APPENDIX)16 b(A.)30 b(PR)o(OCESS)16 b(TOPOLOGY)g(R)o(OUTINES)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(CAR)l(T)121 b Fs(Arrange)16 b(the)g(pro)q(cesses)h(of)f(a)g(pro)q(cess)h(group)f(in)h(a)f (cartesian)g(structure)573 158 y(\(torus/grid\))e(with)i Fh(n)f Fs(dimensions.)75 256 y Ft(SYNOPSIS)179 312 y Fs(in)o(teger)g(function)h(MPI)p 599 312 14 2 v 17 w(CAR)l(T)f(\(gid,)g (ndim,)h(dims,)f(p)q(erio)q(d\))179 369 y(in)o(teger)g(gid)179 425 y(in)o(teger)g(ndim)179 482 y(in)o(teger)g(dims\()p Fo(\003)p Fs(\))179 538 y(in)o(teger)g(p)q(erio)q(d\()p Fo(\003)p Fs(\))75 636 y Ft(INPUT)i(AR)o(GUMENTS)179 693 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q (cess)h(group)f(to)f(b)q(e)i(structured.)179 749 y(ndim)294 b(the)15 b(n)o(um)o(b)q(er)h(of)f(dimensions)i(of)d(the)i(torus.)179 806 y(dims)301 b(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(in)g(eac)o(h) f(co)q(ordinate)h(direction.)179 862 y(p)q(erio)q(d)269 b(p)q(erio)q(dicit)o(y)18 b(sp)q(eci\014cation)f(in)f(eac)o(h)f(co)q (ordinate)h(direction.)75 960 y Ft(DESCRIPTION)179 1016 y Fs(The)f(pro)q(cesses)g(of)g(the)g(group)g(with)g(ID)g(n)o(um)o(b)q (er)g Fh(gid)g Fs(are)f(arranged)h(in)h(a)e(cartesian)h(structure)179 1073 y(\(grid/torus\).)j(By)d(assigning)h(neigh)o(b)q(oring)g(p)q (ositions)g(in)f(the)g(logical)h(pro)q(cess)f(torus)g(to)f(neigh-)179 1129 y(b)q(oring)e(pro)q(cessors)g(in)h(the)f(hardw)o(are)f(the)h (system)g(can)g(optimize)h(the)f(lo)q(calit)o(y)h(of)f(in)o(ter{pro)q (cess)179 1186 y(comm)o(unications.)20 b(The)15 b(n)o(um)o(b)q(er)f(of) g(dimensions)i(is)f(sp)q(eci\014ed)h(b)o(y)e Fh(ndim)p Fs(.)19 b(Arra)o(y)14 b Fh(period)f Fs(sp)q(ec-)179 1242 y(i\014es)18 b(the)g(p)q(erio)q(dicit)o(y)i(in)e(eac)o(h)f(co)q (ordinate)h(direction.)29 b(If)17 b Fh(period\(i\))g Fs(is)h(set)f(to)g Ft(MPI)p 1751 1242 16 2 v 18 w(PER)179 1299 y Fs(then)j(dimension)h Fh(i)e Fs(is)h(treated)f(as)g(p)q(erio)q (dic.)34 b(If)20 b Fh(period\(i\))e Fs(is)i(set)f(to)g Ft(MPI)p 1602 1299 V 18 w(NPER)h Fs(then)179 1355 y(dimension)e Fh(i)d Fs(is)i(treated)f(as)f(non-p)q(erio)q(dic.)25 b(If)17 b(all)g Fh(period\(i\))e Fs(are)g(set)h(to)g Ft(MPI)p 1634 1355 V 18 w(PER)p Fs(,)g(then)179 1412 y(the)f(top)q(ology)g(is)h(a)f(torus.)179 1487 y(Note)c(that)h(a)g (n-dimension)i(h)o(yp)q(ercub)q(e)f(is)g(a)f(torus)f(with)h(2)g(pro)q (cesses)h(p)q(er)f(co)q(ordinate)h(direction)179 1543 y(and)i(MPI)p 359 1543 14 2 v 17 w(PER.)g(Th)o(us,)f(ndim=n,)j (dims\(1:n\)=2)e(and)g(p)q(erio)q(d\(1:n\)=MPI)p 1513 1543 V 17 w(PER.)75 1641 y Ft(RETURN)k(V)-6 b(ALUE)179 1698 y Fs(If)14 b(the)g(calling)h(pro)q(cess)f(is)g(assigned)h(a)e(p)q (osition)i(in)f(the)g(grid,)g Ft(MPI)p 1372 1698 16 2 v 18 w(CAR)l(T)f Fs(returns)h(the)g(total)179 1754 y(n)o(um)o(b)q(er)k (of)f(pro)q(cesses)h(in)h(the)f(grid.)28 b(If)18 b(the)g(calling)h(pro) q(cess)f(is)h(not)e(in)i(the)f(grid,)g(the)g(return)179 1811 y(v)m(alue)e(is)g(set)f(to)f(0.)20 b(A)15 b(v)m(alue)i(of)d(-1)h (is)h(returned)g(if)f(an)h(error)e(o)q(ccurs.)p eop %%Page: 89 94 89 93 bop 1830 -100 a Fs(89)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(MAKDIM)29 b Fs(Divide)18 b(the)e(pro)q(cesses)h(of) e(a)h(pro)q(cess)h(group)f(among)f(n)i(dimensions)g(of)f(a)573 158 y(Cartesian)f(grid.)75 256 y Ft(SYNOPSIS)179 312 y Fs(in)o(teger)g(function)h(MPI)p 599 312 14 2 v 17 w(MAKDIM)f(\(gid,)g(ndim,)g(dims\))179 369 y(in)o(teger)g(gid)179 425 y(in)o(teger)g(ndim)179 482 y(in)o(teger)g(dims\()p Fo(\003)p Fs(\))75 580 y Ft(INPUT)i(AR)o(GUMENTS)179 636 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q (cess)h(group)f(to)f(b)q(e)i(structured.)179 693 y(ndim)294 b(the)15 b(n)o(um)o(b)q(er)h(of)f(dimensions)i(of)d(the)i(torus.)179 749 y(dims)301 b(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(in)g(eac)o(h) f(co)q(ordinate)h(direction.)75 847 y Ft(OUTPUT)j(AR)o(GUMENTS)179 904 y Fs(dims)301 b(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(in)g(eac)o (h)f(co)q(ordinate)h(direction.)75 1002 y Ft(DESCRIPTION)179 1058 y Fs(The)11 b(pro)q(cesses)g(of)g(the)f(group)h(with)g(ID)g(n)o (um)o(b)q(er)g Fh(gid)g Fs(are)f(divided)j(as)e(ev)o(enly)h(as)e(p)q (ossible)j(among)179 1114 y Fh(ndim)k Fs(dimensions,)j(sub)s(ject)e(to) g(the)g(constrain)o(t)g(that)f(all)i(pro)q(cesses)g(in)g(the)f(group)g (are)g(used.)179 1171 y(The)e(caller)h(ma)o(y)e(further)h(constrain)g (the)g(op)q(eration)g(of)g(this)g(routine)h(b)o(y)e(sp)q(ecifying)j (elemen)o(ts)179 1227 y(of)c(arra)o(y)g Fh(dims)p Fs(.)20 b(If)15 b Fh(dims\(i\))f Fs(is)i(set)f(to)f(a)h(p)q(ositiv)o(e)h(n)o (um)o(b)q(er,)f(the)h(routine)f(will)i(\014x)e(the)h(n)o(um)o(b)q(er) 179 1284 y(of)g(pro)q(cesses)g(in)i(co)q(ordinate)e(direction)i Fh(i)e Fs(to)g Fh(dims\(i\))p Fs(.)22 b(Pro)q(cesses)17 b(will)h(then)e(b)q(e)h(distributed)179 1340 y(among)d(directions)j (where)e Fh(dims\(i\))f Fs(is)i(set)f(to)g(0,)f(sub)s(ject)h(to)g(the)g (ab)q(o)o(v)o(e)g(constrain)o(ts.)179 1415 y(Note)j(that)h(an)g(error)f (will)j(o)q(ccur)e(if)g Fh(dims\(i\))f Fs(is)i(p)q(ositiv)o(e)g(and)f (do)q(es)g(not)g(ev)o(enly)h(divide)h(the)179 1472 y(n)o(um)o(b)q(er)d (of)g(pro)q(cesses)g(in)h(the)f(group.)28 b(Negativ)o(e)18 b(input)h(v)m(alues)g(of)f Fh(dims\(i\))f Fs(will)j(also)e(cause)179 1528 y(an)e(error.)22 b(All)17 b(elemen)o(ts)g(of)f Fh(dims\(*\))f Fs(m)o(ust)g(b)q(e)i(set)f(to)f(0)h(for)g(completely)h(automatic)f(pro) q(cess)179 1585 y(distribution.)179 1660 y(When)21 b(the)g(function)g (returns,)h Fh(dims\(i\))d Fs(will)k(b)q(e)e(set)f(to)g(the)h(n)o(um)o (b)q(er)g(of)f(pro)q(cesses)h(in)h(co-)179 1716 y(ordinate)h(direction) i Fh(i)p Fs(.)45 b(F)l(or)22 b Fh(dims\(i\))h Fs(not)g(set)g(b)o(y)g (the)h(user,)h Fh(dims\(i\))e Fs(will)i(b)q(e)f(ordered)179 1773 y(\(MAXIMUM,)18 b(...,)g(MINIMUM\).)g(Arra)o(y)g Fh(dim\(*\))g Fs(is)h(suitable)h(for)e(use)h(as)g(input)g(to)f(routine) 179 1829 y(MPI)p 271 1829 V 16 w(CAR)l(T.)75 1927 y Ft(RETURN)h(V)-6 b(ALUE)179 1984 y Fs(If)18 b(the)g(routine)g(completes)g(without)g (error,)f(the)h(return)f(v)m(alue)i(is)f(set)g(to)f(0.)27 b(A)18 b(v)m(alue)h(of)e(-1)g(is)179 2040 y(returned)e(if)h(an)f(error) g(o)q(ccurs.)p eop %%Page: 90 95 90 94 bop 75 -100 a Fs(90)663 b Fq(APPENDIX)16 b(A.)30 b(PR)o(OCESS)16 b(TOPOLOGY)g(R)o(OUTINES)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(P)l(AR)l(TC)89 b Fs(P)o(artition)15 b(a)g(pro)q(cess)g(group)g(with)h(cartesian)f(structure)g(in)h (subgroups.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)f(function)h (MPI)p 599 256 14 2 v 17 w(P)l(AR)l(TC)f(\(gid,)g(lostdim\))179 312 y(in)o(teger)g(gid)179 369 y(in)o(teger)g(lostdim)75 467 y Ft(INPUT)i(AR)o(GUMENTS)179 523 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q(cess)h(group)f(to)f(b)q(e)i (partitioned.)179 580 y(lostdim)248 b(the)12 b(co)q(ordinate)g (direction)h(b)o(y)f(whic)o(h)g(the)g(pro)q(cess)g(map)f(will)j(b)q(e)e (reduced.)75 678 y Ft(DESCRIPTION)179 734 y Fs(It)19 b(is)h(assumed)f(that)g(the)g(pro)q(cesses)h(of)f(the)g(group)g(with)h (ID)f(n)o(um)o(b)q(er)h Fh(gid)e Fs(are)h(arranged)g(in)179 791 y(a)k(cartesian)h(structure)f(of)h(size)g(\(n1,n2,...\))43 b(b)o(y)24 b(a)f(previous)i(call)g(to)e(MPI)p 1580 791 V 16 w(CAR)l(T.)h(Then,)179 847 y(MPI)p 271 847 V 16 w(P)l(AR)l(TC)e(creates)f(a)f(n)o(um)o(b)q(er)i(of)f(subgroups)g(with)g (cartesian)h(structures,)f(the)h(dimen-)179 904 y(sion)c(of)f(whic)o(h) h(is)g(one)g(lo)o(w)o(er)f(than)h(its)g(paren)o(t's.)26 b(The)18 b(direction)g(of)g(splitting)h(of)e(the)g(paren)o(t)179 960 y(structure)g(is)h(sp)q(eci\014ed)i(b)o(y)e Fh(lostdim)p Fs(.)26 b(If,)18 b(for)f(example,)i Fh(lostdim)k(=)h(2)p Fs(,)17 b(then)i Fh(n2)e Fs(subgroups)179 1016 y(with)e(size)h (\(n1,n3,n4,...\))i(are)d(formed.)75 1114 y Ft(RETURN)k(V)-6 b(ALUE)179 1171 y Fs(Up)q(on)16 b(successful)h(completion)g Ft(MPI)p 852 1171 16 2 v 18 w(P)l(AR)l(TC)d Fs(returns)i(the)f(group)h (ID)g(n)o(um)o(b)q(er)f(of)h(the)f(sub-)179 1227 y(group)d(to)h(whic)o (h)g(the)g(calling)i(pro)q(cess)e(b)q(elongs.)20 b(A)13 b(v)m(alue)h(of)e(-1)h(is)g(returned)h(if)f(an)g(error)f(o)q(ccurs.)p eop %%Page: 91 96 91 95 bop 1830 -100 a Fs(91)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(GRAPH)73 b Fs(Arrange)13 b(the)h(pro)q(cesses)g(of) g(a)f(pro)q(cess)h(group)f(in)i(a)e(general)i(graph)e(struc-)573 158 y(ture.)75 256 y Ft(SYNOPSIS)179 312 y Fs(in)o(teger)i(function)h (MPI)p 599 312 14 2 v 17 w(GRAPH)f(\(gid,)g(no)q(des,)g(index,)i (nglist\))179 369 y(in)o(teger)e(gid)179 425 y(in)o(teger)g(no)q(des) 179 482 y(in)o(teger)g(index\()p Fo(\003)p Fs(\))179 538 y(in)o(teger)g(nglist\()p Fo(\003)p Fs(\))75 636 y Ft(INPUT)i(AR)o(GUMENTS)179 693 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q(cess)h(group)f(to)f(b)q(e)i (structured.)179 749 y(no)q(des)282 b(the)15 b(n)o(um)o(b)q(er)h(of)f (no)q(des)g(in)i(the)e(graph.)179 806 y(index)288 b(index)17 b(v)o(ector)d(in)o(to)h(v)o(ector)g(nglist)h(\(see)f(b)q(elo)o(w\).)179 862 y(nglist)286 b(for)15 b(eac)o(h)g(graph)g(no)q(de:)20 b(list)c(of)f(neigh)o(b)q(oring)i(no)q(des)f(\(see)f(b)q(elo)o(w\).)75 960 y Ft(DESCRIPTION)179 1016 y Fs(The)k(pro)q(cesses)g(of)f(the)g (group)h(with)g(ID)f(n)o(um)o(b)q(er)h Fh(gid)f Fs(are)h(arranged)f(in) h(a)f(graph)h(structure.)179 1073 y(By)d(assigning)h(neigh)o(b)q(oring) h(graph)e(no)q(des)h(to)f(neigh)o(b)q(oring)i(pro)q(cessors)e(in)h(the) f(hardw)o(are)g(the)179 1129 y(system)k(can)h(optimize)h(the)e(lo)q (calit)o(y)i(of)f(in)o(ter{pro)q(cess)f(comm)o(unications.)37 b(The)21 b(n)o(um)o(b)q(er)g(of)179 1186 y(no)q(des)16 b(is)f(sp)q(eci\014ed)j(b)o(y)d Fh(nodes)p Fs(.)179 1242 y(Input)i(to)e(MPI)p 453 1242 V 16 w(GRAPH)h(is)h(for)e(eac)o(h)h(no)q (de)h(its)f(set)g(of)f(neigh)o(b)q(ors,)i(i.)f(e.,)f(the)h(set)g(of)g (pro)q(cesses)179 1299 y(with)g(whic)o(h)h(the)f(no)q(de)h(pro)q(cess)f (is)g(going)g(to)g(comm)o(unicate)g(most)f(frequen)o(tly)l(.)23 b(Since)17 b(there)f(is)179 1355 y(no)e(w)o(eigh)o(ting)h(of)f(no)q(de) h(connections,)g(it)g(migh)o(t)f(b)q(e)h(sensible)i(to)c(sp)q(ecify)j (only)f(those)f(neigh)o(b)q(ors)179 1412 y(with)h(whic)o(h)h(there)g (is)f(a)g(considerable)i(amoun)o(t)e(of)f(comm)o(unication.)179 1468 y(F)l(or)9 b(eac)o(h)i(no)q(de)f Fr(i)j Fs(\(1)f Fo(\024)h Fr(i)f Fo(\024)h Fr(nodes)p Fs(\))d(the)g(list)h(of)f(the)g (neigh)o(b)q(or)h(no)q(des)g(is)g(stored)e(at)h Fr(ng)r(l)q(ist)p Fs(\()p Fr(j)s Fs(\))p Fr(;)18 b(index)p Fs(\()p Fr(i)p Fs(\))11 b Fo(\024)179 1525 y Fr(j)k Fo(\024)e Fr(index)p Fs(\()p Fr(i)8 b Fs(+)h(1\))g Fo(\000)g Fs(1.)19 b(F)l(or)14 b(the)h(last)g(pro)q(cess)g Fr(index)p Fs(\()p Fr(i)8 b Fs(+)h(1\))14 b(p)q(oin)o(ts)h(to)f(the)h(\014rst)f(free)h(p)q (osition)179 1581 y(in)k(v)o(ector)f(nglist.)30 b(Note)18 b(that)g(the)g(matrix)g(is)h(symmetric,)g(i.)g(e.,)f(if)h(no)q(de)g Fi(i)f Fs(is)h(a)f(neigh)o(b)q(or)h(of)179 1637 y(no)q(de)g Fi(j)p Fs(,)g(than)g(no)q(de)h Fi(j)e Fs(is)i(also)e(a)h(neigh)o(b)q (or)h(of)e(no)q(de)h Fi(i)p Fs(.)31 b(This)19 b(prop)q(ert)o(y)g(can)g (b)q(e)g(used)h(for)e(an)179 1694 y(in)o(ternal)e(c)o(hec)o(king)g(of)f (the)g(input)h(argumen)o(ts.)75 1792 y Ft(RETURN)j(V)-6 b(ALUE)179 1848 y Fs(If)17 b(the)f(calling)j(pro)q(cess)e(is)g (assigned)g(a)g(p)q(osition)g(in)h(the)f(graph,)f Ft(MPI)p 1440 1848 16 2 v 18 w(GRAPH)g Fs(returns)h(its)179 1905 y(no)q(de)k(n)o(um)o(b)q(er.)35 b(If)21 b(it)f(is)h(not)f(in)h(the)f (graph,)h(a)f(v)m(alue)i(of)d(0)h(is)h(returned.)36 b(A)20 b(v)m(alue)i(of)d(-1)h(is)179 1961 y(returned)15 b(if)h(an)f(error)g(o) q(ccurs.)p eop %%Page: 92 97 92 96 bop 75 -100 a Fs(92)663 b Fq(APPENDIX)16 b(A.)30 b(PR)o(OCESS)16 b(TOPOLOGY)g(R)o(OUTINES)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(INQMAP)44 b Fs(Find)16 b(out)f(mapping)h(t)o(yp)q(e)f(of)g(a)g(pro)q(cess)g(group.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)g(function)h(MPI)p 599 256 14 2 v 17 w(INQMAP)f(\(gid\))75 354 y Ft(INPUT)i(AR)o(GUMENTS) 179 410 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q (cess)h(group.)75 508 y Ft(DESCRIPTION)179 565 y Fs(The)h(function)h (MPI)p 546 565 V 16 w(INQMAP)f(tells)h(the)f(calling)i(pro)q(cess)e (the)g(t)o(yp)q(e)f(of)h(mapping)g(whic)o(h)h(has)179 621 y(b)q(een)13 b(applied)g(to)e(the)h(pro)q(cess)g(group)f(with)h (the)g(ID)g(n)o(um)o(b)q(er)g Fh(gid)p Fs(,)g(of)f(whic)o(h)h(the)g (calling)i(pro)q(cess)179 678 y(is)h(a)g(mem)o(b)q(er.)75 776 y Ft(RETURN)k(V)-6 b(ALUE)179 832 y Fs(Up)q(on)16 b(successful)g(completion)g Ft(MPI)p 850 832 16 2 v 18 w(INQMAP)e Fs(returns)204 885 y(MPI)p 296 885 14 2 v 16 w(NOMAP:)148 b(if)16 b(no)f(mapping)h(has)f(b)q(een)h(assigned,)204 941 y(MPI)p 296 941 V 16 w(CAR)l(TMAP:)88 b(in)16 b(case)g(of)e(a)h (cartesian)h(mapping,)f(and)204 998 y(MPI)p 296 998 V 16 w(GRAPHMAP:)49 b(in)16 b(case)g(of)e(a)h(graph)g(mapping.)179 1050 y(A)g(v)m(alue)h(of)f(-1)g(is)h(returned)g(if)f(an)g(error)g(o)q (ccurs.)p eop %%Page: 93 98 93 97 bop 1830 -100 a Fs(93)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(INQCAR)l(T)21 b Fs(Lo)q(ok)15 b(up)h(cartesian)f (structure)g(and)h(o)o(wn)e(p)q(osition.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)h(function)h(MPI)p 599 256 14 2 v 17 w(INQCAR)l(T)g(\(gid,)f(maxdim,)g(size,)h(p)q(erio)q(d,)g(p)q(os\)) 179 312 y(in)o(teger)f(gid)179 369 y(in)o(teger)g(maxdim)179 425 y(in)o(teger)g(size\()p Fo(\003)p Fs(\))179 482 y(in)o(teger)g(p)q (erio)q(d\()p Fo(\003)p Fs(\))179 538 y(in)o(teger)g(p)q(os\()p Fo(\003)p Fs(\))75 636 y Ft(INPUT)i(AR)o(GUMENTS)179 693 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q (cess)h(group)f(with)g(a)g(cartesian)g(structure.)179 749 y(maxdim)234 b(the)15 b(dimension)i(of)e(v)o(ectors)g Fh(size)f Fs(and)h Fh(pos)g Fs(in)h(the)g(calling)g(program.)75 847 y Ft(OUTPUT)j(AR)o(GUMENTS)179 904 y Fs(size)324 b(v)o(ector)15 b(returning)g(the)h(size)g(of)f(the)g(grid)g(in)h(eac)o (h)g(co)q(ordinate)f(direction.)179 960 y(p)q(erio)q(d)269 b(p)q(erio)q(dicit)o(y)18 b(c)o(haracteristics)d(in)h(eac)o(h)f(co)q (ordinate)h(direction.)179 1016 y(p)q(os)327 b(v)o(ector)15 b(returning)g(the)h(p)q(osition)g(of)f(the)g(calling)i(pro)q(cess)e(in) h(the)g(grid.)75 1114 y Ft(DESCRIPTION)179 1171 y Fs(After)11 b(the)i(pro)q(cesses)f(of)g(the)g(group)g Fh(gid)f Fs(ha)o(v)o(e)h(b)q (een)h(assigned)g(a)e(cartesian)i(structure)e(\(function)179 1227 y(MPI)p 271 1227 V 16 w(CAR)l(T\),)k(the)g(function)g(MPI)p 806 1227 V 17 w(INQCAR)l(T)h(tells)g(the)f(calling)h(pro)q(cess)f(the)g (parameters)f(of)179 1284 y(that)g(structure)h(and)h(its)f(o)o(wn)g (logical)h(p)q(osition.)179 1359 y(If,)d(for)f(example,)h(a)f (three{dimensional)j(grid)e(mapping)g(has)f(b)q(een)i(applied,)g(then)f Fh(size)f Fs(returns)179 1415 y(the)f(three)h(v)m(alues)h Fh(nx,ny,nz)p Fs(,)e(and)g Fh(period\(i\))p Fs(,)g(i=1,2,3,)g(is)h(set) g(to)f Ft(MPI)p 1475 1415 16 2 v 18 w(NPER)p Fs(.)g(The)h(v)o(ector)179 1472 y Fh(pos)j Fs(con)o(tains)h(the)g(logical)h(pro)q(cess)f(co)q (ordinates)g Fh(i,j,k)f Fs(with)i(1)c Fo(\024)h Fr(i)g Fo(\024)g Fr(nx)p Fs(,)h(1)f Fo(\024)g Fr(j)i Fo(\024)e Fr(ny)r Fs(,)i(and)179 1528 y(1)j Fo(\024)g Fr(k)h Fo(\024)f Fr(nz)r Fs(.)32 b(If)19 b(the)h(calling)g(pro)q(cess)g(is)f(not)g (assigned)g(a)g(p)q(osition)h(in)g(the)f(grid)h(structure,)179 1585 y(the)c(co)q(ordinates)h(are)e(set)h(to)g(zero)q(es.)23 b(If)16 b(the)g(v)o(ectors)g Fh(size)p Fs(,)f Fh(period)h Fs(and)g Fh(pos)g Fs(are)f(to)q(o)h(short,)179 1641 y(MPI)p 271 1641 14 2 v 16 w(INQCAR)l(T)h(rep)q(orts)d(an)i(error.)75 1739 y Ft(RETURN)j(V)-6 b(ALUE)179 1796 y Fs(Up)q(on)18 b(successful)h(completion)g Ft(MPI)p 858 1796 16 2 v 18 w(INQCAR)l(T)f Fs(returns)f(the)h(n)o(um)o(b)q(er)g(of)g(dimensions) h(of)179 1852 y(the)c(cartesian)g(structure,)g Fh(ndim)p Fs(.)k(A)c(v)m(alue)i(of)e(-1)g(is)g(returned)h(if)g(an)f(error)f(o)q (ccurs.)p eop %%Page: 94 99 94 98 bop 75 -100 a Fs(94)663 b Fq(APPENDIX)16 b(A.)30 b(PR)o(OCESS)16 b(TOPOLOGY)g(R)o(OUTINES)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(INQGR)90 b Fs(Lo)q(ok)15 b(up)h(size)g(of)f(graph)g(mapping)h(and)f(p)q(osition)h(of)f(calling)i (pro)q(cess.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)e(function)h (MPI)p 599 256 14 2 v 17 w(INQGR)g(\(gid,)f(no)q(des\))75 354 y Ft(INPUT)i(AR)o(GUMENTS)179 410 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q(cess)h(group)f(with)g(a)g (graph)g(structure.)75 508 y Ft(OUTPUT)k(AR)o(GUMENTS)179 565 y Fs(no)q(des)282 b(the)15 b(total)g(n)o(um)o(b)q(er)h(of)e(no)q (des)i(in)g(the)f(graph)g(structure.)75 663 y Ft(DESCRIPTION)179 719 y Fs(After)24 b(the)h(pro)q(cesses)f(of)h(the)f(group)g Fh(gid)h Fs(ha)o(v)o(e)f(b)q(een)h(structured)g(as)f(a)g(graph)g (\(function)179 776 y(MPI)p 271 776 V 16 w(GRAPH\),)f(the)g(function)h (MPI)p 870 776 V 17 w(INQGR)g(tells)g(the)g(calling)h(pro)q(cess)e(the) h(size)g(of)f(the)179 832 y(graph)c(and)h(its)h(o)o(wn)e(no)q(de)i(n)o (um)o(b)q(er.)34 b(No)q(des)20 b(are)g(n)o(um)o(b)q(ered)g(from)g(1)f (to)g Fh(nodes)h Fs(\(cf.)33 b(func-)179 889 y(tion)15 b(MPI)p 364 889 V 17 w(GRAPH\).)75 987 y Ft(RETURN)k(V)-6 b(ALUE)179 1043 y Fs(Up)q(on)16 b(successful)h(completion)g Ft(MPI)p 852 1043 16 2 v 18 w(INQGR)f Fs(returns)f(the)h(no)q(de)h(n)o (um)o(b)q(er)f Fh(i)f Fs(of)g(the)h(calling)179 1099 y(pro)q(cess,)e(1)e Fo(\024)h Fr(i)f Fo(\024)h Fr(nodes)p Fs(.)20 b(If)14 b(the)g(calling)i(pro)q(cess)e(is)h(not)e(assigned)i(a) e(no)q(de)i(in)g(the)f(graph,)f(zero)179 1156 y(is)i(returned.)21 b(A)15 b(v)m(alue)h(of)f(-1)g(is)h(returned)g(if)f(an)g(error)g(o)q (ccurs.)p eop %%Page: 95 100 95 99 bop 1830 -100 a Fs(95)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(INQPID)74 b Fs(Lo)q(ok)15 b(up)h(PID)f(n)o(um)o(b)q (ers)h(of)f(selected)h(pro)q(cesses.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)f(function)h(MPI)p 599 256 14 2 v 17 w(INQPID)g(\(gid,)f(npro)q(c,)g(co)q(ord,)g(plist\))179 312 y(in)o(teger)g(gid)179 369 y(in)o(teger)g(npro)q(c)179 425 y(in)o(teger)g(co)q(ord\()p Fo(\003)p Fs(\))179 482 y(in)o(teger)g(plist\()p Fo(\003)p Fs(\))75 580 y Ft(INPUT)i(AR)o (GUMENTS)179 636 y Fs(gid)334 b(the)22 b(ID)g(n)o(um)o(b)q(er)g(of)f (the)h(pro)q(cess)g(group)f(with)h(a)f(cartesian)h(or)f(graph)573 693 y(structure.)179 749 y(npro)q(c)282 b(the)22 b(n)o(um)o(b)q(er)g (of)g(pro)q(cesses)g(for)f(whic)o(h)i(the)f(PID)f(n)o(um)o(b)q(ers)i (are)e(to)g(b)q(e)573 806 y(lo)q(ok)o(ed)16 b(up)f(\(see)h(b)q(elo)o (w\).)179 862 y(co)q(ord)284 b(the)42 b(logical)h(pro)q(cess)g(co)q (ordinates)f(of)f(the)h(sp)q(eci\014ed)j(pro)q(cesses)573 918 y(\(see)15 b(b)q(elo)o(w\).)75 1016 y Ft(OUTPUT)k(AR)o(GUMENTS)179 1073 y Fs(plist)309 b(the)15 b(list)h(of)f(PID)g(n)o(um)o(b)q(ers)h(of) f(the)g(sp)q(eci\014ed)i(pro)q(cesses.)75 1171 y Ft(DESCRIPTION)179 1227 y Fs(Usually)c(a)f(pro)q(cess)h(only)g(comm)o(unicates)f(with)h(a) f(relativ)o(ely)h(small)g(n)o(um)o(b)q(er)g(of)f(other)g(pro)q(cesses) 179 1284 y(in)23 b(a)f(large)g(pro)q(cess)g(structure)g(\(a)f (cartesian)i(structure)f(or)f(a)h(graph\).)40 b(Giv)o(en)23 b(the)f(logical)179 1340 y(co)q(ordinates)d(of)f(those)g(other)g(pro)q (cesses,)h(MPI)p 1028 1340 V 16 w(INQPID)h(lo)q(oks)e(up)h(the)g (corresp)q(onding)g(PID)179 1397 y(n)o(um)o(b)q(ers.)h(Those)15 b(PID)g(n)o(um)o(b)q(ers)h(can)f(then)h(b)q(e)f(used)h(for)f(the)g (message{passing.)179 1472 y(The)f(pro)q(cesses)h(of)f(whic)o(h)h(the)f (PID)g(n)o(um)o(b)q(ers)g(are)g(required)i(are)d(sp)q(eci\014ed)k(b)o (y)d(the)g(input)h(argu-)179 1528 y(men)o(ts)g Fh(nproc)f Fs(and)h Fh(coord)p Fs(.)20 b(Tw)o(o)14 b(mec)o(hanisms)i(are)f(a)o(v)m (ailable:)243 1622 y(1.Separately)33 b(sp)q(eci\014ed)i(pro)q(cesses:) 55 b Fh(nproc)31 b Fs(is)j(set)e(to)g(the)g(n)o(um)o(b)q(er)h(of)g(pro) q(cesses)279 1679 y(1)18 b Fo(\024)g Fr(npr)q(oc)p Fs(.)30 b(V)l(ector)18 b Fh(coord)g Fs(con)o(tains)h(the)g(logical)g(co)q (ordinates)g(for)f(eac)o(h)h(pro)q(cess.)30 b(In)279 1735 y(p)q(erio)q(dic)22 b(directions)f(the)f(co)q(ordinates)g(are)f (in)o(terpreted)h(mo)q(dulo)h(the)f(n)o(um)o(b)q(er)g(of)f(pro-)279 1791 y(cesses)f(in)h(the)f(corresp)q(onding)h(direction)g(\(torus)e (top)q(ology\).)28 b(As)18 b(an)g(example,)h(assume)279 1848 y(the)e(pro)q(cesses)g(are)g(mapp)q(ed)h(as)e(a)h(t)o(w)o (o{dimensional)g(torus,)g(and)g(the)g(PID)g(n)o(um)o(b)q(ers)g(of)279 1904 y(three)j(pro)q(cesses)h(are)f(to)f(b)q(e)i(lo)q(ok)o(ed)g(up.)35 b(Then,)21 b Fh(nproc)f Fs(is)h(set)f(to)f(3,)i(and)f(the)g(\014rst)g (6)279 1961 y(en)o(tries)15 b(of)g Fh(coord)g Fs(are)f(set)h(to)g Fr(i)829 1968 y Fa(1)848 1961 y Fr(;)8 b(j)888 1968 y Fa(1)907 1961 y Fr(;)g(i)944 1968 y Fa(2)963 1961 y Fr(;)g(j)1003 1968 y Fa(2)1021 1961 y Fr(;)g(i)1058 1968 y Fa(3)1077 1961 y Fr(;)g(j)1117 1968 y Fa(3)1135 1961 y Fs(.)279 2026 y(Similarly)l(,)14 b(if)e(the)f(mapping)h(w)o(as)f(done)h(with)f (MPI)p 1153 2026 V 17 w(GRAPH,)g(the)g(en)o(tries)h(of)f(v)o(ector)g Fh(coord)279 2082 y Fs(are)k(set)g(to)f(the)h(logical)i(no)q(de)f(n)o (um)o(b)q(ers)f(of)g(the)g(sp)q(eci\014ed)j(graph)c(pro)q(cesses.)243 2155 y(2.No)30 b(individual)j(selection:)52 b(in)31 b(this)g(case)f Fh(nproc)g Fs(is)g(set)h(to)e(a)h(negativ)o(e)h(n)o(um)o(b)q(er)279 2212 y Fr(npr)q(oc)17 b Fo(\024)h(\000)p Fs(1.)29 b(V)l(ector)18 b Fh(plist)g Fs(returns)g(the)g(PID)h(n)o(um)o(b)q(ers)f(of)g(the)g Fi(\014rst)g Fh(|nproc|)f Fs(pro-)279 2268 y(cesses.)30 b(The)19 b(v)o(ector)f Fh(coord)g Fs(is)h(not)f(used)h(in)h(this)f (case.)30 b(Note)18 b(that)g(this)h(feature)g(do)q(es)279 2324 y(not)13 b(scale)i(to)f(v)o(ery)f(high)i(pro)q(cess)g(n)o(um)o(b)q (ers.)k(In)c(small)g(applications,)h(ho)o(w)o(ev)o(er,)d(it)h(can)g(b)q (e)279 2381 y(used)i(to)e(get)h(a)g(global)h(list)g(of)e(all)j(PID)e(n) o(um)o(b)q(ers)g(in)h(the)g(map.)75 2516 y Ft(RETURN)j(V)-6 b(ALUE)179 2573 y Fs(Up)q(on)15 b(successful)h(completion)g Ft(MPI)p 849 2573 16 2 v 18 w(INQPID)e Fs(returns)h(0.)20 b(A)14 b(v)m(alue)i(of)f(-1)f(is)i(returned)f(if)g(an)179 2629 y(error)f(o)q(ccurs.)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .