%!PS-Adobe-2.0 %%Creator: dvips 5.515 Copyright 1986, 1993 Radical Eye Software %%Title: draft.dvi %%CreationDate: Mon Jun 21 13:29:20 1993 %%Pages: 125 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips draft.dvi -o draft.ps %DVIPSSource: TeX output 1993.06.21:1314 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /@rigin{isls{[ 0 -1 1 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{Resolution hsize -72 div mul 0 TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{ /nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{ /sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0] N df-tail}B /E{pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get}B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{ 128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]} if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{ cc 1 add D}B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet TeXDict begin 40258437 52099154 1000 300 300 (/amd/iliamna/projects/dsrg/C/staff/otto/papers/Editor/MPI/draft4/draft.dvi) @start /Fa 3 52 df<03000700FF000700070007000700070007000700070007000700 07000700070007000700070007007FF00C157E9412>49 D<0F8030E040708030C038E038 4038003800700070006000C00180030006000C08080810183FF07FF0FFF00D157E9412> I<0FE030306018701C701C001C00180038006007E000300018000C000E000EE00EE00EC0 0C401830300FE00F157F9412>I E /Fb 1 41 df<00001C00003C0000F80001E00003C0 000780000F00000E00001E00003C00003C00003C00007800007800007800007800007800 007800007800007800007800007800007800007800007800007800007800007800007800 007800007800007800007800007800007800007800007800007800007800007800007800 00780000780000780000780000780000780000780000780000780000F00000F00000F000 01E00001E00003C0000380000700000E00001C0000780000E00000E000007800001C0000 0E000007000003800003C00001E00001E00000F00000F00000F000007800007800007800 007800007800007800007800007800007800007800007800007800007800007800007800 007800007800007800007800007800007800007800007800007800007800007800007800 007800007800007800007800007800007800007800007800007800007800007800003C00 003C00003C00001E00000E00000F000007800003C00001E00000F800003C00001C167C7B 8121>40 D E /Fc 1 106 df<01C003C003C001800000000000000000000000001C0027 0047004700870087000E000E001C001C001C003800388038807080710032001C000A1C7E 9B0E>105 D E /Fd 3 120 df<01E007100C1018083810701070607F80E000E000E000E0 00E000E0086010602030C01F000D127B9113>101 D<3C3C002646004687004707008E07 008E07000E07000E07001C0E001C0E001C0E001C1C00381C40381C403838403838807019 00300E0012127C9117>110 D<1E01832703874703874703838707018707010E07010E07 011C0E021C0E021C0E021C0E04180C04181C04181C081C1C100C263007C3C018127C911C >119 D E /Fe 1 4 df<020002000200C218F2783AE00F800F803AE0F278C21802000200 02000D0E7E8E12>3 D E /Ff 56 122 df<00E001E0038007000E001C001C0038003800 700070007000E000E000E000E000E000E000E000E000E000700070007000380038001C00 1C000E000700038001E000E00B217A9C16>40 DI<387C7E7E3E0E1E1C78F060070B7984 16>44 D<70F8F8F8700505788416>46 D<03E0000FF8001FFC001E3C00380E00780F0070 0700700700E00380E00380E00380E00380E00380E00380E00380E00380F0078070070070 0700780F003C1E001E3C001FFC000FF80003E00011197E9816>48 D<01800380038007800F807F80FF80738003800380038003800380038003800380038003 8003800380038003807FF87FFC7FF80E197C9816>I<07E0001FF8003FFC00781E007807 00300700000700000700000E00003E0007FC0007F00007FC00001E000007000003000003 80000380600380F00380E00700781E003FFC001FF80007E00011197E9816>51 D<70F8F8F870000000000000000070F8F8F8700512789116>58 D<7FFF00FFFF80FFFF80 000000000000000000000000000000FFFF80FFFF807FFF00110B7E9116>61 D<00E00001F00001F00001B00001B00003B80003B80003B800031800071C00071C00071C 00071C00071C000E0E000E0E000FFE000FFE001FFF001C07001C07001C07007F1FC0FF1F E07F1FC013197F9816>65 D<7FF800FFFE007FFF001C0F001C07801C03801C03801C0380 1C07801C07001FFF001FFE001FFE001C1F001C03801C03C01C01C01C01C01C01C01C01C0 1C03C01C07807FFF80FFFF007FFC0012197F9816>I<01F18007FB800FFF801F0F803C07 80380380700380700380F00000E00000E00000E00000E00000E00000E00000E00000F000 007003807003803803803C07001F0F000FFE0007FC0001F00011197E9816>I<7FF800FF FE007FFF001C0F001C07801C03C01C01C01C01C01C01E01C00E01C00E01C00E01C00E01C 00E01C00E01C00E01C00E01C01C01C01C01C03C01C07801C0F807FFF00FFFE007FF80013 19809816>I<7FFFC0FFFFC07FFFC01C01C01C01C01C01C01C01C01C00001C00001C1C00 1C1C001FFC001FFC001FFC001C1C001C1C001C00001C00E01C00E01C00E01C00E01C00E0 7FFFE0FFFFE07FFFE013197F9816>II<03E30007FF000FFF001E 1F003C0F00380700700700700700F00000E00000E00000E00000E00000E03F80E07FC0E0 3F80F00700700700700700380F003C0F001E1F000FFF0007F70003E70012197E9816>I< 7F1FC0FFBFE07F1FC01C07001C07001C07001C07001C07001C07001C07001FFF001FFF00 1FFF001C07001C07001C07001C07001C07001C07001C07001C07001C07007F1FC0FFBFE0 7F1FC013197F9816>II<7F0FE0FF8F F07F0FE01C07801C0F001C0E001C1C001C3C001C78001CF0001CE0001DF0001FF0001FF8 001F38001E1C001C1C001C0E001C0E001C07001C07001C03807F07E0FF8FF07F07E01419 809816>75 DII<7E1FC0FF3FE07F1FC01D 07001D87001D87001D87001DC7001DC7001CC7001CC7001CE7001CE7001CE7001C67001C 67001C77001C77001C37001C37001C37001C17007F1F00FF9F007F0F0013197F9816>I< 1FFC003FFE007FFF00780F00F00780E00380E00380E00380E00380E00380E00380E00380 E00380E00380E00380E00380E00380E00380E00380F00780F00780780F007FFF003FFE00 1FFC0011197E9816>I<7FF800FFFE007FFF001C0F801C03801C03C01C01C01C01C01C01 C01C03C01C03801C0F801FFF001FFE001FF8001C00001C00001C00001C00001C00001C00 001C00007F0000FF80007F000012197F9816>I<7FE000FFF8007FFC001C1E001C0F001C 07001C07001C07001C07001C0F001C1E001FFC001FF8001FFC001C1C001C0E001C0E001C 0E001C0E001C0E201C0E701C0E707F07E0FF87E07F03C014197F9816>82 D<07E3001FFF003FFF00781F00F00700E00700E00700E00000F000007800003F80001FF0 0007FC0000FE00000F00000700000380000380600380E00380E00700F80F00FFFE00FFFC 00C7F00011197E9816>I<7FFFE0FFFFE0FFFFE0E0E0E0E0E0E0E0E0E0E0E0E000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00007FC000FFE0007FC0013197F9816>I<7F07F0FF8FF87F07F01C01C01C01C0 1C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C0 1C01C01C01C00E03800E038007070007FF0003FE0000F8001519809816>III<7F1F807F3F807F1F800E1E000E1C00073C00073800 03B80003F00001F00001E00000E00001E00001F00003F00003B80007B800071C00071C00 0E0E000E0E001C07007F1FC0FF1FE07F1FC013197F9816>II<7F FF80FFFF80FFFF80E00700E00F00E01E00E01C00003C0000780000700000F00001E00001 C00003C0000780000700000F00001E03801C03803C0380780380700380FFFF80FFFF80FF FF8011197E9816>II93 D<1FE0003FF0007FF800783C00300E00000E00000E0003FE001FFE003E 0E00700E00E00E00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<7E0000FE00007E00000E00000E00000E00000E00000E3E000EFF000FFF800F83C00F00 E00E00E00E00700E00700E00700E00700E00700E00700E00E00F01E00F83C00FFF800EFF 00063C001419809816>I<03F80FFC1FFE3C1E780C7000E000E000E000E000E000F00070 0778073E0E1FFC0FF803F010127D9116>I<003F00007F00003F00000700000700000700 00070003C7000FF7001FFF003C1F00780F00700700E00700E00700E00700E00700E00700 E00700700F00700F003C1F001FFFE00FE7F007C7E014197F9816>I<03E00FF81FFC3C1E 780E7007E007FFFFFFFFFFFFE000E000700778073C0F1FFE0FFC03F010127D9116>I<00 1F00007F8000FF8001E78001C30001C00001C0007FFF00FFFF00FFFF0001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C0003FFE007FFF003F FE0011197F9816>I<03E3C007F7E00FFFE01C1CC0380E00380E00380E00380E00380E00 1C1C000FF8001FF0001BE0003800001800001FFC001FFF003FFF807803C0E000E0E000E0 E000E0E000E07001C07C07C03FFF800FFE0003F800131C7F9116>I<7E0000FE00007E00 000E00000E00000E00000E00000E3C000EFE000FFF000F87800F03800E03800E03800E03 800E03800E03800E03800E03800E03800E03800E03807FC7F0FFE7F87FC7F01519809816 >I<018003C003C0018000000000000000007FC07FC07FC001C001C001C001C001C001C0 01C001C001C001C001C001C07FFFFFFF7FFF101A7D9916>I<7E0000FE00007E00000E00 000E00000E00000E00000E7FE00E7FE00E7FE00E0F000E1E000E3C000E78000EF0000FF0 000FF8000FBC000F1E000E0E000E07000E07807F87F0FFCFF07F87F01419809816>107 DII< 7E3C00FEFE007FFF000F87800F03800E03800E03800E03800E03800E03800E03800E0380 0E03800E03800E03807FC7F0FFE7F87FC7F01512809116>I<03E0000FF8001FFC003C1E 00780F00700700E00380E00380E00380E00380E00380F00780700700780F003C1E001FFC 000FF80003E00011127E9116>I<7E3E00FEFF007FFF800F83C00F00E00E00E00E00700E 00700E00700E00700E00700E00700E00E00F01E00F83C00FFF800EFF000E3C000E00000E 00000E00000E00000E00000E00007FC000FFE0007FC000141B809116>I114 D<0FEC3FFC7FFCF03CE01CE01C70007F 801FF007F8003C600EE00EF00EF81EFFFCFFF8C7E00F127D9116>I<0300000700000700 000700000700007FFF00FFFF00FFFF000700000700000700000700000700000700000700 0007010007038007038007038007870003FE0001FC0000F80011177F9616>I<7E1F80FE 3F807E1F800E03800E03800E03800E03800E03800E03800E03800E03800E03800E03800E 03800E0F800FFFF007FBF803E3F01512809116>I<7F1FC0FF1FE07F1FC01C07001E0F00 0E0E000E0E000E0E00071C00071C00071C00071C0003B80003B80003B80001F00001F000 00E00013127F9116>I<7F1FC0FF9FE07F1FC01C07000E07000E0E000E0E00070E00071C 00071C00039C00039C0003980001B80001B80000F00000F00000F00000E00000E00000E0 0001C00079C0007BC0007F80003F00003C0000131B7F9116>121 D E /Fg 7 86 df<01FE100703F01C00F0380070780030700030F00000F00000F00000F0 0000F000007000307800303800601C006007018001FE0014117E9019>67 DI73 D<01FC000F07801C01C03800E07800F0700070F00078F00078F00078F00078F000787000 707800F03800E01C01C00F078003FE0015117E901A>79 D<1FC820784038C018C018F000 FF007FE03FF00FF8007C001CC00CC00CE008F0109FE00E117E9013>83 D<7FFFF0707870607830C07818C078180078000078000078000078000078000078000078 0000780000780000780000780007FF8015117F9018>II E /Fh 50 122 df<00003FE00000E0100001803800038078000300 7800070030000700000007000000070000000E0000000E0000000E000000FFFFE0000E00 E0001C01C0001C01C0001C01C0001C01C0001C0380003803800038038000380380003807 0000380700007007000070071000700E2000700E2000700E2000E00E2000E0064000E003 8000E0000000C0000001C0000001C000003180000079800000F3000000620000003C0000 001D29829F1A>12 D<000100020004000800100020006000C0018001800300070006000E 000C001C0018003800380030007000700060006000E000E000C000C000C000C000C000C0 00C000C000C000C000C000C000C0004000600060002000100010000800102E79A113>40 D<0010000008000004000006000002000003000003000003000001000001800001800001 800001800001800001800001800003800003800003800003000003000003000007000007 00000600000600000E00000C00000C00001C0000180000380000300000700000600000E0 0000C0000180000100000300000600000C0000180000300000600000800000112E80A113 >I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I< 70F8F8F0E005057B840E>I<000F800030E000E07001C0700380300380380700380F0078 0F00780E00781E00781E00703C00F03C00F03C00F03C00F07801E07801E07801E07801C0 7003C0F003C0F00380F00780F00700700700700E00701C003038001870000FC000151F7C 9D17>48 D<000200020006000E003C00DC031C001C003800380038003800700070007000 7000E000E000E000E001C001C001C001C003800380038003800780FFF80F1E7B9D17>I< 001F000061800080E00100E00200700220700420700410700820F00820F00820F00840E0 0881E00703C0000380000700000C00001800006000008000030000040000080040100040 1000802001807E030047FF0041FE0080FC00807800141F7C9D17>I<001F0000718000C0 C00180C00380E00700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C01C03C01C03 C01C07C01C0F800C0F8006378003C700000F00000E00000E00001C00601C00F03800F070 00E0600080C0004380003E0000131F7B9D17>57 D<070F1F1F0E00000000000000000000 70F8F8F0E008147B930E>I<00000200000006000000060000000E0000001E0000001E00 00003F0000002F0000004F0000004F0000008F0000010F0000010F0000020F0000020F00 00040F00000C0F0000080F0000100F0000100F0000200F80003FFF800040078000C00780 0080078001000780010007800200078002000780060007801E000F80FF807FF81D207E9F 22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C003C003C003C 003C003C0078007800780078007800F0007801E000F0078000FFFE0000F00F8000F003C0 01E001C001E001E001E001E001E001E003C001E003C001E003C001E003C001C0078003C0 0780078007800F0007801E000F007800FFFFE0001E1F7D9E20>I<0000FE020007818600 1C004C0038003C0060003C00C0001C01C0001803800018070000180F0000181E0000101E 0000103C0000003C00000078000000780000007800000078000000F0000000F0000000F0 000000F0000000F00000807000008070000080700001003800010038000200180004000C 001800060020000381C00000FE00001F217A9F21>I<01FFFF80001E00E0001E0070001E 0038001E001C003C001C003C000E003C000E003C000E0078000E0078000E0078000E0078 000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01E0007803C0 007003C0007003C000E003C001C0078001C00780038007800E0007801C000F007000FFFF C0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C0004003C00 04003C0004003C00040078080800780800007808000078180000F0300000FFF00000F030 0000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C0004003C000 40078000800780018007800100078007000F001F00FFFFFE001F1F7D9E1F>I<01FFFFFC 001E0038001E0018001E0008001E0008003C0008003C0008003C0008003C000800780010 00780800007808000078080000F0100000F0300000FFF00000F0300001E0200001E02000 01E0200001E0200003C0000003C0000003C0000003C00000078000000780000007800000 078000000F800000FFF800001E1F7D9E1E>I<0000FC040007030C001C00980030007800 E0007801C000380380003003800030070000300E0000301E0000201E0000203C0000003C 00000078000000780000007800000078000000F0000000F000FFF0F0000780F0000780F0 000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E000E00CC0003 83040000FC00001E217A9F23>I<01FFF3FFE0001F003E00001E003C00001E003C00001E 003C00003C007800003C007800003C007800003C007800007800F000007800F000007800 F000007800F00000F001E00000FFFFE00000F001E00000F001E00001E003C00001E003C0 0001E003C00001E003C00003C007800003C007800003C007800003C007800007800F0000 07800F000007800F000007800F00000F801F0000FFF1FFE000231F7D9E22>I<01FFF000 1F00001E00001E00001E00003C00003C00003C00003C0000780000780000780000780000 F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C00003C00007 80000780000780000780000F8000FFF800141F7D9E12>I<01FE00007FC0001E0000FC00 001E0000F80000170001780000170001780000270002F00000270004F00000270004F000 00270008F00000470009E00000470011E00000470021E00000470021E00000870043C000 00838043C00000838083C00000838083C000010381078000010382078000010382078000 0103840780000203840F00000203880F00000203900F00000203900F00000401E01E0000 0401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0002A1F7D9E29> 77 D<01FFFF80001E00E0001E0070001E0038001E003C003C003C003C003C003C003C00 3C003C0078007800780078007800F0007800E000F003C000F00F0000FFFC0000F0000001 E0000001E0000001E0000001E0000003C0000003C0000003C0000003C000000780000007 80000007800000078000000F800000FFF000001E1F7D9E1F>80 D<0007E040001C18C000 3005800060038000C0038001C00180018001000380010003800100038001000380000003 C0000003C0000003F8000001FF800001FFE000007FF000001FF0000001F8000000780000 007800000038000000380020003800200038002000300060007000600060006000E00070 00C000E8038000C606000081F800001A217D9F1A>83 D<0FFFFFF01E0780E01807802010 07802020078020200F0020600F0020400F0020400F0020801E0040001E0000001E000000 1E0000003C0000003C0000003C0000003C00000078000000780000007800000078000000 F0000000F0000000F0000000F0000001E0000001E0000001E0000001E0000003E00000FF FF00001C1F789E21>I<7FFC1FF807C003C00780010007800100078001000F0002000F00 02000F0002000F0002001E0004001E0004001E0004001E0004003C0008003C0008003C00 08003C00080078001000780010007800100078001000F0002000F0002000F0002000F000 4000F0004000700080007001000030020000380400000C18000007E000001D20779E22> II<00F1800389C00707 800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E20F01C 40F01C40703C40705C40308C800F070013147C9317>97 D<07803F800700070007000700 0E000E000E000E001C001C001CF01D0C3A0E3C0E380F380F700F700F700F700FE01EE01E E01EE01CE03CE038607060E031C01F0010207B9F15>I<007E0001C1000300800E07801E 07801C07003C0200780000780000780000F00000F00000F00000F00000F0000070010070 020030040018380007C00011147C9315>I<0000780003F8000070000070000070000070 0000E00000E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C0380 380700780700780700780700F00E00F00E00F00E00F00E20F01C40F01C40703C40705C40 308C800F070015207C9F17>I<007C01C207010E011C013C013802780C7BF07C00F000F0 00F000F0007000700170023804183807C010147C9315>I<00007800019C00033C00033C 000718000700000700000E00000E00000E00000E00000E0001FFE0001C00001C00001C00 001C00003800003800003800003800003800007000007000007000007000007000007000 00E00000E00000E00000E00000C00001C00001C0000180003180007B0000F30000660000 3C00001629829F0E>I<003C6000E27001C1E00380E00700E00F00E00E01C01E01C01E01 C01E01C03C03803C03803C03803C03803C07003C07001C0F001C17000C2E0003CE00000E 00000E00001C00001C00301C00783800F0700060E0003F8000141D7E9315>I<01E0000F E00001C00001C00001C00001C000038000038000038000038000070000070000071E0007 63000E81800F01C00E01C00E01C01C03801C03801C03801C038038070038070038070038 0E10700E20700C20701C20700C40E00CC060070014207D9F17>I<00C001E001E001C000 000000000000000000000000000E003300230043804300470087000E000E000E001C001C 001C003840388030807080310033001C000B1F7C9E0E>I<0001800003C00003C0000380 000000000000000000000000000000000000000000003C00004600008700008700010700 010700020E00000E00000E00000E00001C00001C00001C00001C00003800003800003800 00380000700000700000700000700000E00000E00030E00079C000F180006300003C0000 1228829E0E>I<01E0000FE00001C00001C00001C00001C0000380000380000380000380 000700000700000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC0 0038E000387000387000383840707080707080707080703100E03100601E0013207D9F15 >I<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C00 1C0038003800380038007000700070007100E200E200E200E200640038000A207C9F0C> I<1C0F80F0002630C318004740640C004780680E004700700E004700700E008E00E01C00 0E00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C0708038 038071003803806100380380E10038038062007007006600300300380021147C9325>I< 1C0F802630C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C0 1C01C01C03843803883803083807083803107003303001C016147C931A>I<007C0001C3 000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F007 80F00700700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C04 781C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C03803807803 80700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C 0000FFC000171D809317>I<1C1E002661004783804787804707804703008E00000E0000 0E00000E00001C00001C00001C00001C0000380000380000380000380000700000300000 11147C9313>114 D<00FC030206010C030C070C060C000F800FF007F803FC003E000E70 0EF00CF00CE008401020601F8010147D9313>I<018001C0038003800380038007000700 FFF007000E000E000E000E001C001C001C001C0038003800380038207040704070807080 31001E000C1C7C9B0F>I<0E00C03300E02301C04381C04301C04701C08703800E03800E 03800E03801C07001C07001C07001C07101C0E20180E20180E201C1E200C264007C38014 147C9318>I<0E03803307802307C04383C04301C04700C08700800E00800E00800E0080 1C01001C01001C01001C02001C02001C04001C04001C08000E300003C00012147C9315> I<0E00C1C03300E3C02301C3E04381C1E04301C0E04701C060870380400E0380400E0380 400E0380401C0700801C0700801C0700801C0701001C0701001C0602001C0F02000C0F04 000E13080003E1F0001B147C931E>I<0383800CC4401068E01071E02071E02070C040E0 0000E00000E00000E00001C00001C00001C00001C040638080F38080F38100E5810084C6 0078780013147D9315>I<0E00C03300E02301C04381C04301C04701C08703800E03800E 03800E03801C07001C07001C07001C07001C0E00180E00180E001C1E000C3C0007DC0000 1C00001C00003800F03800F07000E06000C0C0004380003E0000131D7C9316>I E /Fi 60 124 df<000FF83F00007FFDFFC001F81FE3E003E03F87E007C03F87E00F803F 07E00F803F03C00F801F00000F801F00000F801F00000F801F00000F801F00000F801F00 00FFFFFFFC00FFFFFFFC000F801F00000F801F00000F801F00000F801F00000F801F0000 0F801F00000F801F00000F801F00000F801F00000F801F00000F801F00000F801F00000F 801F00000F801F00000F801F00000F801F00000F801F00000F801F00007FF0FFF0007FF0 FFF00023237FA221>11 D<000FF000007FFC0001F80E0003E01F0007C03F000F803F000F 803F000F801E000F800C000F8000000F8000000F8000000F800000FFFFFF00FFFFFF000F 801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F000F 801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F000F801F007F F0FFE07FF0FFE01B237FA21F>I<387CFEFFFF7F3B03030706060C1C18702008117CA210> 39 D45 D<387CFEFEFE7C3807077C8610>I<00FE0007 FFC00F83E01F01F03E00F83E00F87C007C7C007C7C007CFC007CFC007EFC007EFC007EFC 007EFC007EFC007EFC007EFC007EFC007EFC007EFC007EFC007EFC007E7C007C7C007C7C 007C3E00F83E00F81F01F00F83E007FFC000FE0017207E9F1C>48 D<00180000780001F800FFF800FFF80001F80001F80001F80001F80001F80001F80001F8 0001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F80001F8 0001F80001F80001F80001F80001F80001F8007FFFE07FFFE013207C9F1C>I<03FC000F FF003C1FC07007E07C07F0FE03F0FE03F8FE03F8FE01F87C01F83803F80003F80003F000 03F00007E00007C0000F80001F00003E0000380000700000E01801C0180380180700180E 00380FFFF01FFFF03FFFF07FFFF0FFFFF0FFFFF015207D9F1C>I<00FE0007FFC00F07E0 1E03F03F03F03F81F83F81F83F81F81F03F81F03F00003F00003E00007C0001F8001FE00 01FF000007C00001F00001F80000FC0000FC3C00FE7E00FEFF00FEFF00FEFF00FEFF00FC 7E01FC7801F81E07F00FFFC001FE0017207E9F1C>I<0000E00001E00003E00003E00007 E0000FE0001FE0001FE00037E00077E000E7E001C7E00187E00307E00707E00E07E00C07 E01807E03807E07007E0E007E0FFFFFEFFFFFE0007E00007E00007E00007E00007E00007 E00007E000FFFE00FFFE17207E9F1C>I<1000201E01E01FFFC01FFF801FFF001FFE001F F8001BC00018000018000018000018000019FC001FFF001E0FC01807E01803E00003F000 03F00003F80003F83803F87C03F8FE03F8FE03F8FC03F0FC03F07007E03007C01C1F800F FF0003F80015207D9F1C>I<001F8000FFE003F07007C0F00F01F81F01F83E01F83E01F8 7E00F07C00007C0000FC0800FC7FC0FCFFE0FD80F0FF00F8FE007CFE007CFC007EFC007E FC007EFC007E7C007E7C007E7C007E3C007C3E007C1E00F80F00F00783E003FFC000FF00 17207E9F1C>I<6000007800007FFFFE7FFFFE7FFFFC7FFFF87FFFF87FFFF0E00060E000 C0C00180C00300C00300000600000C00001C0000180000380000780000780000F00000F0 0000F00001F00001F00001F00003F00003F00003F00003F00003F00003F00003F00001E0 0017227DA11C>I<00FE0003FFC00703E00E00F01C00F01C00783C00783E00783F00783F 80783FE0F01FF9E01FFFC00FFF8007FFC003FFE007FFF01E7FF83C1FFC7807FC7801FEF0 00FEF0003EF0001EF0001EF0001CF8001C7800383C00381F01F00FFFC001FF0017207E9F 1C>I<01FE0007FF800F83E01E01F03E00F07C00F87C0078FC007CFC007CFC007CFC007E FC007EFC007EFC007E7C00FE7C00FE3E01FE1E037E0FFE7E07FC7E00207E00007C00007C 1E007C3F00F83F00F83F00F03F01E01E03C01C0F800FFE0003F80017207E9F1C>I<387C FEFEFE7C380000000000000000387CFEFEFE7C3807167C9510>I<000070000000007000 000000F800000000F800000000F800000001FC00000001FC00000003FE00000003FE0000 0003FE00000006FF000000067F0000000E7F8000000C3F8000000C3F800000183FC00000 181FC00000381FE00000300FE00000300FE00000600FF000006007F00000E007F80000FF FFF80000FFFFF800018001FC00018001FC00038001FE00030000FE00030000FE00060000 7F000600007F00FFE00FFFF8FFE00FFFF825227EA12A>65 DI<0003FE0080001FFF818000FF01E3 8001F8003F8003E0001F8007C0000F800F800007801F800007803F000003803F00000380 7F000001807E000001807E00000180FE00000000FE00000000FE00000000FE00000000FE 00000000FE00000000FE00000000FE000000007E000000007E000001807F000001803F00 0001803F000003801F800003000F8000030007C000060003F0000C0001F800380000FF00 F000001FFFC0000003FE000021227DA128>IIII<0003FE0040001FFFC0C0007F00F1C001F8003FC003F0 000FC007C00007C00FC00003C01F800003C03F000001C03F000001C07F000000C07E0000 00C07E000000C0FE00000000FE00000000FE00000000FE00000000FE00000000FE000000 00FE00000000FE000FFFFC7E000FFFFC7F00001FC07F00001FC03F00001FC03F00001FC0 1F80001FC00FC0001FC007E0001FC003F0001FC001FC003FC0007F80E7C0001FFFC3C000 03FF00C026227DA12C>III76 DI I<0007FC0000003FFF800000FC07E00003F001F80007E000FC000FC0007E001F80003F00 1F80003F003F00001F803F00001F807F00001FC07E00000FC07E00000FC0FE00000FE0FE 00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00000FE0FE00 000FE07E00000FC07F00001FC07F00001FC03F00001F803F80003F801F80003F000FC000 7E0007E000FC0003F001F80000FC07E000003FFF80000007FC000023227DA12A>II82 D<01FC0407FF8C1F03FC3C007C7C003C78001C78001CF8000CF8000CFC000CFC0000FF00 00FFE0007FFF007FFFC03FFFF01FFFF80FFFFC03FFFE003FFE0003FF00007F00003F0000 3FC0001FC0001FC0001FE0001EE0001EF0003CFC003CFF00F8C7FFE080FF8018227DA11F >I<7FFFFFFF807FFFFFFF807E03F80F807803F807807003F803806003F80180E003F801 C0E003F801C0C003F800C0C003F800C0C003F800C0C003F800C00003F800000003F80000 0003F800000003F800000003F800000003F800000003F800000003F800000003F8000000 03F800000003F800000003F800000003F800000003F800000003F800000003F800000003 F800000003F800000003F800000003F8000003FFFFF80003FFFFF80022227EA127>I87 D<07FC001FFF803F07C03F03E03F01E03F01F01E01F00001F00001F0003FF003FDF01FC1 F03F01F07E01F0FC01F0FC01F0FC01F0FC01F07E02F07E0CF81FF87F07E03F18167E951B >97 DI<00FF8007FFE00F83F01F03F03E03F07E03F07C01E07C0000FC0000FC0000 FC0000FC0000FC0000FC00007C00007E00007E00003E00301F00600FC0E007FF8000FE00 14167E9519>I<0001FE000001FE0000003E0000003E0000003E0000003E0000003E0000 003E0000003E0000003E0000003E0000003E0000003E0001FC3E0007FFBE000F81FE001F 007E003E003E007E003E007C003E00FC003E00FC003E00FC003E00FC003E00FC003E00FC 003E00FC003E00FC003E007C003E007C003E003E007E001E00FE000F83BE0007FF3FC001 FC3FC01A237EA21F>I<00FE0007FF800F87C01E01E03E01F07C00F07C00F8FC00F8FC00 F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C00007E00003E00181F00300FC07003FF C000FF0015167E951A>I<003F8000FFC001E3E003C7E007C7E00F87E00F83C00F80000F 80000F80000F80000F80000F8000FFFC00FFFC000F80000F80000F80000F80000F80000F 80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F 80007FF8007FF80013237FA211>I<03FC1E0FFF7F1F0F8F3E07CF3C03C07C03E07C03E0 7C03E07C03E07C03E03C03C03E07C01F0F801FFF0013FC003000003000003800003FFF80 1FFFF00FFFF81FFFFC3800FC70003EF0001EF0001EF0001EF0001E78003C7C007C3F01F8 0FFFE001FF0018217E951C>II<1C003F007F007F007F003F001C00000000000000 0000000000000000FF00FF001F001F001F001F001F001F001F001F001F001F001F001F00 1F001F001F001F001F001F00FFE0FFE00B247EA310>I<0038007C00FE00FE00FE007C00 38000000000000000000000000000003FE03FE003E003E003E003E003E003E003E003E00 3E003E003E003E003E003E003E003E003E003E003E003E003E003E003E783EFC3EFC3CFC 7C78F87FE01F800F2E83A311>I108 DII<00FE0007FFC00F83E01E00F03E00F87C007C7C007C7C007CFC007EFC007EFC007E FC007EFC007EFC007EFC007E7C007C7C007C3E00F81F01F00F83E007FFC000FE0017167E 951C>II<00FE030007 FF87000FC1C7001F006F003F003F007E003F007E001F007C001F00FC001F00FC001F00FC 001F00FC001F00FC001F00FC001F00FC001F007E001F007E001F003E003F001F007F000F C1DF0007FF9F0001FC1F0000001F0000001F0000001F0000001F0000001F0000001F0000 001F0000001F000000FFE00000FFE01B207E951E>II<0FF3003FFF00781F00600700E0 0300E00300F00300FC00007FE0007FF8003FFE000FFF0001FF00000F80C00780C00380E0 0380E00380F00700FC0E00EFFC00C7F00011167E9516>I<018000018000018000018000 0380000380000780000780000F80003F8000FFFF00FFFF000F80000F80000F80000F8000 0F80000F80000F80000F80000F80000F80000F80000F81800F81800F81800F81800F8180 0F830007C30003FE0000F80011207F9F16>IIIIII<7FFFF07FFFF07C03E07007C0600FC0E01F80C01F 00C03E00C07E0000FC0000F80001F00003F03007E03007C0300F80701F80703F00603E00 E07C03E0FFFFE0FFFFE014167E9519>II E /Fj 72 124 df<007E1F0001C1B1800303E3C00703C3C00E03C1800E01C0000E01C000 0E01C0000E01C0000E01C0000E01C000FFFFFC000E01C0000E01C0000E01C0000E01C000 0E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C000 0E01C0000E01C0000E01C0007F87FC001A1D809C18>11 D<007E0001C1800301800703C0 0E03C00E01800E00000E00000E00000E00000E0000FFFFC00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 7F87F8151D809C17>I<007FC001C1C00303C00703C00E01C00E01C00E01C00E01C00E01 C00E01C00E01C0FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C07FCFF8151D809C17>I<003F07E0 0001C09C18000380F018000701F03C000E01E03C000E00E018000E00E000000E00E00000 0E00E000000E00E000000E00E00000FFFFFFFC000E00E01C000E00E01C000E00E01C000E 00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00 E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C007FC7FCFF80211D80 9C23>I<6060F0F0F8F86868080808080808101010102020404080800D0C7F9C15>34 D<60F0F8680808081010204080050C7C9C0C>39 D<004000800100020006000C000C0018 001800300030007000600060006000E000E000E000E000E000E000E000E000E000E000E0 00E000600060006000700030003000180018000C000C00060002000100008000400A2A7D 9E10>I<800040002000100018000C000C000600060003000300038001800180018001C0 01C001C001C001C001C001C001C001C001C001C001C00180018001800380030003000600 06000C000C00180010002000400080000A2A7E9E10>I<60F0F070101010102020408004 0C7C830C>44 DI<60F0F06004047C830C>I<0001000300060006 0006000C000C000C0018001800180030003000300060006000C000C000C0018001800180 030003000300060006000C000C000C00180018001800300030003000600060006000C000 C00010297E9E15>I<03C00C301818300C300C700E60066006E007E007E007E007E007E0 07E007E007E007E007E007E007E00760066006700E300C300C18180C3007E0101D7E9B15 >I<07C01830201C400C400EF00FF80FF807F8077007000F000E000E001C001C00380070 006000C00180030006010C01180110023FFE7FFEFFFE101C7E9B15>50 D<300C3FF83FF03FC020002000200020002000200023E024302818301C200E000E000F00 0F000F600FF00FF00FF00F800E401E401C2038187007C0101D7E9B15>53 D<00F0030C06040C0E181E301E300C700070006000E3E0E430E818F00CF00EE006E007E0 07E007E007E007600760077006300E300C18180C3003E0101D7E9B15>I<4000007FFF80 7FFF007FFF0040020080040080040080080000100000100000200000600000400000C000 00C00001C000018000018000038000038000038000038000078000078000078000078000 078000078000030000111D7E9B15>I<03C00C301818300C700C600EE006E006E007E007 E007E007E0076007700F300F18170C2707C700060006000E300C780C78187010203030C0 0F80101D7E9B15>57 D<60F0F0600000000000000000000060F0F06004127C910C>I<60 F0F0600000000000000000000060F0F0701010101020204080041A7C910C>I<0FE03038 401CE00EF00EF00EF00E000C001C0030006000C000800180010001000100010001000100 0000000000000000000003000780078003000F1D7E9C14>63 D<00060000000600000006 0000000F0000000F0000000F00000017800000178000001780000023C0000023C0000023 C0000041E0000041E0000041E0000080F0000080F0000180F8000100780001FFF8000300 7C0002003C0002003C0006003E0004001E0004001E000C001F001E001F00FF80FFF01C1D 7F9C1F>65 DI<001F808000E06180018019 80070007800E0003801C0003801C00018038000180780000807800008070000080F00000 00F0000000F0000000F0000000F0000000F0000000F0000000F000000070000080780000 8078000080380000801C0001001C0001000E000200070004000180080000E03000001FC0 00191E7E9C1E>IIII<001F808000E0618001801980070007 800E0003801C0003801C00018038000180780000807800008070000080F0000000F00000 00F0000000F0000000F0000000F0000000F000FFF0F0000F807000078078000780780007 80380007801C0007801C0007800E00078007000B800180118000E06080001F80001C1E7E 9C21>III<1FFF00F80078007800780078007800780078007800780078007800780078 0078007800780078007800787078F878F878F878F0F040E021C01F00101D7F9B15>IIIII<003F800000E0E0000380380007001C000E000E001C0007003C00078038 000380780003C0780003C0700001C0F00001E0F00001E0F00001E0F00001E0F00001E0F0 0001E0F00001E0F00001E0700001C0780003C0780003C0380003803C0007801C0007000E 000E0007001C000380380000E0E000003F80001B1E7E9C20>II<003F800000E0E0000380380007001C000E000E001C0007003C0007 8038000380780003C0780003C0700001C0F00001E0F00001E0F00001E0F00001E0F00001 E0F00001E0F00001E0F00001E0700001C0780003C0780003C0380003803C0E07801C1107 000E208E0007205C0003A0780000F0E020003FE0200000602000003060000038E000003F C000003FC000001F8000000F001B257E9C20>II<07 E0801C1980300580700380600180E00180E00080E00080E00080F00000F800007C00007F C0003FF8001FFE0007FF0000FF80000F800007C00003C00001C08001C08001C08001C0C0 0180C00180E00300D00200CC0C0083F800121E7E9C17>I<7FFFFFC0700F01C0600F00C0 400F0040400F0040C00F0020800F0020800F0020800F0020000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000001F800003FFFC001B1C7F9B1E>II87 D89 D<08081010202040404040808080808080B0B0F8F8787830300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC000F1C00381C00701C00601C 00E01C40E01C40E01C40603C40304E801F870012127E9115>97 DI<07E00C301878307870306000E000E000E000E000E000E0 0060007004300418080C3007C00E127E9112>I<003F0000070000070000070000070000 070000070000070000070000070000070003E7000C1700180F00300700700700600700E0 0700E00700E00700E00700E00700E00700600700700700300700180F000C370007C7E013 1D7E9C17>I<03E00C301818300C700E6006E006FFFEE000E000E000E000600070023002 18040C1803E00F127F9112>I<00F8018C071E061E0E0C0E000E000E000E000E000E00FF E00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E007FE00F 1D809C0D>I<00038003C4C00C38C01C3880181800381C00381C00381C00381C00181800 1C38000C300013C0001000003000001800001FF8001FFF001FFF803003806001C0C000C0 C000C0C000C06001803003001C0E0007F800121C7F9215>II<18003C003C0018000000000000000000000000000000FC001C 001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00FF80091D7F 9C0C>I<00C001E001E000C000000000000000000000000000000FE000E000E000E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E060E0F0C0 F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180E001C0E001C0E001C0E001C0 E001C0E001C06001807003803003001806000E1C0003F00012127F9115>II<03C1000C3300180B00300F00700700700700E00700E00700E00700E0 0700E00700E00700600700700700300F00180F000C370007C70000070000070000070000 0700000700000700000700003FE0131A7E9116>II<1F9030704030C010 C010E010F8007F803FE00FF000F880388018C018C018E010D0608FC00D127F9110>I<04 000400040004000C000C001C003C00FFE01C001C001C001C001C001C001C001C001C001C 101C101C101C101C100C100E2003C00C1A7F9910>III< FF3FCFE03C0F03801C0701801C0701001C0B01000E0B82000E0B82000E1182000711C400 0711C4000720C40003A0E80003A0E80003C0680001C0700001C070000180300000802000 1B127F911E>I<7F8FF00F03800F030007020003840001C80001D80000F0000070000078 0000F800009C00010E00020E000607000403801E07C0FF0FF81512809116>II<7FFC70386038407040F040E041C003C0038007000F040E041C043C 0C380870087038FFF80E127F9112>II E /Fk 17 118 df<78FCFCFCFC7800000000000078FCFCFCFC7806127D910D>58 D68 D73 D77 D<0FF8001C1E003E0F803E07803E07C01C07C00007C0007FC007E7C0 1F07C03C07C07C07C0F807C0F807C0F807C0780BC03E13F80FE1F815127F9117>97 D<03FC000E0E001C1F003C1F00781F00780E00F80000F80000F80000F80000F80000F800 007800007801803C01801C03000E0E0003F80011127E9115>99 D<01FC000F07001C0380 3C01C07801C07801E0F801E0F801E0FFFFE0F80000F80000F800007800007C00603C0060 1E00C00F038001FC0013127F9116>101 D<03F8F00E0F381E0F381C07303C07803C0780 3C07803C07801C07001E0F000E0E001BF8001000001800001800001FFF001FFFC00FFFE0 1FFFF07801F8F00078F00078F000787000707800F01E03C007FF00151B7F9118>103 D<1E003F003F003F003F001E00000000000000000000000000FF00FF001F001F001F001F 001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B1E7F9D0E>105 D108 DII<01FC000F07801C01C03C01E07800F07800F0F800F8 F800F8F800F8F800F8F800F8F800F87800F07800F03C01E01E03C00F078001FC0015127F 9118>II<1FD830786018E018E018F000FF807FE07FF01F F807FC007CC01CC01CE01CE018F830CFC00E127E9113>115 D<03000300030003000700 07000F000F003FFCFFFC1F001F001F001F001F001F001F001F001F001F0C1F0C1F0C1F0C 0F08079803F00E1A7F9913>II E /Fl 88 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8F870000000000070F8F8 F870051C779B18>33 D<030600078F00078F00078F00078F00078F00078F007FFFC0FFFF E0FFFFE07FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFFC0FFFFE0FFFFE07FFF C01E3C001E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18>35 D<00C00001C00001C00001C00003F0000FFC003FFE007DCF0071C700E1C380E1C780E1C7 80E1C780F1C00079C0003DC0001FE0000FF80003FC0001DE0001CF0001C70061C380F1C3 80F1C380E1C380E1C70071C70079DE003FFE001FF80007E00001C00001C00001C00000C0 0011247D9F18>I<3803007C07807C0780EE0F80EE0F00EE0F00EE1F00EE1E00EE1E00EE 3E007C3C007C3C00387C0000780000780000F80000F00001F00001E00001E00003E00003 C00003C00007C0000783800787C00F87C00F0EE00F0EE01F0EE01E0EE01E0EE03E0EE03C 07C03C07C018038013247E9F18>I<01C00007E0000FF0000E70001C38001C38001C3800 1C38001C73F01C73F01CE3F00FE3800FC7000F87000F07001F0E003F0E007B8E0073DC00 E1DC00E0F800E0F800E07070E0787070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C 7C7E3E0E0E0E1C1C38F8F0C0070E789B18>I<007000F001E003C007800F001E001C0038 0038007000700070007000E000E000E000E000E000E000E000E000700070007000700038 0038001C001E000F00078003C001F000F000700C24799F18>I<6000F00078003C001E00 0F000780038001C001C000E000E000E000E00070007000700070007000700070007000E0 00E000E000E001C001C0038007800F001E003C007800F00060000C247C9F18>I<01C000 01C00001C00001C000C1C180F1C780F9CF807FFF001FFC0007F00007F0001FFC007FFF00 F9CF80F1C780C1C18001C00001C00001C00001C00011147D9718>I<00600000F00000F0 0000F00000F00000F00000F00000F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F0 0000F00000F00000F00000F00000600013147E9718>I<1C3E7E7F3F1F070E1E7CF86008 0C788518>I<7FFF00FFFF80FFFF807FFF0011047D8F18>I<3078FCFC78300606778518> I<000300000780000780000F80000F00001F00001E00001E00003E00003C00007C000078 0000780000F80000F00001F00001E00003E00003C00003C00007C0000780000F80000F00 000F00001F00001E00003E00003C00003C00007C0000780000F80000F00000F000006000 0011247D9F18>I<01F00007FC000FFE001F1F001C07003803807803C07001C07001C0E0 00E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0F001E07001C07001C078 03C03803801C07001F1F000FFE0007FC0001F000131C7E9B18>I<01800380038007800F 803F80FF80FB804380038003800380038003800380038003800380038003800380038003 80038003807FFCFFFE7FFC0F1C7B9B18>I<03F0000FFE003FFF007C0F807003C0E001C0 F000E0F000E06000E00000E00000E00001C00001C00003C0000780000F00001E00003C00 00780000F00001E00007C0000F80001E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18> I<07F8001FFE003FFF007807807803C07801C03001C00001C00003C0000380000F0003FF 0003FE0003FF000007800003C00001C00000E00000E00000E0F000E0F000E0F001C0F003 C07C07803FFF001FFE0003F800131C7E9B18>I<001F00003F0000770000770000E70001 E70001C7000387000787000707000E07001E07003C0700380700780700F00700FFFFF8FF FFF8FFFFF8000700000700000700000700000700000700007FF000FFF8007FF0151C7F9B 18>I<1FFF803FFF803FFF80380000380000380000380000380000380000380000380000 3BF8003FFE003FFF003C07801803C00001C00000E00000E06000E0F000E0F000E0E001C0 7003C07C0F803FFF001FFC0003F000131C7E9B18>I<007E0001FF0007FF800F83C01E03 C01C03C0380180380000700000700000E1F800E7FE00FFFF00FE0780F803C0F001C0F000 E0E000E0F000E07000E07000E07000E03801C03C03C01E07800FFF0007FE0001F800131C 7E9B18>II<03F8000FFE001FFF00 3E0F803803807001C07001C07001C07001C03803803C07801FFF0007FC000FFE001F1F00 3C07807001C0F001E0E000E0E000E0E000E0E000E07001C07803C03E0F801FFF000FFE00 03F800131C7E9B18>I<03F0000FFC001FFE003C0F00780780700380E001C0E001C0E001 C0E001E0E001E07001E07803E03C0FE01FFFE00FFEE003F0E00000E00001C00001C00001 C0300380780780780F00783E003FFC001FF00007C000131C7E9B18>I<3078FCFC783000 000000000000003078FCFC78300614779318>I<183C7E7E3C180000000000000000183C 7E7E3E1E0E1C3C78F060071A789318>I<000300000780001F80003F00007E0001FC0003 F00007E0001FC0003F00007E0000FC0000FC00007E00003F00001FC00007E00003F00001 FC00007E00003F00001F8000078000030011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE0 000000000000000000000000FFFFE0FFFFE0FFFFE07FFFC0130C7E9318>I<600000F000 00FC00007E00003F00001FC00007E00003F00001FC00007E00003F00001F80001F80003F 00007E0001FC0003F00007E0001FC0003F00007E0000FC0000F0000060000011187D9918 >I<00700000F80000F80000D80000D80001DC0001DC0001DC00018C00038E00038E0003 8E00038E000306000707000707000707000707000FFF800FFF800FFF800E03800E03801C 01C01C01C07F07F0FF8FF87F07F0151C7F9B18>65 DI<00F8E003FEE007FFE00F07E01E03E03C01E03800E07000E07000E0700000E0 0000E00000E00000E00000E00000E00000E00000E000007000007000E07000E03800E03C 00E01E01C00F07C007FF8003FE0000F800131C7E9B18>I<7FF800FFFE007FFF001C0F80 1C03C01C03C01C01E01C00E01C00E01C00F01C00701C00701C00701C00701C00701C0070 1C00701C00701C00F01C00E01C00E01C01E01C01C01C03C01C0F807FFF00FFFE007FF800 141C7F9B18>III<01F1C003FDC00FFFC01F0FC01C03C03803C03801C07001C07001C0 700000E00000E00000E00000E00000E00000E00FF0E01FF0E00FF07001C07001C07003C0 3803C03803C01C07C01F0FC00FFFC003FDC001F1C0141C7E9B18>I<7F07F0FF8FF87F07 F01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01FFFC01FFFC01FFF C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C07F07F0FF8F F87F07F0151C7F9B18>I<7FFF00FFFF807FFF0001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C0007FFF00FFFF807FFF00111C7D9B18>I<01FFC003FFC0 01FFC0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00F00E00F00E00F03C007FFC00 3FF0000FC000121C7D9B18>I<7F07F0FF87F87F07F01C03C01C07801C07001C0E001C1E 001C3C001C38001C70001CF0001DF0001DF0001FB8001FB8001F1C001E1C001C0E001C0E 001C07001C07001C03801C03801C01C07F03F0FF87F87F03F0151C7F9B18>I<7FE000FF E0007FE0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00700E00700E00700E00700E00707F FFF0FFFFF07FFFF0141C7F9B18>II<7E07 F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C01CC1C01CE1C01CE1C01CE1C01C61 C01C71C01C71C01C31C01C39C01C39C01C39C01C19C01C19C01C1DC01C0DC01C0DC01C0D C07F07C0FF87C07F03C0151C7F9B18>I<0FF8003FFE007FFF00780F00700700F00780E0 0380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E0 0380E00380E00380E00380F00780700700780F007FFF003FFE000FF800111C7D9B18>I< FFFE00FFFF80FFFFC01C03C01C01E01C00E01C00701C00701C00701C00701C00701C00E0 1C01E01C03C01FFFC01FFF801FFE001C00001C00001C00001C00001C00001C00001C0000 1C0000FF8000FF8000FF8000141C7F9B18>I<0FF8003FFE007FFF00780F00700700F007 80E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E003 80E00380E00380E1E380E1E380F0E78070F700787F007FFF003FFE000FFC00001C00001E 00000E00000F0000070000070011227D9B18>I<7FF800FFFE007FFF001C0F801C03801C 03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE001FFE001C0F001C07001C 03801C03801C03801C03801C03801C039C1C039C1C039C7F01F8FF81F87F00F0161C7F9B 18>I<03F3801FFF803FFF807C0F80700780E00380E00380E00380E00000700000780000 3F00001FF00007FE0000FF00000F800003C00001C00000E00000E06000E0E000E0E001E0 F001C0F80780FFFF80FFFE00E7F800131C7E9B18>I<7FFFF8FFFFF8FFFFF8E07038E070 38E07038E070380070000070000070000070000070000070000070000070000070000070 0000700000700000700000700000700000700000700000700007FF0007FF0007FF00151C 7F9B18>IIII<7F8FE07F9FE07F8FE00E 07000F0700070E00078E00039C0003DC0001F80001F80000F00000F00000700000F00000 F80001F80001DC00039E00038E00070F000707000E07800E03801E03C07F07F0FF8FF87F 07F0151C7F9B18>II<3FFFE07FFFE07FFF E07001C07003C0700780700700000F00001E00001C00003C0000780000700000F00001E0 0001C00003C0000780000700000F00001E00E01C00E03C00E07800E07000E0FFFFE0FFFF E0FFFFE0131C7E9B18>II93 D<7FFF00FFFF80FFFF807F FF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C38070E789E18>I<1FE0003FF8 007FFC00781E00300E0000070000070000FF0007FF001FFF007F0700780700E00700E007 00E00700F00F00781F003FFFF01FFBF007E1F014147D9318>I<7E0000FE00007E00000E 00000E00000E00000E00000E00000E3E000EFF800FFFC00FC1E00F80E00F00700E00700E 00380E00380E00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF8006 3E00151C809B18>I<01FE0007FF001FFF803E0780380300700000700000E00000E00000 E00000E00000E00000E000007000007001C03801C03E03C01FFF8007FF0001FC0012147D 9318>I<001F80003F80001F8000038000038000038000038000038003E3800FFB801FFF 803C1F80380F80700780700380E00380E00380E00380E00380E00380E003807007807007 80380F803C1F801FFFF00FFBF803E3F0151C7E9B18>I<01F00007FC001FFE003E0F0038 0780700380700380E001C0E001C0FFFFC0FFFFC0FFFFC0E000007000007001C03801C03E 03C01FFF8007FF0001FC0012147D9318>I<001F80007FC000FFE000E1E001C0C001C000 01C00001C0007FFFC0FFFFC0FFFFC001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C00001C00001C00001C0007FFF007FFF007FFF00131C7F9B18> I<01E1F007FFF80FFFF81E1E301C0E003807003807003807003807003807001C0E001E1E 001FFC001FF80039E0003800001C00001FFE001FFFC03FFFE07801F0700070E00038E000 38E00038E000387800F07E03F01FFFC00FFF8001FC00151F7F9318>I<7E0000FE00007E 00000E00000E00000E00000E00000E00000E3E000EFF800FFFC00FC1C00F80E00F00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFF E7FE7FC3FC171C809B18>I<03800007C00007C00007C000038000000000000000000000 0000007FC000FFC0007FC00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C00001C00001C000FFFF00FFFF80FFFF00111D7C9C18>I<0038 007C007C007C003800000000000000000FFC1FFC0FFC001C001C001C001C001C001C001C 001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C6038F078FFF0 7FE03F800E277E9C18>II<7FE000FFE000 7FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0007FFFC0 FFFFE07FFFC0131C7E9B18>I<7CE0E000FFFBF8007FFFF8001F1F1C001E1E1C001E1E1C 001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C 001C1C1C001C1C1C007F1F1F00FFBFBF807F1F1F001914819318>I<7E3E00FEFF807FFF C00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00 E00E00E00E00E07FC3FCFFE7FE7FC3FC1714809318>I<01F0000FFE001FFF003E0F8038 03807001C07001C0E000E0E000E0E000E0E000E0E000E0F001E07001C07803C03C07803E 0F801FFF000FFE0001F00013147E9318>I<7E3E00FEFF807FFFC00FC1E00F80E00F0070 0E00700E00380E00380E00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC0 0EFF800E3E000E00000E00000E00000E00000E00000E00000E00007FC000FFE0007FC000 151E809318>I<01E38007FB801FFF803E1F80380F80700780700780E00380E00380E003 80E00380E00380E00380700780700780380F803C1F801FFF800FFB8003E3800003800003 80000380000380000380000380000380003FF8003FF8003FF8151E7E9318>I<7F87E0FF 9FF07FBFF803F87803F03003E00003C00003C00003800003800003800003800003800003 80000380000380000380007FFE00FFFF007FFE0015147F9318>I<07F7003FFF007FFF00 780F00E00700E00700E007007C00007FE0001FFC0003FE00001F00600780E00380E00380 F00380F80F00FFFF00FFFC00E7F00011147D9318>I<0180000380000380000380000380 007FFFC0FFFFC0FFFFC00380000380000380000380000380000380000380000380000380 000380400380E00380E00380E001C1C001FFC000FF80003E0013197F9818>I<7E07E0FE 0FE07E07E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E 00E00E00E00E01E00F03E007FFFC03FFFE01FCFC1714809318>I<7F8FF0FF8FF87F8FF0 1E03C00E03800E03800E0380070700070700070700038E00038E00038E00038E0001DC00 01DC0001DC0000F80000F80000700015147F9318>II<7F8FF07F9FF07F8FF0070700078E00039E0001 DC0001F80000F80000700000F00000F80001DC00039E00038E000707000F07807F8FF0FF 8FF87F8FF015147F9318>I<7F8FF0FF8FF87F8FF00E01C00E03800E0380070380070700 070700038700038600038E0001CE0001CE0000CC0000CC0000DC00007800007800007800 00700000700000700000F00000E00079E0007BC0007F80003F00001E0000151E7F9318> I<3FFFF07FFFF07FFFF07001E07003C0700780000F00001E00003C0000F80001F00003C0 000780000F00701E00703C0070780070FFFFF0FFFFF0FFFFF014147F9318>I<0007E000 1FE0007FE000780000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00001E0007FC000FF8000FF80007FC00001E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E000007800007FE0001FE00007E013247E9F 18>I<60F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0 F0F0600424769F18>I<7C0000FF0000FFC00003C00000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000F000007FC0003FE0003FE0007FC000F0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00003C0 00FFC000FF00007C000013247E9F18>I E /Fm 62 124 df<0001FF81FE00001FFFEFFF 80007F80FF87C000FC00FE0FE001F801FE0FE003F801FC0FE007F001FC0FE007F001FC07 C007F001FC000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC00 0007F001FC0000FFFFFFFFF800FFFFFFFFF800FFFFFFFFF80007F001FC000007F001FC00 0007F001FC000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC00 0007F001FC000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC00 0007F001FC000007F001FC000007F001FC000007F001FC000007F001FC000007F001FC00 0007F001FC00007FFF1FFFE0007FFF1FFFE0007FFF1FFFE0002B2A7FA928>11 D<0001FF0000001FFFC000007F80F00000FC00F80001F801F80003F803FC0007F003FC00 07F003FC0007F003FC0007F001F80007F000F00007F000000007F000000007F000000007 F0000000FFFFFFFC00FFFFFFFC00FFFFFFFC0007F001FC0007F001FC0007F001FC0007F0 01FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001 FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC0007F001FC 0007F001FC0007F001FC0007F001FC007FFF1FFFC07FFF1FFFC07FFF1FFFC0222A7FA926 >I<1C007F007F00FF80FFC0FFC07FC07FC01CC000C000C0018001800180030003000600 0C001800300020000A157BA913>39 D45 D<1C003E007F00FF80FF80FF807F003E001C0009097B8813>I<003F800001 FFF00007E0FC000FC07E001F803F001F803F003F001F803F001F807F001FC07F001FC07F 001FC07F001FC0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF 001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE0FF001FE07F001FC07F 001FC07F001FC07F001FC03F001F803F001F801F803F001F803F000FC07E0007E0FC0001 FFF000003F80001B277DA622>48 D<000E00001E00007E0007FE00FFFE00FFFE00F8FE00 00FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE00 00FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE00 00FE0000FE0000FE0000FE0000FE007FFFFE7FFFFE7FFFFE17277BA622>I<00FF800003 FFF0000FFFFC001F03FE003800FF007C007F80FE003FC0FF003FC0FF003FE0FF001FE0FF 001FE07E001FE03C003FE000003FE000003FC000003FC000007F8000007F000000FE0000 00FC000001F8000003F0000003E00000078000000F0000001E0000003C00E0007000E000 E000E001C001C0038001C0070001C00FFFFFC01FFFFFC03FFFFFC07FFFFFC0FFFFFF80FF FFFF80FFFFFF801B277DA622>I<007F800003FFF00007FFFC000F81FE001F00FF003F80 FF003F807F803F807F803F807F801F807F800F007F800000FF000000FF000000FE000001 FC000001F8000007F00000FFC00000FFF0000001FC0000007E0000007F0000007F800000 3FC000003FC000003FE000003FE03C003FE07E003FE0FF003FE0FF003FE0FF003FC0FF00 7FC07E007F807C007F003F01FE001FFFFC0007FFF00000FF80001B277DA622>I<00000E 0000001E0000003E0000007E000000FE000000FE000001FE000003FE0000077E00000E7E 00000E7E00001C7E0000387E0000707E0000E07E0000E07E0001C07E0003807E0007007E 000E007E000E007E001C007E0038007E0070007E00E0007E00FFFFFFF8FFFFFFF8FFFFFF F80000FE000000FE000000FE000000FE000000FE000000FE000000FE000000FE00007FFF F8007FFFF8007FFFF81D277EA622>I<0C0003000F803F000FFFFE000FFFFC000FFFF800 0FFFF0000FFFE0000FFFC0000FFE00000E0000000E0000000E0000000E0000000E000000 0E0000000E7FC0000FFFF8000F80FC000E003E000C003F0000001F8000001FC000001FC0 00001FE000001FE018001FE07C001FE0FE001FE0FE001FE0FE001FE0FE001FC0FC001FC0 78003F8078003F803C007F001F01FE000FFFF80003FFF00000FF80001B277DA622>I<00 07F000003FFC0000FFFE0001FC0F0003F01F8007E03F800FC03F801FC03F801F803F803F 801F003F8000007F0000007F0000007F000000FF000000FF0FC000FF3FF800FF707C00FF C03E00FFC03F00FF801F80FF801FC0FF001FC0FF001FE0FF001FE0FF001FE07F001FE07F 001FE07F001FE07F001FE03F001FE03F001FC01F801FC01F803F800FC03F0007E07E0003 FFFC0000FFF000003FC0001B277DA622>I<380000003E0000003FFFFFF03FFFFFF03FFF FFF07FFFFFE07FFFFFC07FFFFF807FFFFF0070000E0070000E0070001C00E0003800E000 7000E000E0000000E0000001C000000380000007800000078000000F0000000F0000001F 0000001F0000003F0000003E0000003E0000007E0000007E0000007E0000007E000000FE 000000FE000000FE000000FE000000FE000000FE000000FE000000FE0000007C00000038 00001C297CA822>I<003FC00001FFF00003FFFC0007C07E000F003F001E001F001E000F 803E000F803E000F803F000F803F000F803FC00F003FF01F001FFC1E001FFE3C000FFFF8 0007FFE00003FFF80001FFFC0001FFFE0007FFFF000F0FFF801E03FFC03C01FFC07C007F E078001FE0F80007E0F80007E0F80003E0F80003E0F80003E0F80003C07C0003C07C0007 803F000F001FC03E000FFFFC0003FFF800007FC0001B277DA622>I<007F800001FFF000 07FFF8000FE0FC001F807E003F803F007F003F007F001F80FF001F80FF001FC0FF001FC0 FF001FC0FF001FE0FF001FE0FF001FE0FF001FE07F001FE07F003FE03F003FE01F807FE0 0F807FE007C1DFE003FF9FE0007E1FE000001FE000001FC000001FC000001FC000003F80 1F003F803F803F003F803F003F807E003F807C001F01F8001E03F0000FFFE00007FF8000 01FE00001B277DA622>I<01FF000FFFE01E03F03801F87801FCFC01FEFE01FEFE01FEFE 01FE7C01FE3801FC0003F80007F00007C0000F80001F00001E00003C0000380000380000 780000700000700000700000700000700000700000000000000000000000000000000000 000000700000F80001FC0003FE0003FE0003FE0001FC0000F800007000172A7CA920>63 D<000003800000000007C00000000007C0000000000FE0000000000FE0000000000FE000 0000001FF0000000001FF0000000003FF8000000003FF8000000003FF80000000073FC00 00000073FC00000000F3FE00000000E1FE00000000E1FE00000001C0FF00000001C0FF00 000003C0FF80000003807F80000007807FC0000007003FC0000007003FC000000E003FE0 00000E001FE000001E001FF000001C000FF000001FFFFFF000003FFFFFF800003FFFFFF8 0000780007FC0000700003FC0000700003FC0000E00001FE0000E00001FE0001E00001FF 0001C00000FF0001C00000FF00FFFE001FFFFEFFFE001FFFFEFFFE001FFFFE2F297EA834 >65 DI<00003FF001800003FFFE0380000FFFFF8780003FF007DF8000FF8001FF8001FE 00007F8003FC00003F8007F000001F800FF000000F801FE0000007801FE0000007803FC0 000007803FC0000003807FC0000003807F80000003807F8000000000FF8000000000FF80 00000000FF8000000000FF8000000000FF8000000000FF8000000000FF8000000000FF80 00000000FF80000000007F80000000007F80000000007FC0000003803FC0000003803FC0 000003801FE0000003801FE0000007000FF00000070007F000000E0003FC00001E0001FE 00003C0000FF8000F800003FF007E000000FFFFFC0000003FFFF000000003FF800002929 7CA832>IIII<0000 7FE003000003FFFC0700001FFFFF0F00003FF00FFF0000FF8001FF0001FE0000FF0003F8 00003F0007F000003F000FF000001F001FE000000F001FE000000F003FC000000F003FC0 000007007FC0000007007F80000007007F8000000000FF8000000000FF8000000000FF80 00000000FF8000000000FF8000000000FF8000000000FF8000000000FF8000000000FF80 01FFFFF87F8001FFFFF87F8001FFFFF87FC00000FF003FC00000FF003FC00000FF001FE0 0000FF001FE00000FF000FF00000FF0007F00000FF0003F80000FF0001FE0000FF0000FF 8001FF00003FF007BF00001FFFFF1F000003FFFE0F0000007FF003002D297CA836>I73 D76 DII<0000FFE000 000007FFFC0000003FC07F8000007F001FC00001FC0007F00003F80003F80007F00001FC 000FF00001FE001FE00000FF001FE00000FF003FC000007F803FC000007F807FC000007F C07F8000003FC07F8000003FC07F8000003FC0FF8000003FE0FF8000003FE0FF8000003F E0FF8000003FE0FF8000003FE0FF8000003FE0FF8000003FE0FF8000003FE0FF8000003F E0FF8000003FE07F8000003FC07FC000007FC07FC000007FC03FC000007F803FC000007F 801FE00000FF001FE00000FF000FF00001FE0007F00001FC0003F80003F80001FC0007F0 0000FF001FE000003FC07F8000000FFFFE00000000FFE000002B297CA834>II82 D<007F806003FFF0E007FFF9E00F807FE01F001FE03E0007E07C0003E07C0001E0FC0001 E0FC0001E0FC0000E0FE0000E0FE0000E0FF000000FFC000007FFE00007FFFE0003FFFFC 001FFFFE000FFFFF8007FFFFC003FFFFE000FFFFE00007FFF000007FF000000FF8000007 F8000003F8600001F8E00001F8E00001F8E00001F8F00001F0F00001F0F80003F0FC0003 E0FF0007C0FFE01F80F3FFFF00E0FFFE00C01FF0001D297CA826>I<7FFFFFFFFFC07FFF FFFFFFC07FFFFFFFFFC07F803FC03FC07E003FC007C078003FC003C078003FC003C07000 3FC001C0F0003FC001E0F0003FC001E0E0003FC000E0E0003FC000E0E0003FC000E0E000 3FC000E0E0003FC000E000003FC0000000003FC0000000003FC0000000003FC000000000 3FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC000000000 3FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC000000000 3FC0000000003FC0000000003FC0000000003FC0000000003FC0000000003FC00000007F FFFFE000007FFFFFE000007FFFFFE0002B287EA730>IIII<01FF800007FFF0000F81F8001FC07E001FC07E001FC03F000F80 3F8007003F8000003F8000003F8000003F80000FFF8000FFFF8007FC3F800FE03F803F80 3F803F003F807F003F80FE003F80FE003F80FE003F80FE003F807E007F807F00DF803F83 9FFC0FFF0FFC01FC03FC1E1B7E9A21>97 DI<001FF80000FFFE0003F01F0007E03F800F C03F801F803F803F801F007F800E007F0000007F000000FF000000FF000000FF000000FF 000000FF000000FF000000FF0000007F0000007F0000007F8000003F8001C01F8001C00F C0038007E0070003F01E0000FFFC00001FE0001A1B7E9A1F>I<00003FF80000003FF800 00003FF800000003F800000003F800000003F800000003F800000003F800000003F80000 0003F800000003F800000003F800000003F800000003F800000003F800001FE3F80000FF FBF80003F03FF80007E00FF8000FC007F8001F8003F8003F8003F8007F0003F8007F0003 F8007F0003F800FF0003F800FF0003F800FF0003F800FF0003F800FF0003F800FF0003F8 00FF0003F8007F0003F8007F0003F8007F0003F8003F8003F8001F8003F8000F8007F800 07C00FF80003F03BFF8000FFF3FF80003FC3FF80212A7EA926>I<003FE00001FFF80003 F07E0007C01F000F801F801F800F803F800FC07F000FC07F0007C07F0007E0FF0007E0FF 0007E0FFFFFFE0FFFFFFE0FF000000FF000000FF0000007F0000007F0000007F0000003F 8000E01F8000E00FC001C007E0038003F81F0000FFFE00001FF0001B1B7E9A20>I<0007 F0003FFC00FE3E01F87F03F87F03F07F07F07F07F03E07F00007F00007F00007F00007F0 0007F00007F000FFFFC0FFFFC0FFFFC007F00007F00007F00007F00007F00007F00007F0 0007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F0 0007F00007F0007FFF807FFF807FFF80182A7EA915>I<00FF81F003FFE7F80FC1FE7C1F 80FC7C1F007C383F007E107F007F007F007F007F007F007F007F007F007F007F007F003F 007E001F007C001F80FC000FC1F8001FFFE00018FF800038000000380000003C0000003E 0000003FFFF8001FFFFF001FFFFF800FFFFFC007FFFFE01FFFFFF03E0007F07C0001F8F8 0000F8F80000F8F80000F8F80000F87C0001F03C0001E01F0007C00FC01F8003FFFE0000 7FF0001E287E9A22>II<07000FC01FE03FE03FE03FE01FE00FC0070000000000000000 00000000000000FFE0FFE0FFE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00FE00F E00FE00FE00FE00FE00FE00FE00FE00FE00FE0FFFEFFFEFFFE0F2B7DAA14>I<00070000 0F80001FC0003FE0003FE0003FE0001FC0000F8000070000000000000000000000000000 000000000000000001FFE001FFE001FFE0000FE0000FE0000FE0000FE0000FE0000FE000 0FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE000 0FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE0000FE07C0FE0FE0FE0FE 0FC0FE1F80FE1F007C3E003FFC000FF000133784AA15>IIIII<003FE00001FFFC0003F07E000FC01F801F800FC0 3F800FE03F0007E07F0007F07F0007F07F0007F0FF0007F8FF0007F8FF0007F8FF0007F8 FF0007F8FF0007F8FF0007F8FF0007F87F0007F07F0007F03F800FE03F800FE01F800FC0 0FC01F8007F07F0001FFFC00003FE0001D1B7E9A22>II<001FC0380000FFF0780003F838F80007E00DF8000FC007F8001FC0 07F8003F8003F8007F8003F8007F8003F8007F0003F800FF0003F800FF0003F800FF0003 F800FF0003F800FF0003F800FF0003F800FF0003F8007F0003F8007F0003F8007F8003F8 003F8003F8001F8007F8000FC007F80007E01FF80003F07BF80000FFF3F800003FC3F800 000003F800000003F800000003F800000003F800000003F800000003F800000003F80000 0003F800000003F80000003FFF8000003FFF8000003FFF8021277E9A24>II<03FE300FFFF01E03F03800F0700070F00070F00070F80070FC 0000FFE0007FFE007FFF803FFFE01FFFF007FFF800FFF80003FC0000FC60007CE0003CF0 003CF00038F80038FC0070FF01E0F7FFC0C1FF00161B7E9A1B>I<007000007000007000 00700000F00000F00000F00001F00003F00003F00007F0001FFFF0FFFFF0FFFFF007F000 07F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F00007F000 07F03807F03807F03807F03807F03807F03803F03803F87001F86000FFC0001F8015267F A51B>III< FFFE7FFC0FFEFFFE7FFC0FFEFFFE7FFC0FFE0FE007E000E007F003F001C007F003F001C0 07F807F803C003F807F8038003F807F8038001FC0EFC070001FC0EFC070001FE1EFC0F00 00FE1C7E0E0000FE1C7E0E0000FF383F1E00007F383F1C00007F783F3C00003FF01FB800 003FF01FB800003FF01FF800001FE00FF000001FE00FF000000FC007E000000FC007E000 000FC007E00000078003C00000078003C0002F1B7F9A32>III<3FFFFF803FFFFF803F007F003C00FE003801FE007803FC007803F8 007007F800700FF000700FE000001FC000003FC000007F8000007F000000FF000001FE03 8001FC038003F8038007F803800FF007800FE007801FE007003FC00F003F801F007F007F 00FFFFFF00FFFFFF00191B7E9A1F>II E /Fn 22 121 df<0001FF8000000FFFF000003FFFFC0000FF81FF0001FE007F8003FC00 3FC007F8001FE00FF8001FF00FF0000FF01FF0000FF81FF0000FF83FF0000FFC3FE00007 FC3FE00007FC7FE00007FE7FE00007FE7FE00007FE7FE00007FE7FE00007FEFFE00007FF FFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFF E00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE00007FFFFE0 0007FFFFE00007FFFFE00007FFFFE00007FF7FE00007FE7FE00007FE7FE00007FE7FE000 07FE7FE00007FE3FE00007FC3FF0000FFC3FF0000FFC1FF0000FF81FF0000FF80FF0000F F00FF8001FF007F8001FE003FC003FC001FE007F8000FF81FF00003FFFFC00000FFFF000 0001FF800028397CB731>48 D<00001E000000003E00000000FE00000003FE0000003FFE 0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE00007FFFFFFFC07FFFFFFFC07FFFFFFFC0 7FFFFFFFC0223879B731>I<0007FE0000007FFFE00001FFFFF80003FFFFFE0007F01FFF 000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC0007FF0FFE0007FF8FFF0003FF8 FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE0003FFC3FC0003FFC1F80003FFC00 00003FFC0000003FF80000007FF80000007FF00000007FF0000000FFE0000000FFC00000 01FF80000001FF00000003FE00000007FC00000007F80000000FF00000001FC00000003F 800000007F00000000FC00000001F800000001F0003C0003E0003C0007C0003C000F0000 78001E000078003C00007800780000F800F00000F801FFFFFFF803FFFFFFF007FFFFFFF0 0FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFFFFF0FFFFFFFFE0FFFFFFFFE0FF FFFFFFE026387BB731>I<0003FF8000001FFFF000007FFFFE0000FE03FF0001F000FF80 03C000FFC00780007FE00FF0007FF00FF8007FF01FFC007FF81FFE007FF81FFE007FF81F FE007FF81FFE007FF81FFE007FF80FFC007FF007F8007FF003F0007FF0000000FFE00000 00FFC0000001FF80000001FF00000003FE00000007FC0000001FF000000FFFC000000FFF 8000000FFFF800000003FE00000000FF800000007FE00000003FF00000003FF80000003F FC0000001FFC0000001FFE0000001FFE0200001FFF1FC0001FFF3FE0001FFF7FF0001FFF 7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF8001FFEFFF0001FFE7FF0003FFC7F E0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FFC003FFFFFF0001FFFFFE00003F FFF0000007FF800028397CB731>I<00000007C0000000000FC0000000000FC000000000 1FC0000000003FC0000000007FC000000000FFC000000000FFC000000001FFC000000003 FFC000000007FFC00000000FFFC00000000FFFC00000001EFFC00000003CFFC00000007C FFC0000000F8FFC0000000F0FFC0000001E0FFC0000003C0FFC0000007C0FFC000000F80 FFC000000F00FFC000001E00FFC000003C00FFC000007C00FFC00000F800FFC00000F000 FFC00001E000FFC00003C000FFC00007C000FFC0000F8000FFC0000F0000FFC0001E0000 FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFFFFFFFFC0FFFFFFFFFFC0FFFFFF FFFFC0FFFFFFFFFFC0000001FFC000000001FFC000000001FFC000000001FFC000000001 FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000007FF FFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A377DB631>I<04000000C00F8000 07C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000FFFFFFC000FFFFFF8000FFFFFF0 000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F800000000F800000000F80000000 0F800000000F800000000F800000000F800000000F800000000F81FF00000F8FFFE0000F BFFFF8000FFE03FE000FF000FF000FC000FF800F80007FC00F00007FE00700007FF00000 003FF00000003FF80000003FF80000003FF80000003FFC0000003FFC0600003FFC1F8000 3FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFFF0003FFCFFF0003FF8FFE0003F F8FFE0003FF87FC0007FF07F00007FF03C00007FE03E0000FFC01F0000FF800FC003FF00 07F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC000026397BB731>I<00000FF8 0000007FFF000003FFFF80000FFC07C0001FE001E0003FC001F0007F0007F000FF000FF0 01FE001FF803FC003FF807FC003FF80FFC003FF80FF8003FF81FF8001FF01FF8000FE03F F80007C03FF00000003FF00000007FF00000007FF00000007FF00000007FF07FF000FFF0 FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0FFFC001FF0FFFC000FF8FFF800 0FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFFF0000FFFFFF0000FFFFFF0000F FF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF3FF0000FFF3FF0000FFE 3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF8001FF807FC001FF003FC001FF001 FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF8000000FFC00028397CB731>I< 1E00000000001F00000000001FF0000000001FFFFFFFFFC01FFFFFFFFFC01FFFFFFFFFC0 3FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE003FFFFFFFFC003FFFFFFFF800 3FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC0007800001F80007800001F0000 7800003E0000F000007E0000F00000FC0000F00001F80000000003F00000000003E00000 000007E0000000000FC0000000000F80000000001F80000000003F80000000003F000000 00007F00000000007F0000000000FF0000000000FE0000000001FE0000000001FE000000 0003FE0000000003FE0000000003FE0000000007FC0000000007FC0000000007FC000000 000FFC000000000FFC000000000FFC000000000FFC000000000FFC000000001FFC000000 001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000 001FFC000000001FFC000000000FF80000000007F00000000003E00000002A3B7BB931> I<0001FF8000000FFFF800003FFFFE00007F00FF0000F8003F8001F0000FC003E0000FE0 07C00007F007C00007F00FC00003F80FC00003F81FC00003F81FC00003F81FE00003F81F F00003F81FF80003F81FFC0007F01FFF0007F01FFFC007E00FFFF00FE00FFFFC1FC007FF FE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE00007FFFFF00003FFFFF80003FFF FFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1FC007FFFC3F8001FFFE3F8000FF FE7F00003FFF7F00000FFFFE000003FFFE000001FFFE000000FFFE000000FFFE0000007F FE0000007FFE0000007EFE0000007E7F0000007E7F000000FC3F800000FC3FC00001F81F E00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE00003FFFF8000003FF80002839 7CB731>I<0001FF8000001FFFF000007FFFFC0000FF81FE0001FE007F0007FC003F800F F8003FC00FF8001FE01FF8001FF03FF0001FF03FF0000FF87FF0000FF87FF0000FFCFFF0 000FFCFFF0000FFCFFF0000FFEFFF0000FFEFFF0000FFEFFF0000FFEFFF0000FFEFFF000 0FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0001FFF7FF0001FFF3FF0001FFF3FF0001F FF1FF0003FFF0FF8003FFF07F8007FFF03FC006FFF01FE01CFFF00FFEF8FFF007FFF0FFF 000FFE0FFE0000000FFE0000000FFE0000000FFE0000000FFC0000000FFC03E0001FFC07 F0001FF80FF8001FF81FFC001FF01FFC003FF01FFC003FE01FFC003FC01FF8007FC00FF0 00FF800FE001FF0007C003FE0007F01FFC0003FFFFF00001FFFFC000007FFF0000000FF8 000028397CB731>I<00000001F80000000000000001F80000000000000003FC00000000 00000003FC0000000000000007FE0000000000000007FE0000000000000007FE00000000 0000000FFF000000000000000FFF000000000000001FFF800000000000001FFF80000000 0000001FFF800000000000003FFFC00000000000003FFFC00000000000007FFFE0000000 0000007DFFE00000000000007DFFE0000000000000FDFFF0000000000000F8FFF0000000 000001F8FFF8000000000001F07FF8000000000001F07FF8000000000003F07FFC000000 000003E03FFC000000000007E03FFE000000000007C01FFE00000000000FC01FFF000000 00000F801FFF00000000000F800FFF00000000001F800FFF80000000001F0007FF800000 00003F0007FFC0000000003E0003FFC0000000003E0003FFC0000000007E0003FFE00000 00007C0001FFE000000000FC0001FFF000000000F80000FFF000000000F80000FFF00000 0001FFFFFFFFF800000001FFFFFFFFF800000003FFFFFFFFFC00000003FFFFFFFFFC0000 0003E000003FFC00000007E000003FFE00000007C000001FFE0000000FC000001FFF0000 000F8000000FFF0000001F8000000FFF8000001F0000000FFF8000001F00000007FF8000 003F00000007FFC000003E00000003FFC000007E00000003FFE00000FF00000001FFE000 FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0 443B7DBA4B>65 D<0000001FFF000030000001FFFFE000F000000FFFFFFC01F000007FFF FFFE03F00001FFFE007F87F00003FFE0000FCFF0000FFF000003FFF0001FFC000001FFF0 003FF80000007FF0007FF00000003FF000FFC00000003FF001FFC00000001FF003FF8000 00000FF007FF000000000FF00FFF0000000007F00FFE0000000007F01FFE0000000003F0 1FFE0000000003F03FFC0000000003F03FFC0000000001F03FFC0000000001F07FFC0000 000001F07FF80000000001F07FF80000000000007FF8000000000000FFF8000000000000 FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF80000 00000000FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000 FFF80000000000007FF80000000000007FF80000000000007FF80000000000007FFC0000 000000F03FFC0000000000F03FFC0000000000F03FFC0000000000F01FFE0000000000F0 1FFE0000000001E00FFE0000000001E00FFF0000000001E007FF0000000003C003FF8000 000003C001FFC0000000078000FFE00000000F00007FF00000001F00003FF80000003E00 001FFC0000007C00000FFF000001F8000003FFE00007F0000001FFFE003FC00000007FFF FFFF000000000FFFFFFC0000000001FFFFF000000000001FFF0000003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007FE00000FF801FF00001FFC00FF80 001FFC007FC0001FFC007FE0001FFC003FE0000FF8003FF0000FF8003FF00007F0003FF0 0001C0003FF0000000003FF0000000003FF0000000003FF0000000FFFFF000000FFFFFF0 00007FF83FF00001FF803FF00007FE003FF0000FF8003FF0001FF0003FF0003FE0003FF0 007FE0003FF0007FE0003FF000FFC0003FF000FFC0003FF000FFC0003FF000FFC0003FF0 00FFC0007FF0007FE0007FF0007FE000DFF0003FF0039FF8001FFC0F0FFFF007FFFE0FFF F001FFFC07FFF0003FE000FFF02C267DA530>97 D<000000003FC00000003FFFC0000000 3FFFC00000003FFFC00000003FFFC000000001FFC000000000FFC000000000FFC0000000 00FFC000000000FFC000000000FFC000000000FFC000000000FFC000000000FFC0000000 00FFC000000000FFC000000000FFC000000000FFC000000000FFC000000000FFC0000000 00FFC000000000FFC00000FFC0FFC0000FFFF8FFC0003FFFFEFFC000FFE03FFFC001FF00 0FFFC003FE0003FFC007FC0001FFC00FF80000FFC01FF00000FFC01FF00000FFC03FF000 00FFC03FE00000FFC07FE00000FFC07FE00000FFC07FE00000FFC0FFE00000FFC0FFE000 00FFC0FFE00000FFC0FFE00000FFC0FFE00000FFC0FFE00000FFC0FFE00000FFC0FFE000 00FFC0FFE00000FFC07FE00000FFC07FE00000FFC07FE00000FFC03FF00000FFC03FF000 00FFC01FF00000FFC00FF80001FFC007F80003FFC003FC0007FFC001FE000FFFE000FFC0 7EFFFF003FFFFCFFFF000FFFF0FFFF0001FF80FFFF303C7DBB37>100 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007FC001FC00FF8000FE0 0FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE00003FC7FE00003FC7F E00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE0000000FFE0000000FFE0 000000FFE00000007FE00000007FE00000007FE00000003FE00000003FF000003C1FF000 003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E000FFC03FC0003FFFFF 000007FFFC000000FFE00026267DA52D>I<00FF00000000FFFF00000000FFFF00000000 FFFF00000000FFFF0000000007FF0000000003FF0000000003FF0000000003FF00000000 03FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF00000000 03FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF00000000 03FF0000000003FF007FC00003FF01FFF80003FF07FFFC0003FF0F03FE0003FF1C01FF00 03FF3001FF8003FF6000FF8003FFE000FFC003FFC000FFC003FF8000FFC003FF8000FFC0 03FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0 03FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0 03FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0 03FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFF FFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF303C7CBB37>104 D<00F00003FC0007FE00 0FFE000FFF001FFF001FFF001FFF000FFF000FFE0007FE0003FC0000F000000000000000 00000000000000000000000000000000000000000000000000FF00FFFF00FFFF00FFFF00 FFFF0007FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF00 03FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF0003FF00 03FF0003FF0003FF0003FF0003FF0003FF00FFFFF8FFFFF8FFFFF8FFFFF8153D7DBC1B> I<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FFFE1C01FF0007FE3001FF 8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003FF8000FFC003FF8000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFF FFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<00FF01FF8000FFFF0FFFF000FFFF 3FFFFC00FFFFFE03FF00FFFFF000FF8003FFC0007FC003FF80003FE003FF00003FF003FF 00001FF803FF00001FFC03FF00000FFC03FF00000FFE03FF00000FFE03FF000007FE03FF 000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF 000007FF03FF000007FF03FF000007FF03FF000007FE03FF000007FE03FF00000FFE03FF 00000FFC03FF00000FFC03FF00001FF803FF00001FF803FF00003FF003FF80003FE003FF C0007FC003FFF001FF8003FFFC07FF0003FF3FFFFC0003FF0FFFF00003FF01FF000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 00000000FFFFFC000000FFFFFC000000FFFFFC000000FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80FFFE707FC007FE60FFE003FEE0FFE0 03FEC0FFE003FFC0FFE003FF807FC003FF807FC003FF803F8003FF800E0003FF00000003 FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF000000FFFFFE0000FFFFFE00 00FFFFFE0000FFFFFE000023267DA529>114 D<00078000000780000007800000078000 00078000000F8000000F8000000F8000000F8000001F8000001F8000003F8000003F8000 007F800000FF800001FF800007FF80001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C00FF8078 00FFC078007FC070003FE0E0001FFFC00007FF800001FF001E377EB626>116 D120 D E /Fo 10 107 df0 D<70F8F8F87005057C8D0D>I<01800180018001 800180C183F18F399C0FF003C003C00FF0399CF18FC1830180018001800180018010147D 9417>3 D<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF7FFE7FFE3FFC1F F80FF003C010127D9317>15 D<000000C0000003C000000F0000003C000000F0000003C0 0000070000001C00000078000001E00000078000001E00000078000000E0000000780000 001E0000000780000001E0000000780000001C0000000700000003C0000000F00000003C 0000000F00000003C0000000C00000000000000000000000000000000000000000000000 00000000007FFFFF80FFFFFFC01A247C9C23>20 DI<0000000400000000 020000000002000000000100000000008000000000400000000020FFFFFFFFFCFFFFFFFF FC0000000020000000004000000000800000000100000000020000000002000000000400 26107D922D>33 D<000F0038006000E001C001C001C001C001C001C001C001C001C001C0 01C001C001C001C001C0038007001E00F8001E000700038001C001C001C001C001C001C0 01C001C001C001C001C001C001C001C001C000E000600038000F102D7DA117>102 DI106 D E /Fp 1 98 df<001800003C00003C00003C00007E00007E00007E00 00CF0000CF0000CF000187800187800387C00303C00303C007FFE007FFE00601E00C00F0 0C00F01E00F8FF03FFFF03FF18177F961C>97 D E /Fq 40 90 df<387C7C7E3A040404 080810204080070E789F0D>39 D<1C3E7E7E3A0202040408081020C0070E7D840D>44 DI<3078F8787005057C840D>I<007E0001C3000301800701C00E 00C00E00E01C00E01C00E03C01E03801E07801E07801E07801E07801E07801E0F003C0F0 03C0F003C0F003C0F003C0F00380F00780E00780E00700E00700E00E00600E00701C0030 38003870000FC000131F7C9D17>48 D<000C001C00FC0F38003800380038003800380070 0070007000700070007000E000E000E000E000E000E001C001C001C001C001C001C00380 03C0FFFE0F1E7C9D17>I<003F8000C1E00100F00200780400780400780F007C0F807C0F 807C0F00780600780000F80000F00001E00001C0000380000700000E00001C0000380000 600000C0000180000300200600200800401000403FFFC07FFF80FFFF80161E7E9D17>I< 007F000183C00201E00400F00700F00F00F00F01F00F01F00001E00001E00003C0000380 000700000E0000F800000E000007000007800007C00003C00007C03007C07807C0F807C0 F807C0F00780800F00400E00201C0018780007E000141F7D9D17>I<0000600000600000 E00001C00003C00005C0000DC00009C00011C000238000438000C3800083800103800203 80040700080700180700100700200700400700FFFFF0000E00000E00000E00000E00000E 00001C00001E0001FFE0141E7E9D17>I<01803001FFE003FFC003FF0003FC0002000002 0000020000040000040000040000047C000587000603800C01800801C00001C00001E000 01E00001E00001E07003C0F803C0F003C0E00380800780400700400E00201C0018700007 C000141F7D9D17>I<000F8000704000C0200180E00301E00701E00E00C01E00001C0000 3C000038000078F800790E007A07007C0300F80380F80380F003C0F003C0F003C0F003C0 F00780E00780E00780E00700E00F00600E00701C0030180018700007C000131F7C9D17> I<2000003FFFE07FFFC07FFF80400100C002008002008004000008000010000020000040 000040000080000180000300000300000700000600000E00000E00001E00001C00001C00 003C00003C00003C0000780000780000780000300000131F799D17>I<003F0000C1C001 00600200600400300C00300C00300C00300C00600E00600F80C00FC18007F60003FC0001 FC0001FF00063F800C0F801007C03003C06001C06000C0C000C0C000C0C000C0C00080C0 010060030030040018180007E000141F7D9D17>I<007E0001C3000301800601C00E01C0 1C00C03C00E03C00E03C01E07801E07801E07801E07801E07803E07803E03803C03807C0 1C0BC00C13C003E380000780000780000700000E00600E00F01C00F01800E03000806000 41C0003F0000131F7C9D17>I<07E01838201C401C701CF03CF03C603C0038007000E001 C001800300020006000400040008000800080008000000000000000000000030007800F8 00780070000E20799F15>63 D<0000100000001800000038000000380000007800000078 000000FC000001BC0000013C0000033C0000023C0000063C0000043E0000081E0000081E 0000101E0000101E0000201E0000200F0000400F0000400F0000FFFF0000800F0001000F 8001000780020007800200078004000780040007800C0007C03E0007C0FF807FFC1E207E 9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800 F8007800F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007 C001E003C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003 C003C007C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E0618 00380138007000F801E0007803C0007007800030078000300F0000301F0000301E000030 3E0000203C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000 F8000000F80000007800004078000080780000803C0000803C0001001C0002000E000200 06000C000300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F000 3C00780078003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00 F0001F00F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01 E0007C03C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FF FFE000201F7E9E23>I<07FFFFF8007C0078003C0038003C001800780018007800080078 000800780008007800080078080800F0100000F0100000F0100000F0300000FFF00000F0 700001E0200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C0 002003C0002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFF F8007C0078003C0038003C00180078001800780008007800080078000800780008007800 0800F0100000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E020 0001E0200001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C000 000780000007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C 00E0003C01C0003803800018078000180F0000181F0000181E0000183E0000103C000000 7C0000007C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E0 780001E0780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B8003801180 01E06080003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C00780000 7800F000007800F000007800F000007800F000007800F000007800F00000F001E00000F0 01E00000F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003 C00001E003C00001E003C00001E003C00003C007800003C007800003C007800003C00780 0003C007800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0 007C00003C00003C0000780000780000780000780000780000780000F00000F00000F000 00F00000F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C000 03C00003C00003C00007800007C000FFFC00131F7F9E10>I<00FFFC0007C00003C00003 C0000780000780000780000780000780000780000F00000F00000F00000F00000F00000F 00001E00001E00001E00001E00001E00001E00003C00303C00783C00F83C00F83800F078 0080700040E00021C0001F000016207D9E17>I<07FFE07FE0007C001F00003C000C0000 3C00180000780010000078004000007800800000780100000078020000007804000000F0 08000000F010000000F060000000F0F0000000F1F0000000F278000001E478000001E878 000001F03C000001E03C000001E01E000001E01E000003C00F000003C00F000003C00F00 0003C007800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F7E9E23 >I<07FFF000007E0000003C0000003C0000007800000078000000780000007800000078 00000078000000F0000000F0000000F0000000F0000000F0000000F0000001E0000001E0 000001E0000001E0000001E0008001E0010003C0010003C0010003C0030003C0020003C0 060003C0060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC0007C0000F8 00003C00017800003C00017800004E0002F000004E0002F000004E0004F000004E0004F0 00004E0008F000004E0008F00000870011E00000870011E00000870021E00000870021E0 0000870041E00000838041E00001038083C00001038083C00001038103C00001038203C0 000101C203C0000101C403C0000201C40780000201C80780000201C80780000201D00780 000200F00780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A1F7E9E2A >I<07FC01FFC0003E003E00003E001800003E001800004F001000004F00100000478010 0000478010000043C010000043C010000083C020000081E020000081E020000080F02000 0080F020000080782000010078400001007C400001003C400001003C400001001E400001 001E400002000F800002000F800002000F80000200078000020007800006000380000600 0300000F00010000FFE0010000221F7E9E22>I<0003F800001E0E000038070000E00380 01C001C003C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F83C0000F8 7C0000F87C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0F80003E0 780003E0780003C0780007C07C0007803C000F003C001E001E001C000E0038000700F000 03C3C00000FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F0007800F000 7800F8007800F8007800F8007800F8007800F000F001F000F001E000F003C000F0078000 F00F0000FFF80001E0000001E0000001E0000001E0000001E0000001E0000003C0000003 C0000003C0000003C0000003C0000003C000000780000007C00000FFFC00001D1F7E9E1F >I<0003F800001E0E000038070000F0038001E001C003C001E0078001E00F0000F00F00 00F01F0000F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C0000F8F800 01F0F80001F0F80001F0F80001F0F80003E0780003E0780003C0780007C0781E07803C21 0F003C409E001E409C000E80B8000740F00003C1C04000FEC0400000C0400000C0800000 E1800000FF800000FF000000FF0000007E0000003C001D297B9F23>I<07FFFC00007C07 00003C03C0003C01E0007801E0007801F0007801F0007801F0007801F0007801E000F003 E000F003C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E01C0001E01C 0001E01C0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C03E0207803E 0407C01F04FFFC0F18000003E01F207E9E21>I<003F040060CC01803C03801C03001C07 00180600080E00080E00080E00080E00000F00000F80000FE00007FE0003FF8001FFC000 7FE00007E00001E00000E00000F00000F04000E04000E04000E04000E06000C0600180E0 0380F80300C60C0081F80016217D9F19>I<3FFFFFF03C0780F03007803060078030400F 0010400F0010C00F0010800F0010800F0010800F0010001E0000001E0000001E0000001E 0000001E0000001E0000003C0000003C0000003C0000003C0000003C0000003C00000078 0000007800000078000000780000007800000078000000F0000001F800007FFFE0001C1F 7A9E21>IIII<03FFC0FFC0007F007E00003E003800001E003000001E002000000F004000 000F008000000F81000000078200000007C600000003C400000003E800000001F0000000 01F000000000F000000000F800000000F8000000017C000000023C000000043C0000000C 1E000000081E000000101F000000200F000000400F800000C0078000008007C000010003 C000070003E0001F8007E000FFE01FFE00221F7F9E22>II E /Fr 23 123 df<70F8F8F87005057C840D>58 D<70F8FCFC74040404080810102040060E7C840D>I<000001C00000078000001E000000 78000001E00000078000000E00000038000000F0000003C000000F0000003C000000F000 0000F00000003C0000000F00000003C0000000F0000000380000000E0000000780000001 E0000000780000001E0000000780000001C01A1A7C9723>I62 D<007C01C207010E0F1E0F1C0E3C04780078007800F000F000F000F000F0007001700230 0418380FC010147E9314>99 D<0000780003F80000700000700000700000700000E00000 E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C03803807007807 00780700780700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F07 8015207E9F18>I<007C01C207010E011C013C013802780C7BF07C00F000F000F000F000 7000700170023004183807C010147E9315>I<00007C0000CE00019E00039E00030C0007 00000700000700000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C 00001C00001C000038000038000038000038000038000070000070000070000070000070 0000E00000E00000E00000E00000C00001C000318000798000F300006200003C00001729 7E9F16>I<001E3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E 01C01E01C01E01C01E01C01E03801E03800E07800E0B8006170001E70000070000070000 0E00000E00300E00781C00F038006070003FC000151D809316>I<00E001E001E000C000 000000000000000000000000000E00130023804380438043808700070007000E000E001C 001C001C20384038403840388019000E000B1F7E9E10>105 D<0000C00001E00001E000 01C0000000000000000000000000000000000000000000001E0000630000438000838001 0380010380020700000700000700000700000E00000E00000E00000E00001C00001C0000 1C00001C0000380000380000380000380000700000700030700078E000F1C0006380003E 00001328819E13>I<01E0000FE00001C00001C00001C00001C000038000038000038000 0380000700000700000701E00706100E08700E10F00E20F00E40601C80001D00001E0000 1FC000387000383800383800381C20703840703840703840701880E01880600F0014207E 9F18>I<03C01FC0038003800380038007000700070007000E000E000E000E001C001C00 1C001C0038003800380038007000700070007100E200E200E200E200640038000A207E9F 0E>I<1E07C07C00231861860023A032030043C034030043803803804380380380870070 07000700700700070070070007007007000E00E00E000E00E00E000E00E00E000E00E01C 101C01C01C201C01C038201C01C038401C01C0184038038018801801800F0024147E9328 >I<1E07802318C023A06043C0704380704380708700E00700E00700E00700E00E01C00E 01C00E01C00E03821C03841C07041C07081C03083803101801E017147E931B>I<007C00 01C3000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0 F00780F00700700F00700E0030180018700007C00013147E9316>I<03C1E00462180474 1C08781C08701E08701E10E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380 780380700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00 001C0000FFC000171D819317>I<1E1E0023210023C38043C78043878043830087000007 00000700000700000E00000E00000E00000E00001C00001C00001C00001C000038000018 000011147E9315>114 D<007C018203010603060706060E00078007F803FC01FE001F00 077007F006F006E004400820301FC010147E9315>I<00C000E001C001C001C001C00380 0380FFF8038007000700070007000E000E000E000E001C001C001C001C10382038203820 384018800F000D1C7F9B10>I<03C1C00C62201034701038F02038F02038604070000070 0000700000700000E00000E00000E00000E02061C040F1C040F1C080E2C080446300383C 0014147E931A>120 D<0F00601180702180E021C0E041C0E04380E08381C00701C00701 C00701C00E03800E03800E03800E03800E07000C07000C07000E0F00061E0003EE00000E 00000E00001C0078180078380070700060600021C0001F0000141D7E9316>I<01E02003 F04007F8C00C1F8008010000020000040000080000100000600000C00001000002000004 00800801001003003F060061FC0040F80080700013147E9315>I E /Fs 88 128 df<001F83E000F06E3001C078780380F8780300F0300700700007007000 0700700007007000070070000700700007007000FFFFFF80070070000700700007007000 070070000700700007007000070070000700700007007000070070000700700007007000 0700700007007000070070000700700007007000070070007FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700000700000700 00FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000 E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F04 0001C07C06000380F80F000300F00F000700F00F00070070000007007000000700700000 070070000007007000000700700000FFFFFFFF0007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 07007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C07C0F000380F80F000300 F00700070070070007007007000700700700070070070007007007000700700700070070 0700FFFFFFFF000700700700070070070007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 070070070007007007000700700700070070070007007007007FE3FE3FF02420809F26> I<70F8F8F8F8F8F8F8707070707070707070702020202020000000000070F8F8F8700521 7CA00D>33 D<7038F87CFC7EFC7E743A0402040204020804080410081008201040200F0E 7E9F17>I<70F8FCFC74040404080810102040060E7C9F0D>39 D<002000400080010002 0006000C000C00180018003000300030007000600060006000E000E000E000E000E000E0 00E000E000E000E000E000E0006000600060007000300030003000180018000C000C0006 00020001000080004000200B2E7DA112>I<800040002000100008000C00060006000300 030001800180018001C000C000C000C000E000E000E000E000E000E000E000E000E000E0 00E000E000C000C000C001C001800180018003000300060006000C000800100020004000 80000B2E7DA112>I<01800180018001800180C183F18F399C0FF003C003C00FF0399CF1 8FC1830180018001800180018010147DA117>I<00060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 0006000000060000FFFFFFF0FFFFFFF00006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 000600001C207D9A23>I<70F8FCFC74040404080810102040060E7C840D>II<70F8F8F87005057C840D>I<000100030003000600060006000C000C000C00 180018001800300030003000600060006000C000C000C001800180018003000300030006 00060006000C000C000C00180018001800300030003000600060006000C000C000C00010 2D7DA117>I<03F0000E1C001C0E00180600380700700380700380700380700380F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0 7003807003807003807807803807001806001C0E000E1C0003F000121F7E9D17>I<0180 03800F80F380038003800380038003800380038003800380038003800380038003800380 03800380038003800380038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C0010 0E00200700400780800780F007C0F803C0F803C0F803C02007C00007C000078000078000 0F00000E00001C0000380000700000600000C0000180000300000600400C004018004010 00803FFF807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00780F80780780 780780380F80000F80000F00000F00000E00001C0000380003F000003C00000E00000F00 0007800007800007C02007C0F807C0F807C0F807C0F00780400780400F00200E001C3C00 03F000121F7E9D17>I<000600000600000E00000E00001E00002E00002E00004E00008E 00008E00010E00020E00020E00040E00080E00080E00100E00200E00200E00400E00C00E 00FFFFF0000E00000E00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I<18 03001FFE001FFC001FF8001FE00010000010000010000010000010000010000011F00016 1C00180E001007001007800003800003800003C00003C00003C07003C0F003C0F003C0E0 0380400380400700200600100E000C380003E000121F7E9D17>I<007C00018200070100 0E03800C07801C0780380300380000780000700000700000F1F000F21C00F40600F80700 F80380F80380F003C0F003C0F003C0F003C0F003C07003C07003C0700380380380380700 1807000C0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF804001008002 0080020080040000080000080000100000200000200000400000400000C00000C00001C0 000180000380000380000380000380000780000780000780000780000780000780000780 00030000121F7D9D17>I<03F0000C0C0010060030030020018060018060018060018070 01807803003E03003F06001FC8000FF00003F80007FC000C7E00103F00300F8060038040 01C0C001C0C000C0C000C0C000C0C000806001802001001002000C0C0003F000121F7E9D 17>I<03F0000E18001C0C00380600380700700700700380F00380F00380F003C0F003C0 F003C0F003C0F003C07007C07007C03807C0180BC00E13C003E3C0000380000380000380 000700300700780600780E00700C002018001070000FC000121F7E9D17>I<70F8F8F870 0000000000000000000070F8F8F87005147C930D>I<70F8F8F870000000000000000000 0070F0F8F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF0000000000000 0000000000000000000000000000000000000000000000000000FFFFFFF07FFFFFE01C0C 7D9023>61 D<07000F800F800F8007000000000000000000000002000200020002000200 02000600060004000C001C00380078007000F018F03CF03CF01CF008701838200FC00E20 7D9615>I<0FC0307040384038E03CF03CF03C603C0038007000E000C001800180010003 000200020002000200020002000000000000000000000007000F800F800F8007000E207D 9F15>I<000100000003800000038000000380000007C0000007C0000007C0000009E000 0009E0000009E0000010F0000010F0000010F00000207800002078000020780000403C00 00403C0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F0002000780 0200078002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F204F00 07A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000001FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F00 001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004078004 0F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C>II<080410082010201040204020804080408040B85CFC7E FC7E7C3E381C0F0E7B9F17>II<1FE00030300078 1800781C00300E00000E00000E00000E0000FE00078E001E0E00380E00780E00F00E10F0 0E10F00E10F01E10781E103867200F83C014147E9317>97 D<0E0000FE00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E3E000EC3800F01C00F00 E00E00E00E00700E00700E00780E00780E00780E00780E00780E00780E00700E00700E00 E00F00E00D01C00CC300083E0015207F9F19>I<03F80E0C1C1E381E380C70007000F000 F000F000F000F000F00070007000380138011C020E0C03F010147E9314>I<000380003F 8000038000038000038000038000038000038000038000038000038000038003E380061B 801C0780380380380380700380700380F00380F00380F00380F00380F00380F003807003 807003803803803807801C07800E1B8003E3F815207E9F19>I<03F0000E1C001C0E0038 0700380700700700700380F00380F00380FFFF80F00000F00000F0000070000070000038 00801800800C010007060001F80011147F9314>I<007C00C6018F038F07060700070007 000700070007000700FFF007000700070007000700070007000700070007000700070007 00070007000700070007007FF01020809F0E>I<0000E003E3300E3C301C1C30380E0078 0F00780F00780F00780F00780F00380E001C1C001E380033E00020000020000030000030 00003FFE001FFF800FFFC03001E0600070C00030C00030C00030C000306000603000C01C 038003FC00141F7F9417>I<0E0000FE00000E00000E00000E00000E00000E00000E0000 0E00000E00000E00000E00000E3E000E43000E81800F01C00F01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC 16207F9F19>I<1C001E003E001E001C000000000000000000000000000E007E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A1F80 9E0C>I<00E001F001F001F000E0000000000000000000000000007007F000F000700070 007000700070007000700070007000700070007000700070007000700070007000700070 00706070F060F0C061803F000C28829E0E>I<0E0000FE00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E0FF00E03C00E03000E02000E04000E08 000E10000E30000E70000EF8000F38000E1C000E1E000E0E000E07000E07800E03800E03 C00E03E0FFCFF815207F9F18>I<0E00FE000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E00FFE00B20809F0C>I<0E1F01F000FE618618000E81C81C000F00F00E000F00F0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E00 FFE7FE7FE023147F9326>I<0E3E00FE43000E81800F01C00F01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC 16147F9319>I<01F800070E001C03803801C03801C07000E07000E0F000F0F000F0F000 F0F000F0F000F0F000F07000E07000E03801C03801C01C0380070E0001F80014147F9317 >I<0E3E00FEC3800F01C00F00E00E00E00E00F00E00700E00780E00780E00780E00780E 00780E00780E00700E00F00E00E00F01E00F01C00EC3000E3E000E00000E00000E00000E 00000E00000E00000E00000E0000FFE000151D7F9319>I<03E0800619801C05803C0780 380380780380700380F00380F00380F00380F00380F00380F00380700380780380380380 3807801C0B800E138003E380000380000380000380000380000380000380000380000380 003FF8151D7E9318>I<0E78FE8C0F1E0F1E0F0C0E000E000E000E000E000E000E000E00 0E000E000E000E000E000E00FFE00F147F9312>I<1F9030704030C010C010C010E00078 007F803FE00FF00070803880188018C018C018E030D0608F800D147E9312>I<02000200 0200060006000E000E003E00FFF80E000E000E000E000E000E000E000E000E000E000E00 0E080E080E080E080E080610031001E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 03C00603C0030DC001F1FC16147F9319>III<7FC3FC0F01E00701 C007018003810001C20000E40000EC00007800003800003C00007C00004E000087000107 000303800201C00601E01E01E0FF07FE1714809318>II<3FFF380E200E201C40384078407000E001E001C00380078007010E 011E011C0338027006700EFFFE10147F9314>II<30307878F87C78 7830300E057C9E17>127 D E /Ft 70 123 df<000FF07F00007FF9FF8000F83FC7C001 E07F8FC003E07F0FC007C07F0FC007C03F078007C01F000007C01F000007C01F000007C0 1F000007C01F0000FFFFFFF800FFFFFFF80007C01F000007C01F000007C01F000007C01F 000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F00 0007C01F000007C01F000007C01F000007C01F000007C01F00003FF8FFF0003FF8FFF000 2220809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C0007C07C0007C07C00 07C0380007C0000007C0000007C0000007C1FE00FFFFFE00FFFFFE0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>I<00 0FFE00007FFE0000F87E0001E0FE0003E0FE0007C07E0007C07E0007C03E0007C03E0007 C03E0007C03E0007C03E00FFFFFE00FFFFFE0007C03E0007C03E0007C03E0007C03E0007 C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007 C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>I<0018007000E001C00380 038007000E000E001E001C003C003C007800780078007800F800F000F000F000F000F000 F000F000F000F000F80078007800780078003C003C001C001E000E000E00070003800380 01C000E0007000180D2D7DA114>40 DI<00E00001F00000E00000E000E0E0E0F0E1E0FCE7E0FE4FE01F5F0003F80003 F8001F5F00FE4FE0FCE7E0F0E1E0E0E0E000E00000E00001F00000E00013147DA11A>I< 387CFEFFFF7F3B030306060E0C18702008107C860F>44 DI<387CFEFEFE7C3807077C860F>I<0000600000E00000E00001C00001C00003800003 80000380000700000700000700000E00000E00001C00001C00001C000038000038000038 0000700000700000E00000E00000E00001C00001C0000380000380000380000700000700 000700000E00000E00001C00001C00001C0000380000380000380000700000700000E000 00E00000C00000132D7DA11A>I<01FC0007FF001F07C01E03C03E03E07C01F07C01F07C 01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC 01F8FC01F87C01F07C01F07C01F03E03E01E03C01F8FC007FF0001FC00151D7E9C1A>I< 00E00001E0000FE000FFE000F3E00003E00003E00003E00003E00003E00003E00003E000 03E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E000 03E00003E00003E000FFFF80FFFF80111D7C9C1A>I<07F0001FFE00383F007C1F80FE0F C0FE0FC0FE0FE0FE07E07C07E03807E0000FE0000FC0000FC0001F80001F00003E000078 0000F00000E00001C0000380600700600E00601C00E01FFFC03FFFC07FFFC0FFFFC0FFFF C0131D7D9C1A>I<01FC0007FF000E0F801E0FC03F07E03F07E03F07E03F07E01E0FC000 0FC0000F80001F0001FC0001FC00000F800007C00003E00003F00003F83803F87C03F8FE 03F8FE03F8FE03F0FC03F07807E03C0FC01FFF8003FC00151D7E9C1A>I<0001C00003C0 0007C00007C0000FC0001FC0003BC00073C00063C000C3C00183C00383C00703C00E03C0 0C03C01803C03803C07003C0E003C0FFFFFEFFFFFE0007C00007C00007C00007C00007C0 0007C000FFFE00FFFE171D7F9C1A>I<3803803FFF803FFF003FFE003FFC003FF0003F80 0030000030000030000030000033F80037FE003C1F00380F801007C00007C00007E00007 E07807E0FC07E0FC07E0FC07E0FC07C0780FC0600F80381F001FFC0007F000131D7D9C1A >I<003F0001FFC007E0E00F81E01F03F01E03F03E03F07C03F07C01E07C0000FC1000FC FF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8FC01F8FC01F87C01F87C01F87C01F83C 01F03E01F01E03E00F07C007FF8001FE00151D7E9C1A>I<6000007FFFF87FFFF87FFFF0 7FFFE07FFFC0E00180C00300C00300C00600000C00001800003800003800007800007000 00F00000F00001F00001F00001F00001F00003F00003F00003F00003F00003F00003F000 01E00000C000151E7D9D1A>I<01FC0007FF000F07801E03C01C01E03C01E03C01E03E01 E03F01E03FC3C01FE3801FFF000FFE0007FF8007FFC01FFFE03C3FF0780FF07803F8F001 F8F000F8F00078F00078F000707800707C00E03E03C00FFF8003FC00151D7E9C1A>I<01 FC000FFF001F07803E03C07C03E07C01E0FC01F0FC01F0FC01F0FC01F8FC01F8FC01F8FC 01F87C03F87C03F83E05F81FFDF807F9F80041F80001F03C01F07E01F07E03E07E03E07E 07C03C0780381F001FFC0007F000151D7E9C1A>I<0000E000000000E000000001F00000 0001F000000001F000000003F800000003F800000006FC00000006FC0000000EFE000000 0C7E0000000C7E000000183F000000183F000000303F800000301F800000701FC0000060 0FC00000600FC00000C007E00000FFFFE00001FFFFF000018003F000018003F000030001 F800030001F800060001FC00060000FC000E0000FE00FFE00FFFE0FFE00FFFE0231F7E9E 28>65 DI<0007FC02003FFF0E00 FE03DE03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E0000067E 000006FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E0000007E 0000067F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000FE01C000 3FFF800007FC001F1F7D9E26>IIII<0007FC0200003FFF0E0000FE03DE0003F000FE0007E0003E000FC0001E00 1F80001E003F00000E003F00000E007F000006007E000006007E00000600FE00000000FE 00000000FE00000000FE00000000FE00000000FE003FFFE0FE003FFFE07E00007E007E00 007E007F00007E003F00007E003F00007E001F80007E000FC0007E0007E0007E0003F000 FE0000FE01FE00003FFF8E000007FC0600231F7D9E29>III75 DIII<001FF80000FFFF0001F81F8007E007E00FC003F01F8001F81F0000F83F0000FC 7F0000FE7E00007E7E00007EFE00007FFE00007FFE00007FFE00007FFE00007FFE00007F FE00007FFE00007FFE00007F7E00007E7F0000FE7F0000FE3F0000FC3F8001FC1F8001F8 0FC003F007E007E001F81F8000FFFF00001FF800201F7D9E27>II<001FF80000FFFF0001F81F8007E007E00FC003F01F80 01F81F8001F83F0000FC7F0000FE7F0000FE7E00007EFE00007FFE00007FFE00007FFE00 007FFE00007FFE00007FFE00007FFE00007FFE00007F7E00007E7E00007E7F0000FE3F00 00FC3F87C1FC1F8FE1F80FD833F007F81FE001F81F8000FFFF00001FFE0300000E030000 0F0700000FFF000007FF000007FE000007FE000003FC000001F8000000F020287D9E27> II<03FC080FFF381E03F83800F87000787000 38F00038F00018F00018F80000FC00007FC0007FFE003FFF801FFFE00FFFF007FFF000FF F80007F80000FC00007C00003CC0003CC0003CC0003CE00038E00078F80070FE01E0E7FF C081FF00161F7D9E1D>I<7FFFFFFC7FFFFFFC7C07E07C7007E01C6007E00C6007E00CE0 07E00EC007E006C007E006C007E006C007E0060007E0000007E0000007E0000007E00000 07E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00000 07E0000007E0000007E0000007E00003FFFFC003FFFFC01F1E7E9D24>IIIIII<3FFFFF803FFFFF803F803F003E007F0038007E003800FC007001FC007001F8006003 F0006007F0006007E000000FC000001FC000001F8000003F0000007F0000007E000000FC 000001FC018001F8018003F0018007F0018007E003800FC003801FC003001F8007003F00 0F007F001F007E007F00FFFFFF00FFFFFF00191F7D9E20>I<07FC001FFF003F0F803F07 C03F03E03F03E00C03E00003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F803 E0FC05E07E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007FF801F0FC03E0FC03E0FC07C0FC07C03 00FC0000FC0000FC0000FC0000FC0000FC00007C00007E00003E00603F00C01F81C007FF 0001FC0013147E9317>I<0007F80007F80000F80000F80000F80000F80000F80000F800 00F80000F80000F80000F801F8F80FFEF81F83F83E01F87E00F87C00F87C00F8FC00F8FC 00F8FC00F8FC00F8FC00F8FC00F87C00F87C00F87E00F83E01F81F07F80FFEFF03F8FF18 207E9F1D>I<01FE0007FF800F83C01E01E03E00F07C00F07C00F8FC00F8FFFFF8FFFFF8 FC0000FC0000FC00007C00007C00003E00181E00180F807007FFE000FF8015147F9318> I<001F8000FFC001F3E003E7E003C7E007C7E007C3C007C00007C00007C00007C00007C0 00FFFC00FFFC0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C0003FFC003FFC0013207F9F10>I<01FC3C07 FFFE0F079E1E03DE3E03E03E03E03E03E03E03E03E03E01E03C00F07800FFF0009FC0018 00001800001C00001FFF800FFFF007FFF81FFFFC3C007C70003EF0001EF0001EF0001E78 003C78003C3F01F80FFFE001FF00171E7F931A>II<1C003E003F007F003F003E001C00000000000000 000000000000FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F00 1F001F001F00FFE0FFE00B217EA00E>I107 DIII<01FF0007FFC01F83F03E00F83E00F87C007C7C007CFC007EFC007EFC00 7EFC007EFC007EFC007E7C007C7C007C3E00F83E00F81F83F007FFC001FF0017147F931A >II<01F81807FE381F87783F01F8 3E01F87E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87E00F87E00F8 3F01F81F87F80FFEF803F8F80000F80000F80000F80000F80000F80000F80000F80007FF 0007FF181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07FF83FFC1FFE03FE001FC007C007E0 07F006F81EFFFCC7F010147E9315>I<01800180018003800380038007800F803F80FFFC FFFC0F800F800F800F800F800F800F800F800F800F800F860F860F860F860F8607CC03F8 01F00F1D7F9C14>IIIIII<3FFFE03FFFE03C07C0380F80701F80603F00603E00607C00 00F80001F80003F00003E06007C0600F80601F80E03F00C03E01C07C03C0FFFFC0FFFFC0 13147F9317>I E /Fu 37 122 df<00000007FF800000000001FFFFF0000000000FFFFF FC000000003FFFFFFE00000000FFFC00FF00000001FFC0003F80000007FF00007FC00000 0FFE0001FFC000001FFC0001FFE000001FF80003FFE000003FF00003FFE000003FF00003 FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE00000 7FE00001FFC000007FE00000FF8000007FE000003E0000007FE00000000000007FE00000 000000007FE00000000000007FE00000000000007FE00000000000007FE0000000000000 7FE0003FFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFF FFF000FFFFFFFFFFFFF000007FF00000FFF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF00000 7FF000007FF000007FF0003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03F FFFFE03FFFFFE03FFFFFE03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC007F FE00FFFE00FFFF00FFFF00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E78000 0780000780000F00000F00000F00000F00001E00001E00003C00003C0000780000F80000 F00001F00003E00007C0000F80001F00001E00000C00001124788F21>44 D<0000000003E000000000000000000007F000000000000000000007F000000000000000 00000FF80000000000000000000FF80000000000000000000FF80000000000000000001F FC0000000000000000001FFC0000000000000000003FFE0000000000000000003FFE0000 000000000000007FFF0000000000000000007FFF0000000000000000007FFF0000000000 00000000FFFF800000000000000000FFFF800000000000000001FFFFC000000000000000 01FFFFC00000000000000001FFFFC00000000000000003FFFFE00000000000000003EFFF E00000000000000007EFFFF00000000000000007CFFFF00000000000000007C7FFF00000 00000000000FC7FFF8000000000000000F83FFF8000000000000001F83FFFC0000000000 00001F03FFFC000000000000001F01FFFC000000000000003F01FFFE000000000000003E 00FFFE000000000000007E00FFFF000000000000007C007FFF00000000000000FC007FFF 80000000000000F8007FFF80000000000000F8003FFF80000000000001F8003FFFC00000 00000001F0001FFFC0000000000003F0001FFFE0000000000003E0001FFFE00000000000 03E0000FFFE0000000000007E0000FFFF0000000000007C00007FFF000000000000FC000 07FFF800000000000F800007FFF800000000000F800003FFF800000000001F800003FFFC 00000000001F000001FFFC00000000003FFFFFFFFFFE00000000003FFFFFFFFFFE000000 00007FFFFFFFFFFF00000000007FFFFFFFFFFF00000000007FFFFFFFFFFF0000000000FC 0000007FFF8000000000F80000003FFF8000000001F80000003FFFC000000001F0000000 3FFFC000000001F00000001FFFC000000003F00000001FFFE000000003E00000000FFFE0 00000007E00000000FFFF000000007C00000000FFFF000000007C000000007FFF0000000 0FC000000007FFF80000000F8000000003FFF80000001F8000000003FFFC0000001F0000 000003FFFC0000007FC000000001FFFC0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FF FFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF 8051487CC75A>65 DI<000000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E0000 03FFFFFFFE00FE00000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001 FFFC000003FFFE0007FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003F FF000000001FFE007FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FF F00000000003FE03FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFF C00000000000FE0FFFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF 8000000000007E3FFF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF 0000000000003E7FFF000000000000007FFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF000000000000007FFF 000000000000007FFF000000000000007FFF000000000000007FFF8000000000003E3FFF 8000000000003E3FFF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFF C000000000003E0FFFC000000000007C0FFFC000000000007C07FFE000000000007C07FF E00000000000F803FFF00000000000F803FFF00000000001F801FFF80000000001F000FF FC0000000003E0007FFE0000000007E0003FFF000000000FC0001FFF800000001F80000F FFC00000003F000007FFF0000000FE000001FFFC000001FC000000FFFF80000FF8000000 3FFFF8007FF00000000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000 000FFFFFE00000000000003FFE000000474979C756>I69 D<000000003FFE00000E000000000FFFFFC0001E00000000 7FFFFFF8003E00000003FFFFFFFE00FE0000000FFFFFFFFF81FE0000003FFFF800FFC3FE 000000FFFF80000FF7FE000001FFFC000003FFFE000007FFF0000001FFFE00000FFFC000 00007FFE00001FFF800000003FFE00003FFF000000001FFE00007FFE000000000FFE0000 FFFC0000000007FE0001FFF80000000007FE0003FFF00000000003FE0003FFF000000000 01FE0007FFE00000000001FE0007FFE00000000000FE000FFFC00000000000FE000FFFC0 00000000007E001FFFC000000000007E001FFF8000000000007E003FFF8000000000007E 003FFF8000000000003E003FFF8000000000003E007FFF8000000000003E007FFF000000 0000003E007FFF00000000000000007FFF0000000000000000FFFF0000000000000000FF FF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000 000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF00 00000000000000FFFF0000000000000000FFFF0000000000000000FFFF00000000000000 00FFFF0000007FFFFFFFFE7FFF0000007FFFFFFFFE7FFF0000007FFFFFFFFE7FFF800000 7FFFFFFFFE7FFF8000007FFFFFFFFE3FFF8000000000FFFE003FFF8000000000FFFE003F FF8000000000FFFE001FFF8000000000FFFE001FFFC000000000FFFE000FFFC000000000 FFFE000FFFC000000000FFFE0007FFE000000000FFFE0007FFE000000000FFFE0003FFF0 00000000FFFE0003FFF800000000FFFE0001FFF800000000FFFE0000FFFC00000000FFFE 00007FFE00000000FFFE00003FFF00000000FFFE00001FFF80000000FFFE00000FFFE000 0001FFFE000007FFF0000003FFFE000001FFFC000007FFFE000000FFFF80001FFFFE0000 003FFFFC00FFC7FE0000000FFFFFFFFF83FE00000003FFFFFFFE00FE000000007FFFFFF8 003E000000000FFFFFE0000E00000000003FFE000000004F4979C75D>71 D73 D76 DI80 D82 D<00007FF00007000007FFFF000F00001FFFFFC01F00007FFFFFF03F 0000FFFFFFFC7F0003FFC00FFEFF0007FE0000FFFF0007FC00003FFF000FF000001FFF00 1FF0000007FF001FE0000003FF003FE0000001FF003FC0000001FF007FC0000000FF007F C00000007F007FC00000007F00FFC00000003F00FFC00000003F00FFC00000003F00FFE0 0000003F00FFE00000001F00FFF00000001F00FFF80000001F00FFFC0000001F00FFFF00 000000007FFFC0000000007FFFFE000000007FFFFFE00000003FFFFFFF0000003FFFFFFF F000001FFFFFFFFC00001FFFFFFFFF00000FFFFFFFFFC00007FFFFFFFFE00007FFFFFFFF F00003FFFFFFFFF80000FFFFFFFFFC00007FFFFFFFFE00003FFFFFFFFE00000FFFFFFFFF 000001FFFFFFFF0000003FFFFFFF80000001FFFFFF800000000FFFFFC000000000FFFFC0 000000001FFFC0000000000FFFE00000000007FFE00000000003FFE07800000001FFE0F8 00000000FFE0F800000000FFE0F8000000007FE0F8000000007FE0F8000000007FE0FC00 0000007FE0FC000000007FC0FC000000007FC0FE000000007FC0FE000000007F80FF0000 0000FF80FF80000000FF80FFC0000000FF00FFE0000001FE00FFF8000003FE00FFFE0000 07FC00FFFF80000FF800FFFFFC007FF000FE3FFFFFFFE000FC0FFFFFFF8000F803FFFFFF 0000F0007FFFF80000E00003FFC00000334979C742>I<3FFFFFFFFFFFFFFFFF003FFFFF FFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF 003FFF0003FFF8003FFF007FF80003FFF80007FF807FE00003FFF80001FF807FC00003FF F80000FF807F800003FFF800007F807F000003FFF800003F807F000003FFF800003F807E 000003FFF800001F807E000003FFF800001F807E000003FFF800000F807C000003FFF800 000F807C000003FFF800000F807C000003FFF800000F807C000003FFF800000F80FC0000 03FFF800000FC0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007 C0F8000003FFF8000007C0F8000003FFF8000007C000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF8000000000003FFFFFFFFFFF8000000 03FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFF F800004A467CC553>I<0007FFFC000000007FFFFFC0000001FFFFFFF8000003FFFFFFFE 000007FE001FFF000007FF0003FFC0000FFF8001FFE0000FFF8000FFF0000FFF80007FF0 000FFF80007FF8000FFF80007FF80007FF00003FFC0007FF00003FFC0003FE00003FFC00 00F800003FFC00000000003FFC00000000003FFC00000000003FFC00000000003FFC0000 0007FFFFFC000000FFFFFFFC000007FFFFFFFC00003FFFE03FFC0000FFFE003FFC0003FF F0003FFC0007FFC0003FFC000FFF00003FFC001FFE00003FFC003FFC00003FFC007FF800 003FFC007FF800003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF00000 3FFC00FFF000003FFC00FFF000007FFC007FF80000FFFC007FF80001EFFC003FFC0003EF FC003FFF0007CFFF000FFFC03F8FFFF807FFFFFF07FFFC01FFFFFC03FFFC007FFFF001FF FC0003FF80007FF8362E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC000 00000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC00000000000 01FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000 0000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000000000 01FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000 0000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000000000 01FFC0000000000001FFC00FFC00000001FFC07FFFC0000001FFC3FFFFF0000001FFCFFF FFFC000001FFDFF00FFF000001FFFF8003FF800001FFFE0001FFC00001FFF800007FE000 01FFF000007FF00001FFE000003FF80001FFE000001FFC0001FFE000001FFC0001FFE000 001FFE0001FFE000001FFE0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF00 01FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000 000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF80 01FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000000FFF0001FFE000 000FFF0001FFE000001FFE0001FFE000001FFE0001FFE000001FFC0001FFE000001FFC00 01FFE000003FF80001FFF000003FF80001FFF800007FF00001FFFC0000FFE00001FFFE00 01FFC00001FFBF0007FF800001FF1FE01FFE000001FE0FFFFFFC000001FC03FFFFF00000 01F800FFFF80000001F0001FF800000039487CC742>I<00001FFFC0000000FFFFF80000 07FFFFFE00001FFFFFFF80007FFC00FFC000FFE001FFC001FFC003FFE003FF8003FFE007 FF0003FFE00FFE0003FFE00FFE0003FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803F FC00003E007FF8000000007FF8000000007FF800000000FFF800000000FFF800000000FF F800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000FF F800000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC000000003F FC000000001FFC000000F81FFE000000F80FFE000000F80FFF000001F007FF800003F003 FFC00007E001FFE0000FC000FFF0001F80007FFE00FF00001FFFFFFE000007FFFFF80000 00FFFFE00000001FFE00002D2E7CAD35>I<00000000007FC00000000000FFFFC0000000 0000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FF C0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000 000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FF C0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000 000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FF C0000000000001FFC00000000FFC01FFC0000000FFFF81FFC0000007FFFFE1FFC000001F FFFFF9FFC000007FFC03FFFFC00000FFF0007FFFC00001FFC0001FFFC00003FF80000FFF C00007FF000007FFC0000FFE000003FFC0000FFE000003FFC0001FFC000003FFC0001FFC 000003FFC0003FFC000003FFC0003FFC000003FFC0007FF8000003FFC0007FF8000003FF C0007FF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8 000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FF C000FFF8000003FFC000FFF8000003FFC0007FF8000003FFC0007FF8000003FFC0007FF8 000003FFC0003FF8000003FFC0003FFC000003FFC0003FFC000003FFC0001FFC000003FF C0001FFC000003FFC0000FFE000007FFC00007FF00000FFFC00003FF00001FFFC00001FF C0003FFFC00000FFE000FFFFE000007FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FF FF800001FFFF03FFFF8000001FF803FFFF8039487CC742>I<00001FFE00000001FFFFE0 000007FFFFF800001FFFFFFE00007FFC07FF0000FFE001FF8001FFC0007FC003FF80003F E007FF00003FF00FFE00001FF01FFE00000FF81FFC00000FF83FFC00000FFC3FFC000007 FC7FFC000007FC7FF8000007FC7FF8000007FE7FF8000007FEFFF8000007FEFFF8000007 FEFFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFCFFF800000000FFF8000000 00FFF800000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC000000 003FFC000000003FFC0000001C1FFE0000003E0FFE0000003E07FF0000007E07FF000000 FC03FF800001F801FFC00003F0007FF0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE 000000FFFFF80000000FFF80002F2E7DAD36>I<00000000001F8000007FF000FFE00007 FFFF03FFF0001FFFFFC7FFF0007FFFFFFFC7F800FFE03FFE0FF801FF800FFC0FF803FF00 07FE0FF807FE0003FF07F007FE0003FF07F00FFC0001FF81C00FFC0001FF80000FFC0001 FF80001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FF C0001FFC0001FFC0000FFC0001FF80000FFC0001FF80000FFC0001FF800007FE0003FF00 0007FE0003FF000003FF0007FE000001FF800FFC000000FFE03FF8000001FFFFFFF00000 01DFFFFFC0000003C7FFFF00000003C07FF000000007C0000000000007C0000000000007 C0000000000007C0000000000007E0000000000007F0000000000007F8000000000007FF FFFFF0000007FFFFFFFF000003FFFFFFFFE00003FFFFFFFFF80001FFFFFFFFFE0001FFFF FFFFFF0000FFFFFFFFFF80007FFFFFFFFF8003FFFFFFFFFFC00FFFFFFFFFFFC01FF80000 1FFFE03FE0000001FFE07FC00000007FF07FC00000003FF0FF800000001FF0FF80000000 1FF0FF800000001FF0FF800000001FF0FF800000001FF07FC00000003FE07FC00000003F E03FE00000007FC03FF0000000FFC01FFC000003FF800FFF00000FFF0003FFF000FFFC00 00FFFFFFFFF000003FFFFFFFC0000007FFFFFE000000003FFFC0000035447DAE3B>103 D<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0 0000000000FFFFC0000000000003FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC001FFC000 0001FFC00FFFF8000001FFC03FFFFE000001FFC0FFFFFF000001FFC1FC07FF800001FFC3 E003FFC00001FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE0 0001FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF00001FFF0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFF C07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFF E03B487CC742>I<00FC0001FF0003FF8007FFC00FFFC01FFFE01FFFE01FFFE01FFFE01F FFE01FFFE00FFFC007FFC003FF8001FF0000FC0000000000000000000000000000000000 0000000000000000000000000000000000007FC0FFFFC0FFFFC0FFFFC0FFFFC0FFFFC003 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0FFFFFFFF FFFFFFFFFFFFFFFFFFFFFF18497CC820>I<007FC000000000FFFFC000000000FFFFC000 000000FFFFC000000000FFFFC000000000FFFFC00000000003FFC00000000001FFC00000 000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC0000000 0001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000000000 01FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001 FFC00000000001FFC00000000001FFC00000000001FFC0007FFFF801FFC0007FFFF801FF C0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0000FFE0001FFC00007F00001FFC0 000FE00001FFC0003FC00001FFC0007F800001FFC000FE000001FFC001FC000001FFC007 F8000001FFC00FF0000001FFC01FC0000001FFC03F80000001FFC0FF00000001FFC1FE00 000001FFC3FF00000001FFCFFF80000001FFDFFFC0000001FFFFFFC0000001FFFFFFE000 0001FFFFFFF0000001FFFCFFF8000001FFF87FFC000001FFE03FFC000001FFC01FFE0000 01FFC01FFF000001FFC00FFF800001FFC007FFC00001FFC003FFC00001FFC001FFE00001 FFC001FFF00001FFC000FFF80001FFC0007FFC0001FFC0003FFC0001FFC0001FFE0001FF C0000FFF0001FFC0000FFF8001FFC0000FFFC0FFFFFF807FFFFFFFFFFF807FFFFFFFFFFF 807FFFFFFFFFFF807FFFFFFFFFFF807FFFFF38487CC73F>107 D<007FC000FFFFC000FF FFC000FFFFC000FFFFC000FFFFC00003FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC000FFFFFF80FFFFFF80FFFFFF80FFFFFF80FFFFFF8019487CC720>I<007F C001FFC00000FFE00000FFFFC00FFFF80007FFFC0000FFFFC03FFFFE001FFFFF0000FFFF C0FFFFFF007FFFFF8000FFFFC1FC07FF80FE03FFC000FFFFC3E003FFC1F001FFE00003FF C7C001FFC3E000FFE00001FFCF0001FFE78000FFF00001FFDE0000FFEF00007FF00001FF DC0000FFEE00007FF00001FFFC0000FFFE00007FF80001FFF80000FFFC00007FF80001FF F00000FFF800007FF80001FFF00000FFF800007FF80001FFF00000FFF800007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FF E00000FFF000007FF80001FFE00000FFF000007FF800FFFFFFC07FFFFFE03FFFFFF0FFFF FFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFF FFC07FFFFFE03FFFFFF05C2E7CAD63>I<007FC001FFC00000FFFFC00FFFF80000FFFFC0 3FFFFE0000FFFFC0FFFFFF0000FFFFC1FC07FF8000FFFFC3E003FFC00003FFC7C001FFC0 0001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001FFFC0000FFF00001FFF8 0000FFF00001FFF00000FFF00001FFF00000FFF00001FFF00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFF E0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B2E7CAD42>I<00000FFF 0000000000FFFFF000000007FFFFFE0000001FFFFFFF8000003FFC03FFC00000FFE0007F F00001FF80001FF80003FF00000FFC0007FE000007FE000FFE000007FF000FFC000003FF 001FFC000003FF803FFC000003FFC03FF8000001FFC03FF8000001FFC07FF8000001FFE0 7FF8000001FFE07FF8000001FFE0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FF F8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8 000001FFF0FFF8000001FFF07FF8000001FFE07FF8000001FFE07FF8000001FFE07FF800 0001FFE03FFC000003FFC03FFC000003FFC01FFC000003FF801FFE000007FF800FFE0000 07FF0007FF00000FFE0003FF80001FFC0001FFC0003FF80000FFE0007FF000007FFC03FF E000001FFFFFFF80000007FFFFFE00000000FFFFF0000000000FFF000000342E7DAD3B> I<007FC00FFC000000FFFFC07FFFC00000FFFFC3FFFFF00000FFFFCFFFFFFC0000FFFFDF F01FFF0000FFFFFF8007FF800003FFFE0001FFC00001FFF80000FFE00001FFF00000FFF0 0001FFE000007FF80001FFE000003FFC0001FFE000003FFC0001FFE000003FFE0001FFE0 00001FFE0001FFE000001FFF0001FFE000001FFF0001FFE000001FFF0001FFE000000FFF 0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE0 00000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF 8001FFE000000FFF8001FFE000000FFF0001FFE000001FFF0001FFE000001FFF0001FFE0 00001FFE0001FFE000001FFE0001FFE000003FFC0001FFE000003FFC0001FFE000007FF8 0001FFF000007FF80001FFF80000FFF00001FFFC0001FFE00001FFFE0003FFC00001FFFF 0007FF800001FFFFE03FFE000001FFEFFFFFFC000001FFE3FFFFF0000001FFE0FFFF8000 0001FFE01FF800000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0 000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE000000000 0001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0 000000000001FFE0000000000001FFE00000000000FFFFFFC000000000FFFFFFC0000000 00FFFFFFC000000000FFFFFFC000000000FFFFFFC00000000039427CAD42>I<00000FFC 0003C0000000FFFF0007C0000007FFFFC00FC000001FFFFFF01FC000003FFE03F81FC000 00FFF000FC3FC00001FFE0003E7FC00003FFC0001F7FC00007FF80001FFFC0000FFF0000 0FFFC0000FFF000007FFC0001FFE000007FFC0001FFE000003FFC0003FFC000003FFC000 3FFC000001FFC0007FFC000001FFC0007FFC000001FFC0007FF8000001FFC000FFF80000 01FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000 FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF8000001FFC000FFF80000 01FFC0007FF8000001FFC0007FFC000001FFC0007FFC000001FFC0003FFC000001FFC000 3FFC000001FFC0003FFE000003FFC0001FFE000003FFC0001FFE000007FFC0000FFF0000 0FFFC00007FF80001FFFC00003FF80003FFFC00001FFC0007FFFC00000FFF000FFFFC000 007FFC07FBFFC000001FFFFFE3FFC0000007FFFFC3FFC0000001FFFF03FFC00000001FF8 03FFC0000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC000 0000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC00000000000 03FFC0000000000003FFC0000000000003FFC0000000000003FFC0000000000003FFC000 0000000003FFC0000000000003FFC00000000001FFFFFF8000000001FFFFFF8000000001 FFFFFF8000000001FFFFFF8000000001FFFFFF8039427CAD3F>I<00FF803F8000FFFF80 FFF000FFFF83FFFC00FFFF87FFFE00FFFF8FC3FF00FFFF8F07FF0003FF9E0FFF8001FFBC 0FFF8001FFB80FFF8001FFF80FFF8001FFF00FFF8001FFF007FF0001FFF007FF0001FFE0 03FE0001FFE000F80001FFE000000001FFE000000001FFC000000001FFC000000001FFC0 00000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0 00000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0 00000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0 00000001FFC000000001FFC000000001FFC0000000FFFFFFE00000FFFFFFE00000FFFFFF E00000FFFFFFE00000FFFFFFE00000292E7CAD31>I<000FFF00E0007FFFF3E001FFFFFF E007FFFFFFE00FF800FFE01FC0001FE03F80000FE03F000007E07F000003E07F000003E0 FF000003E0FF000003E0FF800003E0FFC0000000FFF0000000FFFE000000FFFFF800007F FFFFC0007FFFFFF0003FFFFFFC001FFFFFFF000FFFFFFF8007FFFFFFC003FFFFFFE000FF FFFFF0003FFFFFF00003FFFFF800001FFFF8000000FFFC0000001FFC7800000FFCF80000 07FCF8000003FCFC000003FCFC000003FCFE000003F8FE000003F8FF000003F8FF800007 F0FFC0000FF0FFF0001FE0FFFC00FFC0FFFFFFFF80FC7FFFFE00F81FFFF800E003FF8000 262E7CAD2F>I<0001F000000001F000000001F000000001F000000001F000000001F000 000003F000000003F000000003F000000007F000000007F000000007F00000000FF00000 000FF00000001FF00000003FF00000003FF00000007FF0000001FFF0000003FFF000000F FFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC000FFF0000000FFF0000000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0 000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF000 0000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF001F0 00FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000 FFF001F0007FF001E0007FF803E0003FF803E0003FFC07C0001FFE0F80000FFFFF800007 FFFE000001FFFC0000001FF00024427EC12E>I<007FE000003FF000FFFFE0007FFFF000 FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF00003FFE000 01FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000 01FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE000 00FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000 01FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE000 00FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000 01FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE000 01FFF00001FFE00001FFF00001FFE00001FFF00001FFE00003FFF00000FFE00007FFF000 00FFE0000F7FF000007FE0001F7FF000007FF0003E7FF800003FFC00FC7FFFE0001FFFFF F87FFFE00007FFFFE07FFFE00001FFFF807FFFE000003FFE007FFFE03B2E7CAD42>III<7FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF007F FFFF801FFFFF00007FF80001FF0000007FFC0001FC0000003FFE0003F80000001FFF0007 F00000000FFF0007E000000007FF800FC000000003FFC01F8000000003FFE03F80000000 01FFF07F0000000000FFF8FE00000000007FF9FC00000000003FFFF800000000003FFFF0 00000000001FFFE000000000000FFFC0000000000007FFC0000000000003FFC000000000 0001FFE0000000000001FFF0000000000001FFF8000000000003FFFC000000000003FFFE 000000000007FFFE00000000000FEFFF00000000001FCFFF80000000003F87FFC0000000 007F03FFE000000000FE01FFE000000001FC00FFF000000001F8007FF800000003F0007F FC00000007F0003FFE0000000FE0001FFF0000001FC0000FFF0000003F800007FF800000 FF800007FFC000FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003F FFFFC0FFFFF8003FFFFFC03A2E7EAD3F>I<7FFFFFC000FFFF807FFFFFC000FFFF807FFF FFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF8000FFF000000FE00000FFF800000F C00000FFF800000FC000007FFC00000F8000007FFC00001F8000003FFC00001F0000003F FE00003F0000001FFE00003E0000001FFF00007E0000000FFF00007C0000000FFF8000FC 00000007FF8000F800000007FFC001F800000003FFC001F000000003FFE003F000000003 FFE003F000000001FFF003E000000001FFF007E000000000FFF007C000000000FFF80FC0 000000007FF80F80000000007FFC1F80000000003FFC1F00000000003FFE3F0000000000 1FFE3E00000000001FFF7E00000000000FFF7C00000000000FFFFC00000000000FFFFC00 0000000007FFF8000000000007FFF8000000000003FFF0000000000003FFF00000000000 01FFE0000000000001FFE0000000000000FFC0000000000000FFC00000000000007F8000 00000000007F800000000000003F000000000000003F000000000000003F000000000000 003E000000000000007E000000000000007C00000000000000FC000000001F8000F80000 00003FC001F8000000007FE001F000000000FFF003F000000000FFF003E000000000FFF0 07E000000000FFF00FC000000000FFF01F8000000000FFF03F80000000007FE07F000000 00007F43FE00000000003FFFF800000000001FFFF0000000000007FFC0000000000001FE 00000000000039427EAD3F>I E /Fv 48 122 df<0000C018000000C018000000C01800 000180300000018030000001803000000180300000030060000003006000000300600000 030060000003006000000600C000000600C000000600C000000600C000000C018000FFFF FFFFC0FFFFFFFFC000180300000018030000001803000000180300000030060000003006 000000300600000030060000FFFFFFFFC0FFFFFFFFC000600C000000C018000000C01800 0000C018000000C018000001803000000180300000018030000001803000000300600000 0300600000030060000003006000000600C000000600C000000600C00000222D7DA229> 35 D<70F8FCFC7404040404080810102040060F7C840E>44 DI< 70F8F8F87005057C840E>I<01F000071C000C06001803003803803803807001C07001C0 7001C07001C0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0 F001E0F001E0F001E0F001E07001C07001C07001C07803C03803803803801C07000C0600 071C0001F00013227EA018>48 D<008003800F80F3800380038003800380038003800380 038003800380038003800380038003800380038003800380038003800380038003800380 0380038007C0FFFE0F217CA018>I<03F0000C1C001007002007804003C04003C08003E0 F003E0F801E0F801E0F801E02003E00003E00003C00003C0000780000700000E00001C00 00180000300000600000C0000180000100000200200400200800201800603000403FFFC0 7FFFC0FFFFC013217EA018>I<03F8000C1E001007002007804007C07807C07803C07807 C03807C0000780000780000700000F00000E0000380003F000001C00000F000007800007 800003C00003C00003E02003E07003E0F803E0F803E0F003C04003C0400780200780100F 000C1C0003F00013227EA018>I<000200000600000E00000E00001E00001E00002E0000 4E00004E00008E00008E00010E00020E00020E00040E00040E00080E00100E00100E0020 0E00200E00400E00800E00FFFFF8000E00000E00000E00000E00000E00000E00000E0000 1F0001FFF015217FA018>I<1000801E07001FFF001FFE001FF80013E000100000100000 10000010000010000010000010F800130E001407001803801003800001C00001C00001E0 0001E00001E00001E07001E0F001E0F001E0E001C08001C04003C0400380200700100600 0C1C0003F00013227EA018>I<007E0001C1000300800601C00E03C01C03C01801803800 00380000780000700000700000F0F800F30C00F40600F40300F80380F801C0F001C0F001 E0F001E0F001E0F001E0F001E07001E07001E07001E03801C03801C01803801C03000C06 00070C0001F00013227EA018>I<01F800060E000803001001802001802000C06000C060 00C06000C07000C07801803E01003F02001FC4000FF80003F80003FC00067F00083F8010 0F803007C06001C06000E0C000E0C00060C00060C00060C000606000406000C030008018 03000E0E0003F00013227EA018>56 D<01F000060C000C06001807003803807003807003 80F001C0F001C0F001C0F001E0F001E0F001E0F001E0F001E07001E07003E03803E01805 E00C05E00619E003E1E00001C00001C00001C0000380000380300300780700780600700C 002018001030000FC00013227EA018>I<0001800000018000000180000003C0000003C0 000003C0000005E0000005E000000DF0000008F0000008F0000010F80000107800001078 0000203C0000203C0000203C0000401E0000401E0000401E0000800F0000800F0000FFFF 000100078001000780030007C0020003C0020003C0040003E0040001E0040001E00C0000 F00C0000F03E0001F8FF800FFF20237EA225>65 D<0007E0100038183000E0063001C001 70038000F0070000F00E0000701E0000701C0000303C0000303C0000307C000010780000 1078000010F8000000F8000000F8000000F8000000F8000000F8000000F8000000F80000 0078000000780000107C0000103C0000103C0000101C0000201E0000200E000040070000 400380008001C0010000E0020000381C000007E0001C247DA223>67 D69 DI< FFFC0FC00780078007800780078007800780078007800780078007800780078007800780 078007800780078007800780078007800780078007800780078007800FC0FFFC0E227EA1 12>73 D<03FFF0001F00000F00000F00000F00000F00000F00000F00000F00000F00000F 00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F 00000F00000F00000F00700F00F80F00F80F00F80E00F01E00401C0020380018700007C0 0014237EA119>I77 DI80 D82 D<03F0200C0C601802603001E07000E0600060E00060E00060E00020E0 0020E00020F00000F000007800007F00003FF0001FFE000FFF0003FF80003FC00007E000 01E00000F00000F0000070800070800070800070800070C00060C00060E000C0F000C0C8 0180C6070081FC0014247DA21B>I<7FFFFFF87807807860078018400780084007800840 078008C007800C8007800480078004800780048007800400078000000780000007800000 078000000780000007800000078000000780000007800000078000000780000007800000 078000000780000007800000078000000780000007800000078000000780000007800000 0FC00003FFFF001E227EA123>I<0FE0001838003C0C003C0E0018070000070000070000 070000FF0007C7001E07003C0700780700700700F00708F00708F00708F00F087817083C 23900FC1E015157E9418>97 D<0E0000FE00001E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E1F000E61C00E80600F00300E00380E00 3C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E0E001C0E003C0E00380F00 700C80600C41C0083F0017237FA21B>I<01FE000703000C07801C078038030078000070 0000F00000F00000F00000F00000F00000F00000F000007000007800403800401C00800C 010007060001F80012157E9416>I<0000E0000FE00001E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E001F8E00704E00C02E01C01E03800E0 7800E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F000E07000E07800E03800E0 1801E00C02E0070CF001F0FE17237EA21B>I<01FC000707000C03801C01C03801C07801 E07000E0F000E0FFFFE0F00000F00000F00000F00000F000007000007800203800201C00 400E008007030000FC0013157F9416>I<003C00C6018F038F030F070007000700070007 000700070007000700FFF807000700070007000700070007000700070007000700070007 000700070007000700070007807FF8102380A20F>I<00007001F198071E180E0E181C07 001C07003C07803C07803C07803C07801C07001C07000E0E000F1C0019F0001000001000 001800001800001FFE000FFFC00FFFE03800F0600030400018C00018C00018C000186000 306000303800E00E038003FE0015217F9518>I<0E0000FE00001E00000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00000E00000E1F800E60C00E80E00F 00700F00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C001E003E001E001C000000 00000000000000000000000000000E00FE001E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E00FFC00A227FA10E>I<01C003E003E003E001 C00000000000000000000000000000000001E00FE001E000E000E000E000E000E000E000 E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E060E0F0 C0F18061803E000B2C82A10F>I<0E0000FE00001E00000E00000E00000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E03FC0E01F00E01C00E01800E02000E 04000E08000E10000E38000EF8000F1C000E1E000E0E000E07000E07800E03C00E01C00E 01E00E00F00E00F8FFE3FE17237FA21A>I<0E00FE001E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E00FFE00B237FA20E>I<0E1FC07F00FE60E183801E8072 01C00F003C00E00F003C00E00E003800E00E003800E00E003800E00E003800E00E003800 E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E0 0E003800E00E003800E00E003800E0FFE3FF8FFE27157F942A>I<0E1F80FE60C01E80E0 0F00700F00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E0070 0E00700E00700E00700E00700E0070FFE7FF18157F941B>I<01FC000707000C01801800 C03800E0700070700070F00078F00078F00078F00078F00078F00078F000787000707800 F03800E01C01C00E038007070001FC0015157F9418>I<0E1F00FE61C00E80600F00700E 00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E0E003C0E003C0E 00380F00700E80E00E41C00E3F000E00000E00000E00000E00000E00000E00000E00000E 00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F060F000E000E000E000E000E 000E000E000E000E000E000E000E000E000F00FFF010157F9413>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C800CC00CC0 08E018D0308FC00E157E9413>I<02000200020002000600060006000E001E003E00FFF8 0E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E040E040708 030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F006017003827800 FC7F18157F941B>III121 D E /Fw 20 118 df45 D68 D73 D77 D 80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E0780000E070 0000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F80000007C 0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC0000FFFF0000 0FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F8000000F880 00007880000078800000788000007880000078C0000078C0000070E00000F0E00000E0F0 0000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E003E001E 0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E001F001E 003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E0478003E 047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700 040F00041E001E1C003E3C003E7C003E7C0008780000F80000F80000F80000F80000F800 00F80000F80000F80000F800007800007C00007C00003C00011E00011E00020F00020700 0403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E00000 01E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E00000 01E0000001E0000001E0000001E0000001E0000001E0001F81E000F061E001C019E00780 05E00F0003E00E0003E01E0001E03C0001E03C0001E07C0001E0780001E0F80001E0F800 01E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0780001E07800 01E03C0001E03C0001E01C0001E01E0003E00E0005E0070009E0038011F000E061FF003F 81FF20327DB125>I<003F800000E0E0000380380007003C000E001E001E001E001C000F 003C000F007C000F0078000F8078000780F8000780F8000780FFFFFF80F8000000F80000 00F8000000F8000000F8000000F8000000780000007C0000003C0000003C0000801E0000 800E0001000F0002000780020001C00C0000F03000001FC000191F7E9E1D>I<0007E000 1C1000383800707C00E07C01E07C01C03803C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C000FFFFC0FFFFC003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00007E0007FFF007F FF0016327FB114>I<000000F0007F030801C1C41C0380E81C070070080F0078001E003C 001E003C003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E003C 000F007800070070000780E00009C1C000087F0000180000001800000018000000180000 00180000001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E0300000 F070000070E0000038E0000038E0000038E0000038E00000387000007070000070380000 E01C0001C00700070001C01C00003FE0001E2F7E9F21>I<07000F801F801F800F800700 000000000000000000000000000000000000000000000780FF80FF800F80078007800780 078007800780078007800780078007800780078007800780078007800780078007800780 0780078007800FC0FFF8FFF80D307EAF12>105 D<0780FE001FC000FF83078060F000FF 8C03C18078000F9001E2003C0007A001E4003C0007A000F4001E0007C000F8001E0007C0 00F8001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000 F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0 001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F000 1E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E 000FC001F8003F00FFFC1FFF83FFF0FFFC1FFF83FFF0341F7E9E38>109 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C000F000 07C000F000078000F000078000F000078000F000078000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F0000FC001 F800FFFC1FFF80FFFC1FFF80211F7E9E25>I<001FC00000F0780001C01C00070007000F 0007801E0003C01C0001C03C0001E03C0001E0780000F0780000F0780000F0F80000F8F8 0000F8F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8780000F07C0001F03C 0001E03C0001E01E0003C01E0003C00F00078007800F0001C01C0000F07800001FC0001D 1F7E9E21>I<0783E0FF8C18FF907C0F907C07A07C07C03807C00007C00007C000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0780000780000780000780000780000780000FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E00010E00010F00010F800007E00 003FF0001FFF000FFFC003FFE0003FF00001F80000F880003C80003C80001CC0001CC000 1CE0001CE00018F00038F00030CC0060C301C080FE00161F7E9E1A>I<00400000400000 400000400000400000C00000C00000C00001C00001C00003C00007C0000FC0001FFFE0FF FFE003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C01003C01003C01003C01003C01003C01003C01003 C01001C02001E02000E0400078C0001F00142C7FAB19>I<078000F000FF801FF000FF80 1FF0000F8001F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078001F000 078001F000078001F000038002F00003C004F00001C008F800007030FF80001FC0FF8021 1F7E9E25>I E /Fx 5 85 df<00000000600000000000700000000000F00000000001F0 0000000001F00000000003F00000000003F00000000007F00000000007F0000000000FF0 000000000FF0000000001BF00000000033F00000000033F00000000063F00000000063F8 00000000C1F800000000C1F80000000181F80000000381F80000000301F80000000601F8 0000000601F80000000C01F80000000C01F80000001801F80000001801F80000003001F8 0000006001F80000006001F8000000C001F8000000C001FC000001FFFFFC000001FFFFFC 0000030000FC0000070000FC0000060000FC00000C0000FC00000C0000FC0000180000FC 0000180000FC0000300000FC0000700000FC0000600000FC0000E00000FC0001E00000FC 0003E00000FE000FF00001FE00FFFE003FFFF0FFFE003FFFF02C327CB135>65 D<000FFFFFFF0000000FFFFFFFC00000003F8007F00000003F8001F80000003F00007C00 00003F00007E0000007F00003E0000007F00001F0000007E00001F0000007E00001F8000 00FE00000F800000FE00000F800000FC00000FC00000FC00000FC00001FC00000FC00001 FC00000FC00001F800000FC00001F800000FC00003F800000FC00003F800001FC00003F0 00001FC00003F000001FC00007F000001FC00007F000001F800007E000003F800007E000 003F80000FE000003F80000FE000003F00000FC000007F00000FC000007F00001FC00000 7E00001FC00000FE00001F800000FC00001F800000FC00003F800001F800003F800001F0 00003F000003F000003F000007E000007F000007C000007F00000FC000007E00001F8000 007E00003F000000FE00007E000000FE0000F8000000FC0001F0000000FC0007E0000001 FC003F800000FFFFFFFE000000FFFFFFF000000032317CB036>68 D<000FFFFFFFFE000FFFFFFFFE00003F8000FE00003F80003E00003F00001E00003F0000 1E00007F00000C00007F00000C00007E00000C00007E00000C0000FE00000C0000FE0000 0C0000FC00000C0000FC00000C0001FC00001C0001FC00C0180001F800C0000001F800C0 000003F801C0000003F801C0000003F00180000003F00380000007F00F80000007FFFF80 000007FFFF00000007E00F0000000FE0070000000FE0070000000FC0060000000FC00600 00001FC00E0000001FC00E0000001F800C0000001F80000000003F80000000003F800000 00003F00000000003F00000000007F00000000007F00000000007E00000000007E000000 0000FE0000000000FE0000000000FC0000000000FC0000000001FC00000000FFFFFC0000 00FFFFFC0000002F317CB02F>70 D<000FFFFFF800000FFFFFFF0000003F801FC000003F 8007E000003F0003F000003F0001F800007F0000FC00007F0000FC00007E0000FC00007E 0000FC0000FE0000FC0000FE0001FC0000FC0001FC0000FC0001FC0001FC0001F80001FC 0003F80001F80003F00001F80007E00003F80007E00003F8000F800003F0003F000003F0 007E000007F003F8000007FFFFE0000007FFFF80000007E007C000000FE003F000000FE0 01F000000FC000F800000FC000F800001FC000FC00001FC000FC00001F8000FC00001F80 00FC00003F8001FC00003F8001FC00003F0001FC00003F0001FC00007F0003F800007F00 03F800007E0003F800007E0003F80600FE0003F80E00FE0003F80C00FC0003F80C00FC00 03F81C01FC0001F838FFFFF000FC70FFFFF0007FE0000000001F802F327CB034>82 D<07FFFFFFFFF807FFFFFFFFF80FE007F001F80F8007F000F80E0007E000701E0007E000 701C000FE0007018000FE0007038000FC0007038000FC0007030001FC0006070001FC000 6060001F80006060001F80006060003F8000E0E0003F8000C000003F00000000003F0000 0000007F00000000007F00000000007E00000000007E0000000000FE0000000000FE0000 000000FC0000000000FC0000000001FC0000000001FC0000000001F80000000001F80000 000003F80000000003F80000000003F00000000003F00000000007F00000000007F00000 000007E00000000007E0000000000FE0000000000FE0000000000FC0000000000FC00000 00001FC0000000001FC0000000001F80000000003F80000000007FC00000007FFFFFC000 007FFFFFC000002D3174B033>84 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 795 908 a Fx(D)26 b(R)g(A)f(F)h(T)225 999 y Fw(Do)r(cumen)n(t) 20 b(for)i(a)f(Standard)g(Message-P)n(assing)f(In)n(terface)621 1194 y Fv(Message)c(P)o(assing)h(In)o(terface)e(F)l(orum)828 1320 y(June)i(21,)f(1993)87 1378 y(This)g(w)o(ork)g(w)o(as)h(supp)q (orted)g(b)o(y)f(ARP)l(A)g(and)g(NSF)g(under)g(con)o(tract)g(n)o(um)o (b)q(er)f(###,)g(b)o(y)g(the)192 1436 y(National)h(Science)f(F)l (oundation)i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q (erativ)o(e)76 1494 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o (y)e(the)h(Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1552 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 75 378 a Fu(Con)m(ten)m(ts)75 648 y Ft(Ac)o(kno)o(wledgmen)o (ts)1362 b(1)75 752 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(2)143 810 y Fs(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(2)143 868 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(3)143 926 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(3)143 983 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(3)143 1041 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b Fr(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b Fs(4)75 1145 y Ft(2)42 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)g(Comm)o(unication)1001 b(5)143 1203 y Fs(2.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(5)143 1260 y(2.2)46 b(Data)14 b(T)o(yp)q(es)30 b Fr(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(5)248 1318 y(2.2.1)50 b(Handle)14 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(5)248 1376 y(2.2.2)50 b(Arra)o(y)14 b(of)h(handles)35 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(6)248 1434 y(2.2.3)50 b(State)14 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(7)248 1491 y(2.2.4)50 b(Named)15 b(constan)o(ts)23 b Fr(:)f(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)91 b Fs(7)248 1549 y(2.2.5)50 b(Choice)20 b Fr(:)i(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)91 b Fs(7)143 1607 y(2.3)46 b(Pro)q(cesses)40 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b Fs(8)248 1665 y(2.3.1)50 b(Error)14 b(Handling)29 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(8)143 1722 y(2.4)46 b(Messages)f Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(9)248 1780 y(2.4.1)50 b(Data)19 b Fr(:)j(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fs(9)248 1838 y(2.4.2)50 b(En)o(v)o(elop)q(e)45 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(10)143 1895 y(2.5)46 b(Comm)o(unication)15 b(Bu\013ers)41 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(10)248 1953 y(2.5.1)50 b(Comm)o(unication)16 b(bu\013er)f(comp)q(onen)o(ts)24 b Fr(:)e(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fs(10)248 2011 y(2.5.2)50 b(Bu\013er)15 b(op)q(erations)22 b Fr(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(13)248 2069 y(2.5.3)50 b(Comm)o(unication)16 b(Bu\013ers)f({)g (Extension)35 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b Fs(17)248 2126 y(2.5.4)50 b(T)o(yp)q(e)16 b(Matc)o(hing)25 b Fr(:)d(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)69 b Fs(25)248 2184 y(2.5.5)50 b(Data)14 b(con)o(v)o(ersion)46 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fs(25)143 2242 y(2.6)46 b(Receiv)o(e)17 b(Criteria)d Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(28)143 2300 y(2.7)46 b(Comm)o(unication)15 b(Mo)q(de)35 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(28)143 2357 y(2.8)46 b(Comm)o(unication)15 b(Ob)s(jects)28 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(29)248 2415 y(2.8.1)50 b(Comm)o(unication)16 b(Ob)s(ject)f (Creation)31 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(30)248 2473 y(2.8.2)50 b(Comm)o(unication)16 b(Start)39 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)69 b Fs(32)248 2530 y(2.8.3)50 b(Comm)o(unication)16 b(Completion)i Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(32)248 2588 y(2.8.4)50 b(Multiple)17 b(Completions)41 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(34)143 2646 y(2.9)46 b(Blo)q(c)o(king)16 b(Comm)o(unication)44 b Fr(:)23 b(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b Fs(35)143 2704 y(2.10)23 b(Non)o(blo)q(c)o (king)16 b(Comm)o(unication)41 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(36)969 2828 y(i)p eop %%Page: 2 3 2 2 bop 75 -100 a Fs(ii)1518 b Fq(CONTENTS)143 45 y Fs(2.11)23 b(Con)o(tiguous)15 b(Bu\013er)g(Comm)o(unication)g(Op)q(erations)47 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(37)143 104 y(2.12)23 b(Prob)q(e)15 b(and)g(Cancel)g Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(39)143 163 y(2.13)23 b(send-receiv)o(e)k Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(42)143 222 y(2.14)23 b(Null)16 b(pro)q(cesses)i Fr(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(46)75 329 y Ft(3)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(47)143 388 y Fs(3.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(47)143 447 y(3.2)46 b(Group)15 b(F)l(unctions)35 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(48)143 506 y(3.3)46 b(Comm)o(unication)15 b(F)l(unctions)24 b Fr(:)f(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(49)143 565 y(3.4)46 b(Sync)o(hronization)16 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(49)143 624 y(3.5)46 b(Data)14 b(mo)o(v)o(e)g(functions)29 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(50)143 683 y(3.6)46 b(Global)16 b(Compute)e(Op)q(erations)20 b Fr(:)i(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(55)75 789 y Ft(4)42 b(Pro)q(cess)17 b(T)l(op)q(ologies)1250 b(59)143 849 y Fs(4.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(59)143 908 y(4.2)46 b(Virtual)16 b(T)l(op)q(ologies)37 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(60)143 967 y(4.3)46 b(Em)o(b)q(edding)16 b(in)g(MPI)22 b Fr(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(60)143 1026 y(4.4)46 b(Ov)o(erview)16 b(of)e(the)i(prop)q(osed)f (MPI)g(functions)20 b Fr(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(61)248 1085 y(4.4.1)50 b(Lo)o(w{lev)o(el)16 b(top)q(ology)f(functions)37 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b Fs(62)248 1144 y(4.4.2)50 b(High{lev)o(el)17 b(top)q(ology)e(functions)25 b Fr(:)d(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(63)248 1203 y(4.4.3)50 b(T)l(op)q(ology)15 b(inquiry)i(functions) 42 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(65)248 1262 y(4.4.4)50 b(P)o(artitioning)16 b(of)e(cartesian)i(structures)22 b Fr(:)g(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fs(66)143 1321 y(4.5)46 b(A)15 b(simple)i(example)f Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(68)75 1427 y Ft(5)42 b(Language)19 b(Binding)1268 b(70)143 1487 y Fs(5.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(70)143 1546 y(5.2)46 b(Data)14 b(T)o(yp)q(es)h(and)g(Naming)h (Con)o(v)o(en)o(tions)43 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(70)143 1605 y(5.3)46 b(F)l(ortran)14 b(77)g(Binding)j(Issues)34 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(71)143 1664 y(5.4)46 b(C)15 b(Binding)i(Issues)35 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(71)143 1723 y(5.5)46 b(Sp)q(eci\014c)17 b(Bindings)32 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(71)248 1782 y(5.5.1)50 b(C)15 b(bindings)i(for)e(P)o(oin)o(t-to-P) o(oin)o(t)f(Comm)o(unication)k Fr(:)k(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b Fs(72)248 1841 y(5.5.2)50 b(C)15 b(Bindings)i(for)e(Collectiv)o(e)h(Comm)o(unication)g(Chapter)29 b Fr(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(74)248 1900 y(5.5.3)50 b(F)l(ortran)14 b(Bindings)j(for)e(P)o(oin)o(t-to-P)o (oin)o(t)f(Routines)34 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)69 b Fs(75)248 1959 y(5.5.4)50 b(F)l(ortran)14 b(Bindings)j(for)e(Collectiv)o(e)h(Comm)o(unciation)g(Chapter)i Fr(:)k(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(77)75 2066 y Ft(6)42 b(Correctness)16 b(Issues)1258 b(79)248 2125 y Fs(6.0.5)50 b(Order)37 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)69 b Fs(79)248 2184 y(6.0.6)50 b(Progress)14 b(and)i(F)l(airness)39 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(79)248 2243 y(6.0.7)50 b(Collectiv)o(e)17 b(Comm)o(unications)35 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(81)75 2349 y Ft(7)42 b(En)o(vironmen)o(tal)18 b(Managemen)o(t)f(and)h(Inquiry)743 b(84)143 2408 y Fs(7.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fs(84)248 2467 y(7.1.1)50 b(Program)14 b(Startup)h(Issues)36 b Fr(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(84)143 2526 y(7.2)46 b(MPI)15 b(Program)f(Startup)g(Prop)q(osal{MPI)p 995 2526 14 2 v 17 w(INIT)39 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(86)143 2585 y(7.3)46 b(MPI-Sp)q(eci\014c)17 b(Enquiry)f(F)l(unctions)f Fr(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(86)248 2645 y(7.3.1)50 b(P)o(arallel)16 b(programming)39 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(88)248 2704 y(7.3.2)50 b(non-MPI)d Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fs(89)p eop %%Page: 3 4 3 3 bop 75 -100 a Fq(CONTENTS)1504 b Fs(iii)75 45 y Ft(8)42 b(Pro\014ling)1484 b(90)143 102 y Fs(8.1)46 b(Requiremen)o(ts)28 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fs(90)143 158 y(8.2)46 b(Discussion)21 b Fr(:)i(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(90)143 214 y(8.3)46 b(Logic)16 b(of)e(the)i(design)29 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(91)143 271 y(8.4)46 b(Examples)36 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(91)248 327 y(8.4.1)50 b(Pro\014ler)16 b(implemen)o(tation)35 b Fr(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(91)248 384 y(8.4.2)50 b(MPI)15 b(library)h(implemen)o(tation)h Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)69 b Fs(91)143 440 y(8.5)46 b(Multiple)17 b(lev)o(els)f(of)f(in)o(terception)25 b Fr(:)d(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fs(93)75 542 y Ft(9)42 b(Maps,)17 b(Groups)g(and)h(Con)o (texts)1032 b(94)143 599 y Fs(9.1)46 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fs(94)143 655 y(9.2)46 b(Maps)13 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)69 b Fs(95)248 712 y(9.2.1)50 b(Op)q(erations)16 b(on)f(maps)35 b Fr(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(96)248 768 y(9.2.2)50 b(Map)15 b(constructors)h Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(97)143 824 y(9.3)46 b(Con)o(text)p 411 824 14 2 v 15 w(id)15 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(98)248 881 y(9.3.1)50 b(Op)q(erations)16 b(on)f(con)o(text)p 840 881 V 16 w(id's)f Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(98)143 937 y(9.4)46 b(Comm)o(unicators)20 b Fr(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b Fs(99)248 994 y(9.4.1)50 b(Op)q(erations)16 b(on)f(comm)o(unicators)h Fr(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fs(99)248 1050 y(9.4.2)50 b(Comm)o(unicator)15 b(constructors)23 b Fr(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(100)143 1107 y(9.5)g(W)l(orking)15 b(without)g(con)o(text)p 750 1107 V 16 w(id's)33 b Fr(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(101)143 1163 y(9.6)g(W)l(orking) 15 b(without)g(maps)g(and)g(con)o(text)p 957 1163 V 16 w(id's)38 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(102)143 1220 y(9.7)g(Examples)36 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fs(103)248 1276 y(9.7.1)k(Nonreen)o(tran)o(t)14 b(parallel)j(pro)q(cedures)29 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(103)248 1333 y(9.7.2)k(P)o(arallel)16 b(pro)q(cedures)g(that)f(are)g(nonreen)o (tran)o(t)f(within)j(eac)o(h)e(executing)h(group)d(104)248 1389 y(9.7.3)50 b(W)l(ell)17 b(nested)e(parallel)i(pro)q(cedures)28 b Fr(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b Fs(104)143 1445 y(9.8)g(Left)41 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(105)75 1547 y Ft(10)16 b(Initial)k(Implemen)o(tation)e(Subset)967 b(106)143 1604 y Fs(10.1)23 b(In)o(tro)q(duction)15 b Fr(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(106)143 1660 y(10.2)23 b(Criteria)15 b(and)h(Rational)c Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(106)143 1717 y(10.3)23 b(Subset)15 b(F)l(unctionalit)o(y)31 b Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(108)248 1773 y(10.3.1)27 b(Itemized)17 b(F)l(unctionalit)o(y)23 b Fr(:)f(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(108)248 1830 y(10.3.2)27 b(P)o(oin)o(t)15 b(to)g(P)o(oin)o(t)f(F)l(unctionalit)o(y)i Fr(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(108)248 1886 y(10.3.3)27 b(Collectiv)o(e)17 b(Comm)o(unication)e(F)l(unctionalit)o(y)24 b Fr(:)e(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fs(109)75 1988 y Ft(A)28 b(Pro)q(cess)17 b(T)l(op)q(ology)h(Routines) 1035 b(112)p eop %%Page: 0 5 0 4 bop 875 722 a Ft(Abstract)75 828 y Fs(The)18 b(Message)f(P)o (assing)h(In)o(terface)f(F)l(orum)h(\(MPIF\),)e(with)i(participation)h (from)e(o)o(v)o(er)g(40)g(organiza-)75 885 y(tions,)f(has)h(b)q(een)g (meeting)g(since)h(Jan)o(uary)e(1993)f(to)h(discuss)h(and)g(de\014ne)g (a)f(set)h(of)f(library)h(in)o(terface)75 941 y(standards)f(for)h (message)f(passing.)25 b(MPIF)16 b(is)i(not)e(sanctioned)i(or)e(supp)q (orted)h(b)o(y)g(an)o(y)g(o\016cial)g(stan-)75 998 y(dards)e (organization.)166 1054 y(This)22 b(is)f(a)g(draft)f(of)h(what)g(will)h (b)q(ecome)g(the)f(Final)h(Rep)q(ort,)h(V)l(ersion)f(1.0,)f(of)f(the)i (Message)75 1111 y(P)o(assing)c(In)o(terface)h(F)l(orum.)29 b(This)19 b(do)q(cumen)o(t)g(con)o(tains)g(all)g(the)g(tec)o(hnical)h (features)e(prop)q(osed)h(for)75 1167 y(the)c(in)o(terface.)20 b(This)c(cop)o(y)f(of)g(the)g(draft)g(w)o(as)f(pro)q(cessed)i(b)o(y)f (L)1174 1161 y Fp(a)1195 1167 y Fs(T)1220 1181 y(E)1246 1167 y(X)g(on)g(June)h(21,)f(1993.)166 1224 y(MPIF)j(in)o(vites)h (commen)o(ts)f(on)h(the)f(tec)o(hnical)i(con)o(ten)o(t)e(of)g(MPI,)g (as)g(w)o(ell)i(as)e(on)g(the)h(editorial)75 1280 y(presen)o(tation)14 b(in)g(the)g(do)q(cumen)o(t.)19 b(Commen)o(ts)13 b(receiv)o(ed)i(b)q (efore)e(July)i(1,)e(1993)g(will)i(b)q(e)f(considered)h(in)75 1336 y(pro)q(ducing)i(the)e(\014nal)h(draft)e(of)h(V)l(ersion)h(1.0)e (of)h(the)g(Message)g(P)o(assing)g(In)o(terface)g(Sp)q(eci\014cation.) 166 1393 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o (terface,)g(simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g (used)75 1449 y(standard)g(for)f(writing)i(message-passing)f(programs.) 24 b(As)17 b(suc)o(h)h(the)f(in)o(terface)g(should)h(establishing)75 1506 y(a)d(practical,)h(p)q(ortable,)f(e\016cien)o(t,)g(and)h (\015exible)h(standard)e(for)f(message)h(passing.)p eop %%Page: 1 6 1 5 bop 75 -100 a Fq(CONTENTS)1519 b Fs(1)75 75 y Fu(Ac)m(kno)m (wledgmen)m(ts)166 282 y Fs(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f (w)o(as)e(carried)i(out)f(b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o (as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d)h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P) o(assing)g(In)o(terface)75 395 y(\(MPI\),)f(man)o(y)h(p)q(eople)i(serv) o(ed)e(in)h(p)q(ositions)g(of)f(resp)q(onsibilit)o(y:)143 501 y Fo(\017)23 b Fs(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o (er,)g(Con)o(v)o(eners)g(and)g(Meeting)h(Chairs)143 595 y Fo(\017)23 b Fs(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes) 143 689 y Fo(\017)23 b Fs(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i (Lusk,)f(P)o(oin)o(t)g(to)g(P)o(oin)o(t)f(Comm)o(unications)143 782 y Fo(\017)23 b Fs(Al)16 b(Geist,)e(Marc)h(Snir,)h(Stev)o(e)f(Otto,) f(Collectiv)o(e)j(Comm)o(unications)143 876 y Fo(\017)23 b Fs(Rolf)15 b(Hemp)q(el,)i(Pro)q(cess)e(T)l(op)q(ologies)143 970 y Fo(\017)23 b Fs(Scott)14 b(Berryman,)h(Language)g(Binding)143 1064 y Fo(\017)23 b Fs(Stev)o(e)15 b(Zenith,)g(F)l(ormal)g(Seman)o (tics)143 1158 y Fo(\017)23 b Fs(William)17 b(Gropp,)d(En)o(vironmen)o (tal)i(Inquiry)143 1251 y Fo(\017)23 b Fs(James)15 b(Co)o(wnie,)g (Pro\014ling)143 1345 y Fo(\017)23 b Fs(T)l(on)o(y)16 b(Skjellum,)h(Lyndon)g(Clark)o(e,)f(Marc)g(Snir,)h(Ric)o(hard)g (Little\014eld,)i(Marc)d(Sears,)g(Comm)o(u-)189 1402 y(nication)g(Con)o(texts)143 1496 y Fo(\017)23 b Fs(Stev)o(e)15 b(Otto,)f(Editor.)166 1602 y(Do)h(w)o(e)f(w)o(an)o(t)h(an)g (organization/compan)o(y)f(list)i(here?)166 1658 y(The)22 b(follo)o(wing)h(list)f(includes)i(some)e(of)f(the)h(activ)o(e)g (participan)o(ts)h(in)f(the)g(MPI)g(pro)q(cess)g(not)75 1715 y(men)o(tioned)16 b(ab)q(o)o(v)o(e:)100 1758 y(Ed)f(Anderson)114 b(Jo)q(e)15 b(Baron)193 b(Eric)15 b(Barszcz)100 1815 y(Rob)g(Bjornson)99 b(Anne)16 b(Elster)159 b(Jim)16 b(F)l(eeney)100 1871 y(Sam)f(Fineb)q(erg)98 b(Jon)15 b(Flo)o(w)o(er)176 b(Daniel)16 b(F)l(ry)o(e)100 1928 y(Ian)f(Glendinning)54 b(Adam)15 b(Green)o(b)q(erg)56 b(Rob)q(ert)15 b(Harrison)100 1984 y(Leslie)i(Hart)153 b(T)l(om)15 b(Haupt)166 b(Don)15 b(Heller)100 2041 y(T)l(om)g(Henderson)60 b(C.)14 b(T.)h(Ho)o(w)o(ard)f (Ho)50 b(Stev)o(en)15 b(Huss-Lederman)100 2097 y(John)h(Kap)q(enga)86 b(Bob)15 b(Leary)188 b(Arth)o(ur)14 b(Maccab)q(e)100 2154 y(P)o(eter)h(Madams)83 b(Oliv)o(er)16 b(McBry)o(an)77 b(Phil)16 b(McKinley)100 2210 y(Ch)o(uc)o(k)f(Mosher)87 b(Dan)15 b(Nessett)152 b(P)o(eter)14 b(P)o(ac)o(heco)100 2266 y(P)o(aul)h(Pierce)147 b(Sanja)o(y)14 b(Rank)m(a)124 b(P)o(eter)14 b(Rigsb)q(ee)100 2323 y(Arc)o(h)h(Robison)103 b(Am)o(buj)15 b(Singh)136 b(Alan)16 b(Sussman)100 2379 y(Rob)q(ert)f(G.)g(V)l(oigt)50 b(Dennis)16 b(W)l(eeks)120 b(Stephen)16 b(Wheat)166 2435 y(The)c(follo)o(wing)h(organizations)f (made)g(the)h(draft)e(a)o(v)m(ailable)j(b)o(y)e(anon)o(ymous)g(FTP)f (access)i(and/or)75 2491 y(mail)h(serv)o(ers:)k(Oak)c(Ridge)g(National) g(Lab)q(oratory)l(,)e(These)i(outlets)f(w)o(ere)g(instrumen)o(tal)h(in) g(distribut-)75 2547 y(ing)i(the)f(do)q(cumen)o(t.)166 2604 y(T)l(alk)g(ab)q(out)g(\014nancial)i(supp)q(ort)f(here.)p eop %%Page: 2 7 2 6 bop 75 356 a Fn(Chapter)34 b(1)75 564 y Fu(In)m(tro)s(duction)41 b(to)g(MPI)75 805 y Fm(1.1)70 b(Ov)n(erview)21 b(and)j(Goals)75 906 y Fs(Message)11 b(passing)i(is)f(a)g(paradigm)f(used)i(widely)g(on) f(certain)h(classes)f(of)f(parallel)j(mac)o(hines;)f(esp)q(ecially)75 963 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1019 y(pro)q(cesses)12 b(comm)o(unicating)h(through)e(messages)g(is) i(w)o(ell)f(understo)q(o)q(d.)20 b(Ov)o(er)12 b(the)f(last)h(ten)g(y)o (ears,)f(sub-)75 1076 y(stan)o(tial)16 b(progress)g(has)g(b)q(een)h (made)f(in)h(casting)g(signi\014can)o(t)g(applications)h(in)f(this)f (paradigm.)23 b(Eac)o(h)75 1132 y(v)o(endor)c(has)h(implemen)o(ted)h (their)f(o)o(wn)f(v)m(arian)o(t.)33 b(More)19 b(recen)o(tly)l(,)j(sev)o (eral)d(systems)g(ha)o(v)o(e)h(demon-)75 1188 y(strated)12 b(that)h(a)g(message)f(passing)i(system)f(can)g(b)q(e)h(e\016cien)o (tly)h(and)e(b)q(e)h(p)q(ortably)f(implemen)o(ted.)21 b(It)13 b(is)75 1245 y(th)o(us)h(an)f(appropriate)h(time)g(to)f(try)g (to)g(de\014ne)i(b)q(oth)f(the)g(syn)o(tax)f(and)h(seman)o(tics)g(of)f (a)g(core)h(of)f(library)75 1301 y(routines)h(that)e(will)j(b)q(e)f (useful)h(to)e(a)g(wide)h(range)f(of)g(users)g(and)h(e\016cien)o(tly)g (implemen)o(table)i(on)d(a)g(wide)75 1358 y(range)i(of)g(computers.)166 1414 y(In)21 b(designing)i(MPI)e(w)o(e)f(ha)o(v)o(e)h(sough)o(t)f(to)g (mak)o(e)g(use)h(of)g(the)g(most)f(attractiv)o(e)f(features)i(of)f(a)75 1471 y(n)o(um)o(b)q(er)12 b(of)g(existing)h(message)e(passing)h (systems,)g(rather)f(than)h(selecting)h(one)f(of)g(them)g(and)g (adopting)75 1527 y(it)f(as)f(the)h(standard.)18 b(Th)o(us,)11 b(MPI)g(has)f(b)q(een)i(strongly)f(in\015uenced)i(b)o(y)d(w)o(ork)g(at) g(the)h(IBM)g(T.)f(J.)h(W)l(atson)75 1584 y(Researc)o(h)18 b(Cen)o(ter)g([1)o(,)g(2)o(],)g(In)o(tel's)g(NX/2)g([16)o(],)f(Express) h([15)o(],)g(nCUBE's)g(V)l(ertex)g([13)o(],)f(P4)h([3)o(],)g(and)75 1640 y(P)l(ARMA)o(CS)11 b([11)o(,)g(12)o(].)18 b(Other)11 b(imp)q(ortan)o(t)g(con)o(tributions)g(ha)o(v)o(e)g(come)g(from)f(Zip)q (co)q(de)i([17)o(,)f(18)o(],)g(Chimp)75 1697 y([7)o(,)k(8],)f(PVM)h([9) o(,)g(19],)f(and)h(PICL)h([10)o(].)166 1753 y(One)j(of)f(the)g(ob)s (jectiv)o(es)g(of)g(this)g(pap)q(er)h(is)g(to)e(promote)h(a)f (discussion)j(within)g(the)e(concurren)o(t)75 1810 y(computing)d (researc)o(h)f(comm)o(unit)o(y)f(of)h(the)g(issues)h(that)f(m)o(ust)f (b)q(e)i(addressed)f(in)h(establishing)h(a)e(prac-)75 1866 y(tical,)j(p)q(ortable,)g(and)f(\015exible)j(standard)d(for)g (message)f(passing.)24 b(This)17 b(co)q(op)q(erativ)o(e)g(pro)q(cess)g (b)q(egan)75 1922 y(with)f(a)e(w)o(orkshop)h(on)g(standards)g(for)f (message)h(passing)h(held)g(in)g(April)h(1992)d([20)o(].)166 1979 y(The)k(main)h(adv)m(an)o(tages)e(of)h(establishing)i(a)e(message) g(passing)g(standard)g(are)g(p)q(ortabilit)o(y)h(and)75 2035 y(ease-of-use.)h(In)14 b(a)g(distributed)h(memory)e(comm)o (unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f(higher)h(lev)o (el)75 2092 y(routines)c(and/or)g(abstractions)f(are)h(build)i(up)q(on) e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i(routines)f(the)g(b)q (ene\014ts)75 2148 y(of)22 b(standardization)h(are)f(particularly)h (apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g (message)75 2205 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop) q(osed)g(here,)h(pro)o(vides)f(v)o(endors)h(with)f(a)g(clearly)h (de\014ned)h(base)75 2261 y(set)10 b(of)g(routines)g(that)g(they)g(can) h(implemen)o(t)g(e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o (vide)f(hardw)o(are)g(supp)q(ort)75 2318 y(for,)k(thereb)o(y)h (enhancing)i(scalabilit)o(y)l(.)166 2374 y(The)h(goal)g(of)g(the)g (Message)g(P)o(assing)g(In)o(terface)g(simply)i(stated)d(is)i(to)f(dev) o(elop)h(a)f(widely)h(used)75 2431 y(standard)e(for)f(writing)i (message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface) g(should)h(establishing)75 2487 y(a)d(practical,)h(p)q(ortable,)f (e\016cien)o(t,)g(and)h(\015exible)h(standard)e(for)f(message)h (passing.)166 2543 y(A)g(complete)h(list)g(of)f(goals)g(follo)o(w.)143 2647 y Fo(\017)23 b Fs(Design)11 b(an)f(application)i(programming)e(in) o(terface)h(\(not)f(necessarily)i(for)d(compilers)j(or)e(a)g(system)189 2704 y(implemen)o(tation)16 b(library\).)964 2828 y(2)p eop %%Page: 3 8 3 7 bop 75 -100 a Fq(1.2.)34 b(WHO)15 b(SHOULD)h(USE)g(THIS)g(ST)l(AND) o(ARD?)824 b Fs(3)143 45 y Fo(\017)23 b Fs(Allo)o(w)15 b(e\016cien)o(t)h(comm)o(unication:)21 b(Av)o(oid)16 b(memory)e(to)h(memory)g(cop)o(ying)g(and)h(allo)o(w)f(o)o(v)o(erlap) 189 102 y(of)e(computation)g(and)h(comm)o(unication)g(and)g(o\017oad)f (to)g(comm)o(unication)h(copro)q(cessor,)g(where)189 158 y(a)o(v)m(ailable.)143 255 y Fo(\017)23 b Fs(Allo)o(w)15 b(\(but)g(no)h(mandate\))e(extensions)i(for)f(use)g(in)h(heterogeneous) f(en)o(vironmen)o(t.)143 352 y Fo(\017)23 b Fs(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C,)f(F)l(ortran)f(77,)g(F)l(ortran)g(90,)g(and) i(C++)f(bindings)i(for)e(in)o(terface.)143 449 y Fo(\017)23 b Fs(Assume)15 b(a)g(reliable)i(comm)o(unication)f(in)o(terface:)k (User)15 b(need)h(not)f(cop)q(e)h(with)f(comm)o(unication)189 506 y(failures.)21 b(Suc)o(h)15 b(failures)i(are)d(dealt)i(b)o(y)f(the) h(underlying)h(comm)o(unication)f(subsystem.)143 603 y Fo(\017)23 b Fs(F)l(o)q(cus)15 b(on)g(a)g(prop)q(osal)h(that)e(can)h (b)q(e)h(agreed)f(up)q(on)h(in)g(6)f(mon)o(ths.)143 700 y Fo(\017)23 b Fs(De\014ne)f(an)g(in)o(terface)g(that)f(is)h(not)f(to)q (o)g(di\013eren)o(t)h(from)f(curren)o(t)h(practice,)h(suc)o(h)f(as)g (PVM,)189 756 y(Express,)14 b(P4,)h(etc.)143 853 y Fo(\017)23 b Fs(De\014ne)12 b(an)f(in)o(terface)h(that)f(can)h(b)q(e)g(quic)o(kly) h(implemen)o(ted)g(on)f(man)o(y)f(v)o(endor's)g(platforms,)h(with)189 909 y(no)j(signi\014can)o(t)h(c)o(hanges)f(in)h(the)f(underlying)j (comm)o(unication)d(and)h(system)f(soft)o(w)o(are.)143 1007 y Fo(\017)23 b Fs(The)15 b(in)o(terface)h(should)g(not)f(con)o (tain)g(more)g(functions)h(than)f(are)g(really)h(necessary)l(.)143 1104 y Fo(\017)23 b Fs(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h (should)g(b)q(e)g(language)f(indep)q(enden)o(t.)75 1251 y Fm(1.2)70 b(Who)23 b(Should)g(Use)g(This)f(Standard?)75 1354 y Fs(This)16 b(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h (all)g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 1411 y(programs)21 b(in)j(F)l(ortran)d(77,)i(C,)f (F)l(ortran)g(90,)h(or)f(C++.)43 b(This)23 b(includes)i(individual)g (application)75 1467 y(programmers,)14 b(dev)o(elop)q(ers)j(of)e(soft)o (w)o(are)f(designed)j(to)e(run)h(on)g(parallel)h(mac)o(hines,)f(and)g (creators)e(of)75 1524 y(en)o(vironmen)o(ts,)g(and)f(to)q(ols.)19 b(In)14 b(order)f(to)g(b)q(e)h(attractiv)o(e)e(to)h(this)h(wide)g (audience,)h(the)f(standard)f(m)o(ust)75 1580 y(pro)o(vide)j(a)f (simple,)i(easy-to-use)e(in)o(terface)h(for)e(the)i(basic)g(user)f (while)i(not)e(seman)o(tically)i(precluding)75 1637 y(the)e(high-p)q (erformance)i(message-passing)e(op)q(erations)g(a)o(v)m(ailable)i(on)e (adv)m(anced)i(mac)o(hines.)75 1784 y Fm(1.3)70 b(What)23 b(Platforms)f(Are)h(T)-6 b(argets)23 b(F)-6 b(or)24 b(Implemen)n (tation?)75 1887 y Fs(The)17 b(attractiv)o(eness)f(of)g(the)h (message-passing)g(paradigm)f(at)g(least)h(partially)h(stems)e(from)g (its)h(wide)75 1944 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o(y)f(ma)o(y)g(run)h(on)g (distributed-memory)h(m)o(ultipro)q(cessors,)75 2000 y(net)o(w)o(orks)i(of)i(w)o(orkstations,)e(and)i(com)o(binations)g(of)g (all)g(of)f(these.)34 b(In)20 b(addition,)h(shared-memory)75 2057 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 2113 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 2170 y(th)o(us)d(should)i(b)q(e)f(b)q(oth) g(p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f (on)h(a)f(great)g(v)m(ariet)o(y)g(of)75 2226 y(mac)o(hines,)i (including)h(those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h (of)e(other)h(mac)o(hines,)g(parallel)75 2283 y(or)d(not,)f(connected)i (b)o(y)f(a)g(comm)o(unication)h(net)o(w)o(ork.)166 2340 y(The)21 b(in)o(terface)f(is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i (general)f(MIMD)f(programs,)g(as)g(w)o(ell)h(as)f(those)75 2396 y(written)d(in)g(the)g(more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g (Although)h(no)f(explicit)j(supp)q(ort)e(for)f(threads)g(is)75 2453 y(pro)o(vided,)g(the)f(in)o(terface)g(has)g(b)q(een)i(designed)f (so)f(as)g(not)g(to)f(prejudice)j(their)f(use.)75 2601 y Fm(1.4)70 b(What)23 b(Is)g(Included)f(In)h(The)f(Standard?)75 2704 y Fs(The)15 b(standard)g(includes)j(\(this)d(is)h(temp)q(orarily)f (as)g(inclusiv)o(e)j(as)d(p)q(ossible\):)p eop %%Page: 4 9 4 8 bop 75 -100 a Fs(4)903 b Fq(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)143 45 y Fo(\017)23 b Fs(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 139 y Fo(\017)23 b Fs(Collectiv)o(e)16 b(op)q(erations)143 233 y Fo(\017)23 b Fs(Pro)q(cess)15 b(groups)143 327 y Fo(\017)23 b Fs(Comm)o(unication)15 b(con)o(texts)143 420 y Fo(\017)23 b Fs(A)15 b(simple)i(w)o(a)o(y)d(to) g(create)h(pro)q(cesses)h(for)e(the)i(SPMD)e(mo)q(del)143 514 y Fo(\017)23 b Fs(Bindings)17 b(for)d(F)l(ortran)g(77,)g(F)l (ortran)h(90,)f(C)h(and)g(C++)143 608 y Fo(\017)23 b Fs(A)15 b(mo)q(del)h(implemen)o(tation)143 702 y Fo(\017)23 b Fs(A)15 b(formal)g(sp)q(eci\014cation.)143 796 y Fo(\017)23 b Fs(Pro)q(cess)15 b(top)q(ology)143 889 y Fo(\017)23 b Fs(A)15 b(v)m(alidation)i(suite)75 1033 y Fm(1.5)70 b(What)23 b(Is)g(Not)g(Included)f(In)h(The)f(Standard?)75 1134 y Fs(The)15 b(standard)g(do)q(es)h(not)f(sp)q(ecify:)143 1240 y Fo(\017)23 b Fs(Explicit)17 b(shared-memory)e(op)q(erations)143 1334 y Fo(\017)23 b Fs(Op)q(erations)c(that)e(require)i(more)f(op)q (erating)h(system)f(supp)q(ort)g(than)g(is)h(curren)o(tly)g(standard;) 189 1391 y(for)14 b(example,)i(in)o(terrupt-driv)o(en)g(receiv)o(es,)g (remote)f(execution,)h(or)e(activ)o(e)i(messages)143 1484 y Fo(\017)23 b Fs(Program)13 b(construction)j(to)q(ols)143 1578 y Fo(\017)23 b Fs(Debugging)15 b(facilities)143 1672 y Fo(\017)23 b Fs(Auxiliary)17 b(functions)e(suc)o(h)h(as)f (timers)143 1766 y Fo(\017)23 b Fs(Explicit)17 b(supp)q(ort)e(for)g (threads)166 1872 y(There)g(are)f(man)o(y)h(features)f(that)g(ha)o(v)o (e)g(b)q(een)i(considered)g(and)f(not)g(included)i(in)f(this)f (standard.)75 1929 y(This)j(happ)q(ened)h(for)e(a)g(n)o(um)o(b)q(er)h (of)f(reasons,)g(one)g(of)g(whic)o(h)i(is)f(the)f(time)h(constrain)o(t) f(that)g(w)o(as)f(self)75 1985 y(imp)q(osed)g(in)h(\014nishing)g(the)e (standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g(included)j(can)e (alw)o(a)o(ys)e(b)q(e)i(o\013ered)f(as)75 2042 y(extensions)h(b)o(y)f (sp)q(eci\014c)i(implemen)o(tations.)p eop %%Page: 5 10 5 9 bop 75 357 a Fn(Chapter)34 b(2)75 565 y Fu(P)m(oin)m(t)41 b(to)f(P)m(oin)m(t)h(Comm)m(unication)876 788 y Fs(Marc)14 b(Snir)656 844 y(William)j(Gropp)e(and)g(Ewing)h(Lusk)75 989 y Fm(2.1)70 b(In)n(tro)r(duction)75 1092 y Fs(This)22 b(c)o(hapter)g(is)g(a)f(draft)g(of)g(the)h(curren)o(t)g(prop)q(osal)f (for)g(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication.)40 b(The)75 1148 y(MPI)18 b(pro)q(cedures)h(are)f(sp)q(eci\014ed)i(using)f (a)f(language)h(indep)q(enden)o(t)h(notation.)29 b(The)18 b(parameters)f(of)75 1205 y(pro)q(cedure)f(calls)g(are)f(mark)o(ed)g (as)f Fl(IN)h Fs(or)g Fl(OUT)p Fs(.)f(The)h(call)i(do)q(es)e(not)g(up)q (date)g(a)g(parameter)g(mark)o(ed)f Fl(IN)75 1261 y Fs(and)i(ma)o(y)e (up)q(date)i(a)g(parameter)e(mark)o(ed)h Fl(OUT)p Fs(.)g(If)h(a)f (parameter)f(is)i(a)f(handle)i(to)e(an)g(opaque)h(ob)s(ject,)75 1318 y(and)i(the)g(ob)s(ject)f(is)h(up)q(dated)h(b)o(y)f(the)f(pro)q (cedure)i(call,)g(then)f(the)g(parameter)f(is)h(mark)o(ed)g Fl(OUT)p Fs(.)f(The)75 1374 y(F)l(ortran)d(77)h(and)g(C)g(bindings)i (are)e(describ)q(ed)i(in)f(Chapter)f(5.)166 1431 y(Discussions)h(that)f (are)g(not)f(part)h(of)g(the)g(standard)g(are)g(indicated)i(b)o(y)e (small)h(t)o(yp)q(e.)166 1564 y Fk(Discussion:)45 b Fj(The)17 b(follo)o(wing)d(sections)k(of)e(the)h(in)o(tro)q(duction)g(con)o(tain) f(general)h(information)d(on)i(the)75 1614 y(design)g(of)f(MPI)h(pro)q (cedures.)27 b(The)16 b(material)e(should)h(b)q(e)i(mo)o(v)o(ed)d(to)i (a)g(general)g(in)o(tro)q(duction)f(for)h(the)g(en)o(tire)75 1663 y(do)q(cumen)o(t.)i(The)c(actual)f(binding)g(of)h(these)h(ob)r (jects)g(in)e(F)m(ortran)h(and)g(C)g(is)f(describ)q(ed)j(in)d(Chapter)i (5.)75 1891 y Fm(2.2)70 b(Data)23 b(T)n(yp)r(es)75 1995 y Fi(2.2.1)55 b(Handle)75 2082 y Fs(MPI)16 b(pro)q(cedures)h(use)f(at)f (v)m(arious)h(places)h Fh(hand)r(les)p Fs(.)22 b(Handles)17 b(are)e(used)i(to)e(access)h(opaque)g(ob)s(jects.)75 2138 y(Suc)o(h)d(ob)s(ject)e(can)h(b)q(e)h(created,)f(up)q(dated)h(and) f(destro)o(y)o(ed)f(only)i(b)o(y)f(calling)h(suitable)h(MPI)d(pro)q (cedures,)75 2195 y(and)18 b(pro)o(viding)h(the)e(handle)i(as)f (parameter.)26 b(A)18 b(handle)h(to)e(an)h(opaque)f(ob)s(ject)h(is)g (signi\014can)o(t)g(only)75 2251 y(at)d(the)g(pro)q(cess)h(where)g(the) g(ob)s(ject)f(w)o(as)f(created,)h(and)h(cannot)g(b)q(e)g(transferred)f (to)g(another)g(pro)q(cess)75 2308 y(\(ob)s(jects)f(are)h(lo)q(cal\).) 166 2364 y(Opaque)k(ob)s(jects)g(hide)h(from)e(the)h(user)g(the)g(in)o (ternal)g(represen)o(tation)g(used)g(for)g(v)m(arious)g(MPI)75 2421 y(ob)s(jects,)12 b(th)o(us)g(allo)o(wing)i(to)e(ha)o(v)o(e)g (similar)i(calls)f(in)h(C)e(and)h(F)l(ortran,)e(allo)o(wing)j(to)e(o)o (v)o(ercome)f(problems)75 2477 y(with)19 b(the)f(t)o(yping)h(rules)g (in)g(these)g(languages,)g(and)g(allo)o(wing)g(for)f(future)g (extension)h(of)f(their)h(func-)75 2534 y(tionalit)o(y)l(.)k(The)16 b(mec)o(hanism)h(for)e(opaque)h(ob)s(jects)f(used)i(here)f(lo)q(osely)h (follo)o(ws)f(the)g(POSIX)h(F)l(ortran)75 2590 y(binding)g(standard.) 166 2647 y(An)i(opaque)h(ob)s(ject)e(can)i(b)q(e)f Fh(p)n(ersistent)f Fs(or)h Fh(ephemer)n(al)p Fs(.)32 b(A)19 b(p)q(ersisten)o(t)h(ob)s (ject)e(p)q(ersists)i(un)o(til)75 2704 y(destro)o(y)o(ed)c(b)o(y)g(an)g (explicit)i(op)q(eration.)23 b(An)16 b(ephemeral)i(ob)s(ject)d(is)i(go) q(o)q(d)f(for)f(a)h(single)i(use;)e(th)o(us,)g(an)964 2828 y(5)p eop %%Page: 6 11 6 10 bop 75 -100 a Fs(6)631 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(ephemeral)d(ob)s(ject)d (asso)q(ciated)i(with)g(a)f(comm)o(unication)h(op)q(eration)g(disapp)q (ears)g(once)g(this)g(op)q(eration)75 102 y(is)g(completed)g(\(or)f (once)g(this)h(ob)s(ject)f(is)h(not)f(needed)i(an)o(ymore)d(for)h(the)h (completion)g(of)f(the)g(op)q(eration\).)166 158 y(Opaque)21 b(ob)s(ject)e(are)h(created)g(b)o(y)g(calls)h(that)f(are)f(sp)q (eci\014c)j(to)e(eac)o(h)g(ob)s(ject)f(t)o(yp)q(e.)35 b(They)20 b(are)75 214 y(destro)o(y)o(ed)i(b)o(y)g(a)g(call)h(to)f Fl(MPI)p 629 214 15 2 v 17 w(FREE)p Fs(.)f(Additional)j(MPI)f (functions)g(are)f(a)o(v)m(ailable)i(to)d(access)i(and)75 271 y(up)q(date)16 b(sp)q(eci\014c)h(opaque)e(ob)s(jects.)166 363 y Ft(MPI)p 275 363 16 2 v 18 w(FREE\(handle\))75 487 y(IN)i(handle)24 b Fs(handle)16 b(to)f(ob)s(ject)166 577 y(Mark)k(the)g(ob)s(ject)g(asso)q(ciated)h(with)g(the)g(handle)g (for)f(destruction.)34 b(The)20 b(ob)s(ject)e(can)i(b)q(e)g(de-)75 633 y(stro)o(y)o(ed)e(as)g(so)q(on)g(as)g(there)h(there)f(is)h(no)g(p)q (ending)h(op)q(eration)f(that)f(is)h(using)g(this)g(ob)s(ject,)f(and)h (the)75 690 y(handle)d(b)q(ecomes)e(unde\014ned.)22 b(F)l(or)14 b(example,)g(if)h(a)f(comm)o(unication)h(ob)s(ject)f(is)h(freed,)f (after)g(the)g(com-)75 746 y(m)o(unication)21 b(started)f(but)h(b)q (efore)f(it)h(completed,)h(then)f(the)g(p)q(ending)h(op)q(eration)f(is) g(not)f(a\013ected;)75 802 y(one)e(can)f(use)h(the)g(handle)h(in)f (calls)h(to)e Fl(MPI)p 849 802 15 2 v 17 w(WAIT)g Fs(or)g Fl(MPI)p 1109 802 V 17 w(STATUS)f Fs({)i(one,)g(in)g(fact,)f(m)o(ust)g (complete)75 859 y(the)i(comm)o(unication)g(with)g(a)f(successful)i (call)f(to)f Fl(MPI)p 1055 859 V 17 w(WAIT)g Fs(or)g Fl(MPI)p 1317 859 V 17 w(STATUS)p Fs(.)f(Ho)o(w)o(ev)o(er,)h(once)h (the)75 915 y(op)q(eration)c(completes,)h(it)f(is)h(erroneous)f(to)g (reuse)g(the)g(freed)h(handle)g(for)f(a)g(new)g(comm)o(unication.)166 1055 y Fk(Implemen)o(tati)o(on)j(note:)55 b Fj(Some)17 b(ob)r(jects,)k(suc)o(h)e(as)g(bu\013er)h(descriptor)g(ob)r(jects,)g (can)f(b)q(e)h(used)f(b)o(y)75 1111 y(sev)o(eral)e(distinct)f (concurren)o(t)i(comm)o(unication)13 b(op)q(erations.)26 b(Th)o(us,)16 b(to)g(\014nd)h(out)f(whether)i(suc)o(h)f(ob)r(ject)g (can)75 1167 y(b)q(e)e(freed,)f(one)g(needs)h(a)f(reference)i(coun)o(t) e(of)f(p)q(ending)h(op)q(erations.)-370 1258 y Fg(TO)d(DISCUSS)166 1389 y Fk(Discussion:)32 b Fj(W)m(e)12 b(do)h(not)f(ha)o(v)o(e)g(y)o (et)h(a)f(mec)o(hanism)e(to)i(con)o(trol)h(what)f(storage)h(is)f(allo)q (cated)g(for)g(opaque)75 1446 y(ob)r(jects.)19 b(Some)13 b(b)q(eliev)o(e)h(it)g(is)f(desirable)i(to)e(allo)o(w)g(the)h(user)h (to)f(con)o(trol)f(this)h(allo)q(cation.)166 1620 y Ft(MPI)p 275 1620 16 2 v 18 w(ASSOCIA)l(TED\(handle,)k(t)o(yp)q(e\))75 1745 y(IN)f(handle)24 b Fs(handle)16 b(to)f(ob)s(ject)75 1837 y Ft(OUT)j(t)o(yp)q(e)23 b Fs(state)166 1926 y(Returns)17 b(the)g(t)o(yp)q(e)g(of)f(the)h(ob)s(ject)g(the)g(handle)h(is)f(curren) o(tly)g(asso)q(ciated)g(with,)h(if)f(suc)o(h)g(exists.)75 1983 y(Returns)d(the)f(sp)q(ecial)i(t)o(yp)q(e)e Fl(MPI)p 643 1983 15 2 v 17 w(NULL)f Fs(if)i(the)f(handle)h(is)g(not)f(curren)o (tly)g(asso)q(ciated)h(with)f(an)o(y)g(ob)s(ject.)166 2039 y(MPI)19 b(ma)o(y)g(pro)o(vide)h(prede\014ned)h(opaque)f(ob)s (jects)f(and)g(prede\014ned,)j(static)d(handles)i(to)e(these)75 2095 y(ob)s(jects.)g(Suc)o(h)d(ob)s(jects)f(ma)o(y)f(not)h(b)q(e)h (destro)o(y)o(ed.)75 2216 y Fi(2.2.2)55 b(Arra)n(y)19 b(of)g(handles)75 2302 y Fs(An)j(MPI)f(call)i(ma)o(y)e(need)i(a)e (parameter)g(that)g(is)h(an)f Fh(arr)n(ay)i(of)f(hand)r(les)p Fs(.)39 b(The)22 b(arra)o(y-of-handles)75 2359 y(is)17 b(a)g(regular)f(arra)o(y)g(with)h(en)o(tries)g(of)f(t)o(yp)q(e)h Fl(handle)f Fs(in)h(consecutiv)o(e)h(initial)h(lo)q(cations)e(in)h(the) e(arra)o(y)l(.)75 2415 y(Whenev)o(er)11 b(suc)o(h)g(arra)o(y)f(is)i (used,)g(an)e(additional)j Fl(len)d Fs(parameter)g(is)i(required)g(to)e (indicate)j(the)e(n)o(um)o(b)q(er)75 2471 y(of)k(v)m(alid)i(en)o(tries) e(\(unless)h(this)g(n)o(um)o(b)q(er)f(can)h(b)q(e)g(deriv)o(ed)g (otherwise\).)166 2604 y Fk(Discussion:)166 2654 y Fj(The)h(reason)h (for)f(not)g(ha)o(ving)f(self-delimiting)e(arra)o(ys)k(is)f(that)g (records)h(are)g(not)f(\014rst-class)h(citizens)g(in)75 2704 y(F)m(ortran)13 b(and)g(that)g(ha)o(ving)f(an)h(arra)o(y)g(where)h (one)f(en)o(try)h(enco)q(des)h(length,)e(whereas)h(the)g(other)f(en)o (tries)i(enco)q(de)p eop %%Page: 7 12 7 11 bop 75 -100 a Fq(2.2.)34 b(D)o(A)l(T)l(A)15 b(TYPES)1375 b Fs(7)75 45 y Fj(handles)13 b(w)o(as)g(deemed)g(to)g(b)q(e)g(to)q(o)g (error-prone.)19 b(This)13 b(line)f(of)g(argumen)o(tation)f(really)h (implies)f(that)i(w)o(e)g(do)g(not)75 95 y(view)e(arra)o(y-of-handles)g (as)h(an)f(opaque)h(ob)r(ject,)g(and)g(that)f(w)o(e)h(exp)q(ect)h(the)f (user)h(to)f(build)e(these)j(ob)r(jects)g(explic-)75 145 y(itly)m(,)f(not)h(via)g(MPI)h(calls.)k(The)c(alternativ)o(e)f(is)h (for)f(arra)o(y-of-handles)g(to)h(b)q(e)g(an)f(opaque)h(ob)r(ject,)g (manipulated)75 195 y(via)f(MPI)h(calls.)75 451 y Fi(2.2.3)55 b(State)75 554 y Fs(MPI)15 b(pro)q(cedures)h(use)g(at)e(v)m(arious)i (places)g(argumen)o(ts)f(with)g Fh(state)g Fs(t)o(yp)q(es.)20 b(The)c(v)m(alues)g(of)f(suc)o(h)h(data)75 610 y(t)o(yp)q(e)i(are)h (all)g(iden)o(ti\014ed)h(b)o(y)f(names,)f(and)h(no)f(op)q(eration)h(is) g(de\014ned)h(on)e(them.)29 b(F)l(or)18 b(example,)i(the)75 667 y Fl(MPI)p 150 667 15 2 v 17 w(APPEND)14 b Fs(routine)i(has)f(a)g (state)f(t)o(yp)q(e)h(parameter)g(with)g(v)m(alues)i Fl(MPI)p 1343 667 V 16 w(INTEGER)p Fs(,)d Fl(MPI)p 1626 667 V 17 w(REAL,)h Fs(etc.)75 840 y Fi(2.2.4)55 b(Named)17 b(constan)n(ts)75 944 y Fs(MPI)11 b(pro)q(cedures)h(sometimes)f(assign) g(a)g(sp)q(ecial)i(meaning)f(to)e(a)h(sp)q(ecial)i(v)m(alue)f(of)f(a)g (basic)g(t)o(yp)q(e)g(param-)75 1000 y(eter;)17 b(e.g.)24 b Fl(tag)17 b Fs(is)g(an)g(in)o(teger)g(v)m(alued)h(parameter)e(of)g(p) q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)f(op)q(erations,)75 1057 y(with)h(a)f(sp)q(ecial)i(wildcard)g(v)m(alue,)g Fl(MPI)p 766 1057 V 17 w(ANY)p 855 1057 V 17 w(TAG)p Fs(.)d(Suc)o(h)i(parameters)f(will)i(ha)o(v)o(e)e(a)h(range)f(of)g (regular)75 1113 y(v)m(alues,)h(whic)o(h)g(is)f(a)f(prop)q(er)h (subrange)g(of)f(the)h(range)g(of)f(v)m(alues)i(of)e(the)h(corresp)q (onding)h(basic)f(t)o(yp)q(e;)75 1169 y(sp)q(ecial)e(v)m(alues)f(\(suc) o(h)f(as)g Fl(MPI)p 601 1169 V 17 w(ANY)p 690 1169 V 16 w(TAG)p Fs(\))g(will)h(b)q(e)g(outside)g(the)f(regular)g(range.)19 b(The)13 b(range)g(of)g(regular)75 1226 y(v)m(alues)g(can)g(b)q(e)g (queried,)g(and)g(sometimes)f(set,)g(using)h(en)o(vironmen)o(t)g (inquiry)h(or)d(en)o(vironmen)o(t)i(setting)75 1282 y(functions)j (\(Chapter)f(7\).)166 1424 y Fk(Discussion:)166 1483 y Fj(Implemen)o(ters)f(should)g(detect,)j(whenev)o(er)f(p)q(ossible,)f (illegal)e(uses)j(of)e(\\sp)q(ecial)h(v)n(alues".)21 b(Th)o(us,)15 b(the)g(use)75 1533 y(of)e(the)i Ff(MPI)p 263 1533 14 2 v 15 w(ANY)p 344 1533 V 15 w(TAG)e Fj(v)n(alue)g(to)h (tag)g(a)f(message)h(sen)o(t)g(will)f(b)q(e)h(\015agged)g(as)g(an)g (error.)75 1789 y Fi(2.2.5)55 b(Choice)75 1892 y Fs(MPI)16 b(functions)g(sometimes)g(use)g(parameters)f(with)h(a)g Fh(choic)n(e)f Fs(\(or)g(union\))i(data)e(t)o(yp)q(e.)21 b(I.e.,)16 b(distinct)75 1948 y(calls)i(to)e(the)h(same)f(routine)h(ma) o(y)f(pass)h(b)o(y)f(reference)i(actual)e(parameters)g(of)g(di\013eren) o(t)h(t)o(yp)q(es.)25 b(The)75 2005 y(mec)o(hanism)16 b(for)e(pro)o(viding)j(suc)o(h)e(parameters)f(will)j(di\013er)f(from)e (language)i(to)e(language.)166 2146 y Fk(Discussion:)166 2205 y Fj(The)j(F)m(ortran)g(77)g(standard)g(sp)q(eci\014es)i(that)e (the)h(t)o(yp)q(e)f(of)f(actual)h(argumen)o(ts)f(need)i(to)f(agree)h (with)e(the)75 2255 y(t)o(yp)q(e)f(of)f(dumm)o(y)d(argumen)o(ts;)i(no)i (construct)h(equiv)n(alen)o(t)d(to)i(C)f(p)q(oin)o(ters)h(is)f(a)o(v)n (ailable.)j(Th)o(us,)e(it)f(w)o(ould)f(seem)75 2305 y(that)19 b(there)i(is)e(no)g(standard)h(conforming)d(mec)o(hanism)g(to)i(supp)q (ort)h(c)o(hoice)f(parameters.)35 b(Ho)o(w)o(ev)o(er,)20 b(most)75 2355 y(F)m(ortran)g(compiler)e(either)j(don't)e(c)o(hec)o(k)i (t)o(yp)q(e)f(consistency)h(of)e(calls)h(to)f(external)i(routines,)g (or)f(supp)q(ort)g(a)75 2405 y(sp)q(ecial)14 b(mec)o(hanism)d(to)j (link)f(foreign)g(\(e.g.,)g(C\))g(routines.)19 b(W)m(e)13 b(accept)i(this)f(nonconformit)o(y)e(with)h(F)m(ortran)g(77)75 2454 y(standard.)25 b(I.e.,)15 b(w)o(e)i(accept)g(that)f(the)h(same)e (routine)h(ma)o(y)e(b)q(e)j(passed)g(an)e(actual)h(parameter)g(of)f(a)h (di\013eren)o(t)75 2504 y(t)o(yp)q(e)e(at)g(distinct)g(calls.)166 2563 y(Generic)j(routines)g(can)f(b)q(e)h(used)h(in)d(F)m(ortran)i(90)e (to)h(pro)o(vide)h(a)f(standard)g(conforming)f(solution.)24 b(This)75 2613 y(solution)13 b(will)f(b)q(e)j(consisten)o(t)g(with)f (our)f(nonstandard)i(conforming)c(F)m(ortran)j(77)f(solution.)p eop %%Page: 8 13 8 12 bop 75 -100 a Fs(8)631 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fm(2.3)70 b(Pro)r(cesses)75 235 y Fk(Discussion:)15 b Fj(This)f(material)e(b)q(elongs)i(to)g(an)f (in)o(tro)q(duction)h(or)g(en)o(vironmen)o(t)f(section)166 376 y Fs(An)i(MPI)f(program)g(is)h(executed)h(b)o(y)e(sev)o(eral)h (autonomous)f(pro)q(cesses)h(that)e(eac)o(h)i(execute)g(their)75 433 y(o)o(wn)k(co)q(de,)h(in)g(a)e(MIMD)h(st)o(yle.)31 b(The)19 b(co)q(des)h(executed)g(b)o(y)f(eac)o(h)g(pro)q(cess)g(need)h (not)f(b)q(e)g(iden)o(tical.)75 489 y(The)h(pro)q(cesses)g(comm)o (unicate)g(via)g(calls)g(to)f(MPI)h(comm)o(unication)g(primitiv)o(es.) 35 b(T)o(ypically)l(,)22 b(eac)o(h)75 546 y(pro)q(cessor)17 b(executes)i(in)f(its)g(o)o(wn)f(address)g(space,)h(although)g (shared-memory)g(implemen)o(tations)g(of)75 602 y(MPI)i(are)f(p)q (ossible.)35 b(This)20 b(do)q(cumen)o(t)g(sp)q(eci\014es)i(the)d(b)q (eha)o(vior)h(of)g(a)f(parallel)i(program)e(assuming)75 659 y(that)c(only)g(MPI)h(calls)g(are)f(used)h(for)e(comm)o(unication.) 21 b(The)16 b(in)o(teraction)f(of)g(an)g(MPI)g(program)g(with)75 715 y(other)g(p)q(ossible)i(means)e(of)g(comm)o(unication)h(\(e.g.,)d (shared)j(memory\))e(is)i(not)e(sp)q(eci\014ed.)166 774 y(MPI)d(do)q(es)g(not)g(sp)q(ecify)h(the)f(execution)h(mo)q(del)g(for)e (eac)o(h)h(pro)q(cess.)19 b(A)11 b(pro)q(cess)g(can)h(b)q(e)f(sequen)o (tial,)75 831 y(or)h(can)g(b)q(e)h(m)o(ultithreaded,)g(with)g(threads)f (p)q(ossibly)h(executing)h(concurren)o(tly)l(.)19 b(Care)12 b(has)g(b)q(een)h(tak)o(en)75 887 y(to)i(mak)o(e)f(MPI)h (\\thread-safe",)f(b)o(y)i(a)o(v)o(oiding)f(the)g(use)h(of)f(implicit)i (global)f(states.)166 947 y(MPI)h(do)q(es)g(not)g(sp)q(ecify)h(either)g (the)f(in)o(teraction)g(of)g(pro)q(cesses)g(with)h(signals,)g(in)f(a)g (Unix)h(en)o(vi-)75 1003 y(ronmen)o(t,)g(or)g(other)g(ev)o(en)o(ts)g (that)g(do)g(not)g(relate)h(to)f(MPI)g(comm)o(unication.)30 b(I.e.,)19 b(signals)g(are)f(not)75 1059 y(signi\014can)o(t)j(from)e (the)h(view)g(p)q(oin)o(t)h(of)e(MPI,)h(and)g(implemen)o(ters)h(should) g(attempt)d(to)i(implemen)o(t)75 1116 y(MPI)14 b(so)g(that)f(signals)i (are)f(transparen)o(t:)19 b(an)14 b(MPI)g(call)h(susp)q(ended)h(b)o(y)e (a)g(signal)h(should)g(resume)g(and)75 1172 y(complete)h(after)e(the)i (signal)g(is)f(handled.)166 1232 y(MPI)j(do)q(es)g(not)f(pro)o(vide)h (mec)o(hanisms)g(to)f(sp)q(ecify)i(the)f(initial)i(allo)q(cation)f(of)e (pro)q(cesses)h(to)f(an)75 1288 y(MPI)d(computation)g(and)g(their)g (binding)i(to)d(ph)o(ysical)j(pro)q(cessors.)j(It)14 b(is)g(exp)q(ected)h(that)f(v)o(endors)f(will)75 1344 y(pro)o(vide)20 b(mec)o(hanisms)h(to)e(do)g(so)h(either)g(at)f(load)h (time)g(or)g(at)f(run)h(time.)34 b(Suc)o(h)20 b(mec)o(hanisms)g(will)75 1401 y(allo)o(w)c(the)f(sp)q(eci\014cation)j(of)d(the)h(initial)i(n)o (um)o(b)q(er)d(of)h(required)g(pro)q(cesses,)g(the)g(co)q(de)g(to)f(b)q (e)h(executed)75 1457 y(b)o(y)k(eac)o(h)h(initial)i(pro)q(cess,)e(and)g (the)g(allo)q(cation)g(of)f(pro)q(cesses)h(to)f(pro)q(cessors.)36 b(Also,)21 b(the)g(curren)o(t)75 1514 y(prop)q(osal)f(do)q(es)h(not)e (pro)o(vide)i(for)e(dynamic)i(creation)f(or)g(deletion)h(of)f(pro)q (cesses)g(during)h(program)75 1570 y(execution)c(\(the)f(total)f(n)o (um)o(b)q(er)i(of)f(pro)q(cesses)g(is)h(\014xed\),)f(although)g(it)h (is)f(in)o(tended)i(to)d(b)q(e)i(consisten)o(t)75 1627 y(with)i(suc)o(h)f(extension.)31 b(Finally)l(,)20 b(the)f(curren)o(t)f (prop)q(osal)h(do)q(es)f(not)g(sp)q(ecify)i(a)e(naming)h(sc)o(heme)g (for)75 1683 y(pro)q(cesses.)33 b(W)l(e)20 b(prop)q(ose)f(to)g(alw)o(a) o(ys)g(iden)o(tify)i(pro)q(cesses)f(according)g(to)f(their)h(relativ)o (e)g(rank)f(in)h(a)75 1740 y(group,)15 b(so)f(that,)g(e\013ectiv)o(ely) l(,)i(pro)q(cesses)g(are)f(iden)o(ti\014ed)i(b)o(y)e(consecutiv)o(e)h (in)o(tegers.)75 1877 y Fi(2.3.1)55 b(Error)18 b(Handling)75 2051 y Fk(Discussion:)34 b Fj(This)14 b(section)g(should)g(b)q(e)g(mo)o (v)o(ed)f(to)h(a)f(general)h(in)o(tro)q(duction.)166 2193 y Fs(MPI)k(pro)o(vides)g(the)h(user)f(with)g(reliable)i(message)e (transmission:)26 b(A)18 b(message)f(sen)o(t)h(is)h(alw)o(a)o(ys)75 2249 y(receiv)o(ed)f(correctly)l(,)f(and)h(the)f(user)g(do)q(es)g(not)f (need)i(to)e(c)o(hec)o(k)i(for)e(transmission)h(errors,)f(time-outs,)75 2306 y(or)i(other)g(error)f(conditions.)31 b(In)19 b(other)f(w)o(ords,) g(MPI)g(do)q(es)h(not)f(pro)o(vide)h(mec)o(hanisms)g(for)f(dealing)75 2362 y(with)i(failures)h(in)g(the)f(comm)o(unication)h(system.)34 b(Where)20 b(the)g(MPI)g(implemen)o(tation)h(is)g(built)g(on)75 2419 y(an)h(unreliable)i(underlying)g(mec)o(hanism,)f(then)g(it)f(is)g (the)g(job)f(of)h(the)g(implemen)o(ter)h(of)e(the)h(MPI)75 2475 y(subsystem)16 b(to)g(insulate)i(the)e(user)h(from)f(this)h (unreliabilit)o(y)l(,)i(or)d(to)g(re\015ect)h(unreco)o(v)o(erable)g (errors)e(as)75 2531 y(global)20 b(program)e(failures.)34 b(Similarly)21 b(MPI)f(itself)g(pro)o(vides)g(no)f(mec)o(hanisms)h(for) f(handling)i(no)q(de)75 2588 y(failures.)166 2647 y(Of)16 b(course,)h(MPI)f(programs)f(ma)o(y)h(still)i(b)q(e)f(erroneous.)23 b(A)16 b Ft(program)i(error)d Fs(can)i(o)q(ccur)f(when)75 2704 y(an)f(MPI)g(call)g(is)h(called)g(with)f(an)g(incorrect)g (parameter)f(\(non-existing)i(destination)g(in)g(a)e(send)i(op)q(er-)p eop %%Page: 9 14 9 13 bop 75 -100 a Fq(2.4.)34 b(MESSA)o(GES)1423 b Fs(9)75 45 y(ation,)17 b(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i(op)q (eration,)f(etc.\))25 b(This)17 b(t)o(yp)q(e)g(of)f(error)g(w)o(ould)h (o)q(ccur)h(in)f(an)o(y)75 102 y(implemen)o(tation.)31 b(In)19 b(addition,)h(a)f Ft(resource)h(error)d Fs(ma)o(y)h(o)q(ccur)h (when)g(a)f(program)f(exceeds)j(the)75 158 y(amoun)o(t)d(of)f(a)o(v)m (ailable)j(system)e(resources)h(\(n)o(um)o(b)q(er)f(of)g(p)q(ending)i (messages,)e(system)g(bu\013ers,)g(etc.\).)75 214 y(The)h(o)q (ccurrence)h(of)f(this)g(t)o(yp)q(e)g(of)g(error)f(dep)q(ends)j(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(in)h(the)75 271 y(system)13 b(and)h(the)g(resource)f(allo)q(cation)i(mec)o(hanism)f (used;)h(this)f(ma)o(y)f(di\013er)h(from)f(system)g(to)g(system.)75 327 y(The)i(recommended)g(implemen)o(tation)g(will)h(pro)o(vide)f (generous)g(limits)g(on)g(some)f(of)g(the)g(resources)g(so)75 384 y(as)i(to)g(alleviate)i(the)f(p)q(ortabilit)o(y)g(problem)h(this)f (represen)o(ts.)24 b(One)17 b(can)g(also)f(write)h Ft(safe)g Fs(programs,)75 440 y(that)d(are)h(not)g(sub)s(ject)g(to)g(resource)g (errors.)166 499 y(All)21 b(MPI)f(calls)h(return)f(an)f(error)h (parameter)f(that)g(indicates)i(successful)g(completion)g(of)f(the)75 556 y(op)q(eration,)15 b(or)g(the)g(error)g(condition)h(that)f(o)q (ccurred,)g(otherwise.)21 b(Ho)o(w)o(ev)o(er,)13 b(MPI)j(do)q(es)f(not) g(require)75 612 y(the)f(user)f(to)g(c)o(hec)o(k)h(error)e(parameters.) 19 b(It)13 b(is)h(recommended)h(to)e(pro)o(vide)h(an)f(implemen)o (tation)i(where)75 668 y(the)g(en)o(vironmen)o(t)g(for)f(an)h(MPI)g (execution)g(can)g(b)q(e)h(set)e(so)h(that)f(an)o(y)g(error)g(encoun)o (tered)i(in)f(an)g(MPI)75 725 y(call)k(will)h(cause)e(a)f(program)g(to) g(fail)i(with)f(a)g(suitable)h(error)e(message)h(returned)g(to)g(the)g (user.)28 b(The)75 781 y(en)o(vironmen)o(t)16 b(for)f(an)g(MPI)h (execution)g(can)g(b)q(e)g(also)g(set)f(so)g(that)g(\(nonfatal\))g (errors)g(do)g(not)g(cause)h(a)75 838 y(program)e(to)h(fail,)g(and)h (the)f(user)g(can)h(pro)o(vide)f(his)h(or)f(her)g(o)o(wn)g(error)g (reco)o(v)o(ery)f(co)q(de.)166 897 y(MPI)f(calls)h(ma)o(y)f(initiate)h (op)q(erations)g(that)e(con)o(tin)o(ue)i(async)o(hronously)g(after)e (the)i(call)g(returned.)75 953 y(Th)o(us,)22 b(the)g(op)q(eration)f(ma) o(y)g(return)g(with)h(a)f(co)q(de)h(indicating)h(successful)g (completion,)g(y)o(et)e(later)75 1010 y(cause)15 b(an)g(error)g (exception)h(to)e(b)q(e)i(raised.)k(If)15 b(there)g(is)h(a)f(subsequen) o(t)g(call)h(that)f(relates)g(to)f(the)h(same)75 1066 y(op)q(eration)f(\(e.g.,)f(a)g(call)i(that)e(v)o(eri\014es)i(that)e(an) h(async)o(hronous)f(op)q(eration)h(has)g(completed\))h(then)f(the)75 1122 y(error)f(parameter)g(asso)q(ciated)h(with)g(this)g(call)h(will)g (b)q(e)g(used)f(to)f(indicate)i(the)f(nature)g(of)f(the)h(error.)k(In) 75 1179 y(a)c(few)h(cases,)f(the)g(error)g(ma)o(y)g(o)q(ccur)h(after)f (all)h(calls)h(that)d(relate)i(to)f(the)h(op)q(eration)f(ha)o(v)o(e)g (completed,)75 1235 y(so)g(that)g(no)g(error)g(parameter)g(can)g(b)q(e) i(used)f(to)e(indicate)k(the)d(nature)g(of)g(the)h(error)f(\(e.g.,)f (an)h(error)g(in)75 1292 y(a)f(send)h(with)g(the)g(ready)f(mo)q(de\).) 19 b(Suc)o(h)c(error)d(m)o(ust)h(b)q(e)i(treated)e(as)g(fatal,)g(since) i(information)e(cannot)75 1348 y(b)q(e)j(returned)f(for)g(the)g(user)h (to)e(reco)o(v)o(er)h(from)f(it.)166 1490 y Fk(Discussion:)32 b Fj(A)13 b(p)q(ossible)f(mec)o(hanism)e(is)i(to)g(ha)o(v)o(e)g(MPI)h (calls)f(generate)h(a)f(signal)f(that)i(can)f(b)q(e)h(ignored)75 1546 y(when)e(an)g(error)h(is)f(encoun)o(tered.)19 b(A)11 b(more)f(mo)q(dular)f(mec)o(hanism)g(that)i(allo)o(ws)f(to)g(handle)h (an)g(exception)h(within)75 1603 y(its)i(con)o(text)g(is)g(preferred.) 75 1842 y Fm(2.4)70 b(Messages)75 1948 y Fs(A)15 b(message)g(consists)g (of)g(an)g Fh(envelop)n(e)f Fs(and)i Fh(data)p Fs(.)75 2083 y Fi(2.4.1)55 b(Data)75 2174 y Fs(The)19 b(data)f(part)g(of)g(a)h (message)f(consists)h(of)f(a)h(sequence)g(of)g(v)m(alues,)h(eac)o(h)f (of)f(a)h(basic)g(datat)o(yp)q(e)f(in)75 2230 y(the)d(host)f(language.) 20 b(Th)o(us,)14 b(in)i(F)l(ortran)d(77,)h(a)h(message)f(consists)h(of) f(a)g(sequence)i(of)e(v)m(alues)i(that)e(are)75 2286 y(eac)o(h)20 b(of)g(t)o(yp)q(e)h Fl(INTEGER)p Fs(,)e Fl(REAL)p Fs(,)g Fl(DOUBLE)k(PRECISION)p Fs(,)c Fl(COMPLEX)p Fs(,)g Fl(DOUBLE)k(PRECISION)f(COMPLEX)p Fs(,)75 2343 y Fl(LOGICAL)p Fs(,)12 b(or)h(\(length)h(1\))f Fl(CHARACTER)p Fs(.)e(A)j(message)f(ma)o(y)f(also)i(con)o(tain)f(a)h(v)m(alue)g(of)f (t)o(yp)q(e)h Fl(BYTE)p Fs(,)e(whic)o(h)75 2399 y(consists)18 b(of)g(8)g(binary)h(digits.)29 b(A)19 b(b)o(yte)e(is)i(di\013eren)o(t)g (from)e(a)h(c)o(haracter.)28 b(Di\013eren)o(t)18 b(mac)o(hines)h(ma)o (y)75 2456 y(ha)o(v)o(e)f(di\013eren)o(t)h(represen)o(tations)g(for)f (the)h(same)f(c)o(haracter,)h(or)f(ma)o(y)g(use)h(more)g(than)f(one)h (b)o(yte)g(to)75 2512 y(represen)o(t)12 b(a)g(c)o(haracter.)18 b(On)13 b(the)f(other)g(hand,)h(a)e(b)o(yte)h(has)g(the)h(same)e (binary)i(v)m(alue)h(on)e(an)o(y)f(mac)o(hine.)75 2569 y(A)k(message)g(ma)o(y)f(mix)i(v)m(alues)g(of)f(di\013eren)o(t)h(t)o (yp)q(es.)166 2704 y Fk(Missing:)p eop %%Page: 10 15 10 14 bop 75 -100 a Fs(10)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fj(Need)d(to)f (agree)h(on)e(the)i(C)f(t)o(yp)q(es)h(\(including)e(handling)g(of)h (signed/unsigned\),)g(and)g(on)g(F)m(ortran)g(90)f(t)o(yp)q(es)75 95 y(\(including)i(handling)g(of)g(kinds\).)75 312 y Fi(2.4.2)55 b(En)n(v)n(elop)r(e)75 402 y Fs(The)15 b(follo)o(wing)h (information)g(is)f(asso)q(ciated)h(with)f(eac)o(h)h(message:)75 504 y Ft(source)22 b Fs(The)16 b(rank)f(of)f(the)i(sending)g(pro)q (cess)g(\(in)o(teger\))75 606 y Ft(destination)25 b Fs(The)15 b(rank)g(of)g(the)g(receiving)i(pro)q(cess)e(\(in)o(teger\))75 709 y Ft(tag)24 b Fs(User)15 b(de\014ned)h(\(in)o(teger\))75 811 y Ft(con)o(text)23 b Fs(comm)o(unication)16 b(con)o(text)e (\(handle\))166 913 y(The)g(range)g(of)g(v)m(alid)h(v)m(alues)h(for)d (the)h Ft(source)g Fs(and)g Ft(destination)i Fs(\014elds)g(is)e Fl(0)24 b(...)47 b(n-1)p Fs(,)14 b(where)75 970 y Fl(n)g Fs(is)h(the)f(n)o(um)o(b)q(er)h(of)f(pro)q(cesses)g(in)h(the)g(sp)q (eci\014ed)h(con)o(text.)j Fl(source)k(=)h(destination)13 b Fs(is)h(allo)o(w)o(ed:)20 b(a)75 1026 y(pro)q(cess)15 b(can)h(send)g(a)e(message)h(to)g(itself.)166 1085 y(The)h(ranges)f(of) h(v)m(alid)h(v)m(alues)g(for)e Fl(tag)h Fs(is)g(implemen)o(tation)h (dep)q(enden)o(t,)g(and)f(can)g(b)q(e)h(found)f(b)o(y)75 1141 y(calling)h(a)e(suitable)h(query)g(function,)f(as)g(describ)q(ed)i (in)f(Chapter)f(7.)166 1200 y(The)g Fl(tag)g Fs(\014eld)h(can)f(b)q(e)h (arbitrarily)g(set)f(b)o(y)g(the)g(application,)h(and)g(can)f(b)q(e)h (used)f(to)g(distinguish)75 1256 y(di\013eren)o(t)g(messages.)166 1315 y Fl(Context)f Fs(should)i(b)q(e)g(a)f(con)o(text)g(shared)g(b)o (y)g(b)q(oth)g(source)h(and)f(destination.)166 1374 y(The)e(actual)h (mec)o(hanism)g(used)f(to)g(asso)q(ciate)g(an)g(en)o(v)o(elop)q(e)i (with)e(a)g(message)g(is)h(implemen)o(tation)75 1430 y(dep)q(enden)o(t;)k(some)e(of)g(the)g(information)g(\(e.g.,)f Ft(sender)g Fs(or)h Ft(receiv)o(er)p Fs(\))f(ma)o(y)g(b)q(e)i (implicit,)i(and)d(need)75 1487 y(not)f(b)q(e)h(explicitly)i(carried)d (b)o(y)h(a)e(message.)166 1628 y Fk(Discussion:)44 b Fj(\\Con)o(text")17 b(has)f(no)o(w)g(b)q(ecome)h(\\comm)o(uni)o (cator",)d(but)j(this)f(w)o(ord)h(is)f(akw)o(ard)g(to)h(use.)75 1684 y(Suggestions?)75 1922 y Fm(2.5)70 b(Comm)n(unication)20 b(Bu\013ers)75 2028 y Fs(The)f(basic)h(p)q(oin)o(t)g(to)e(p)q(oin)o(t)i (comm)o(unication)g(op)q(erations)f(are)g Ft(send)g Fs(and)g Ft(receiv)o(e)p Fs(.)31 b(A)19 b Ft(send)g Fs(op-)75 2084 y(eration)g(creates)g(a)f(message;)i(the)g(message)e(data)g(is)i (tak)o(en)f(from)f(the)h Ft(send)i(bu\013er)p Fs(.)31 b(A)19 b Ft(receiv)o(e)75 2141 y Fs(op)q(eration)i(consumes)h(a)f (message;)i(the)e(message)g(data)f(is)i(put)f(in)o(to)g(the)g Ft(receiv)o(e)j(bu\013er)p Fs(.)37 b(The)75 2197 y(sp)q(eci\014cation) 18 b(of)d(send)h(or)f(receiv)o(e)i(bu\013ers)f(uses)g(the)g(same)f(syn) o(tax.)21 b(The)16 b(syn)o(tax)f(is)h(designed)h(to)e(b)q(e)75 2254 y(extendable)i(so)d(that)h(new)g(t)o(yp)q(es)g(of)g(bu\013ers)g (can)h(added)g(b)o(y)f(v)o(endors.)75 2388 y Fi(2.5.1)55 b(Comm)n(unication)17 b(bu\013er)h(comp)r(onen)n(ts)75 2478 y Fs(A)e(bu\013er)g(consists)g(of)g(a)f(sequence)i(of)f Ft(bu\013er)h(comp)q(onen)o(ts)p Fs(.)23 b(Eac)o(h)15 b(comp)q(onen)o(t)h(consists)h(of)e(a)h(se-)75 2534 y(quence)21 b(of)f(\(not)f(necessarily)i(distinct\))g(v)m(ariables)h(of)d(the)h (same)g(basic)h(datat)o(yp)q(e.)33 b(The)20 b(datat)o(yp)q(e)75 2591 y(of)k(suc)o(h)g(a)g(v)m(ariable)h(is)g(sp)q(eci\014ed)h(b)o(y)e (a)g(state)f(parameter.)45 b(The)25 b(p)q(ossible)h(v)m(alues)f(for)e (F)l(ortran)75 2647 y(are)10 b Fl(MPI)p 221 2647 15 2 v 17 w(INTEGER)p Fs(,)f Fl(MPI)p 500 2647 V 17 w(REAL)p Fs(,)h Fl(MPI)p 708 2647 V 17 w(DOUBLE)p Fs(,)f Fl(MPI)p 963 2647 V 17 w(COMPLEX)p Fs(,)g Fl(MPI)p 1242 2647 V 17 w(DOUBLE)p 1403 2647 V 16 w(COMPLEX)p Fs(,)h Fl(MPI)p 1682 2647 V 16 w(LOGICAL)p Fs(,)75 2704 y Fl(MPI)p 150 2704 V 17 w(CHARACTER)f Fs(and)h Fl(MPI)p 547 2704 V 17 w(BYTE)p Fs(,)f(corresp)q(onding)i(to)f(datat)o(yp)q(es)f Fl(INTEGER)p Fs(,)g Fl(REAL)p Fs(,)g Fl(DOUBLE)23 b(PRECISION)p Fs(,)p eop %%Page: 11 16 11 15 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(11)75 45 y Fl(COMPLEX)p Fs(,)9 b Fl(DOUBLE)23 b(PRECISION)g(COMPLEX)p Fs(,)9 b Fl(LOGICAL)p Fs(,)h Fl(CHARACTER)f Fs(and)i(to)f(un)o(t)o(yp)q(ed)i Fl(BYTE)p Fs(.)e(The)h(p)q(os-)75 102 y(sible)i(v)m(alues)g(for)f(C)f (are)h Fl(MPI)p 566 102 15 2 v 17 w(CHAR)p Fs(,)f Fl(MPI)p 775 102 V 16 w(SHORT)p Fs(,)g Fl(MPI)p 1007 102 V 17 w(INT)p Fs(,)g Fl(MPI)p 1192 102 V 17 w(LONG)p Fs(,)f Fl(MPI)p 1400 102 V 17 w(FLOAT)p Fs(,)h Fl(MPI)p 1633 102 V 17 w(DOUBLE)g Fs(and)75 158 y Fl(MPI)p 150 158 V 17 w(BYTE)p Fs(.)166 291 y Fk(Discussion:)166 340 y Fj(Ma)o(y)k(need)h(to)f(tak)o(e)g(care)h(of)e(signed/unsigned)i(in)f (C.)f(More)i(elab)q(orate)f(de\014nitions)g(b)q(elong)g(to)g(the)h (lan-)75 390 y(guage)e(binding.)166 529 y Fs(There)h(are)g(\014v)o(e)h (kinds)g(of)f(bu\013er)g(comp)q(onen)o(ts:)75 647 y Ft(Con)o(tiguous)j (comp)q(onen)o(t)75 733 y Fs(A)d(sequence)i(of)d(con)o(tiguous)i(v)m (alues)g(of)f(the)g(same)g(basic)h(t)o(yp)q(e,)f(sp)q(eci\014ed)i(b)o (y)75 815 y Ft(start)23 b Fs(Initial)17 b(elemen)o(t)75 904 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fl(count)c Fo(\025)h Fl(0)p Fs(\))75 993 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 1074 y(Th)o(us,)e(if)h Fl(A)e Fs(is)i(an)f(arra)o(y)f(of)h(in)o(tegers,)g(then)g Fl(A\(1\),)24 b(4,)f(MPI)p 1210 1074 V 17 w(INT)13 b Fs(is)g(the)g(bu\013er)g(comp)q(onen)o(t)h(with)75 1131 y(en)o(tries)i Fl(A\(1\),)23 b(A\(2\),)g(A\(3\),)g(A\(4\))p Fs(.)75 1249 y Ft(V)l(ector)17 b(comp)q(onen)o(t)75 1334 y Fs(A)f(sequence)i(of)e(equally)i(spaced)f(and)f(equally)i(sized)g (blo)q(c)o(ks)f(of)f(elemen)o(ts)h(of)f(the)g(same)g(basic)h(t)o(yp)q (e,)75 1391 y(sp)q(eci\014ed)g(b)o(y)75 1473 y Ft(start)23 b Fs(Initial)17 b(elemen)o(t)75 1562 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fl(count)c Fo(\025)h Fl(0)p Fs(\))75 1651 y Ft(stride)23 b Fs(Num)o(b)q(er)15 b(of)g(elemen)o(ts)h(b)q(et)o(w)o(een)f(the)h(start)e(of)g(eac)o(h)i (blo)q(c)o(k)75 1740 y Ft(len)o(blk)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(in)g(eac)o(h)g(blo)q(c)o(k)f(\()p Fl(lenblk)d Fr(>)h Fl(0)p Fs(\))75 1829 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 1910 y(Assume)c(that)f(an)h (elemen)o(t)h(of)e(t)o(yp)q(e)h Fl(datatype)f Fs(o)q(ccupies)i Fl(m)f Fs(b)o(ytes.)18 b(Then,)12 b(in)g(a)e(b)o(yte)h(addressable)75 1967 y(mac)o(hine,)h(the)g(bu\013er)f(comp)q(onen)o(t)g(describ)q(ed)i (b)o(y)e(these)g(\014v)o(e)g(parameters)g(consists)g(of)g(the)g Fl(count)f Fs(v)m(ari-)75 2023 y(ables)16 b(at)e(addresses)i Fl(start,)23 b(start+m,)g(...)47 b(,)24 b(start+m*\(lenblk-1\),)d (start+m*stride,)75 2080 y(start+m*\(stride+1\),)g(...,)j (start+m*\(stride+lenblk-)o(1\),)d(...)p Fs(.)h(F)l(or)16 b(example,)h(if)f Fl(A)g Fs(is)h(an)75 2136 y(arra)o(y)d(of)h(in)o (tegers,)g(then)143 2218 y Fo(\017)23 b Fl(A\(1\),)g(5,)g(3,)h(2,)g (MPI)p 622 2218 V 16 w(INT)17 b Fs(is)g(the)g(bu\013er)g(comp)q(onen)o (t)g(with)g(en)o(tries)g Fl(A\(1\),)24 b(A\(2\),)f(A\(4\),)189 2274 y(A\(5\),)g(A\(7\))p Fs(.)143 2363 y Fo(\017)g Fl(A\(1\),)g(5,)g (0,)h(2,)g(MPI)p 622 2363 V 16 w(INT)17 b Fs(is)g(the)g(bu\013er)g (comp)q(onen)o(t)g(with)g(en)o(tries)g Fl(A\(1\),)24 b(A\(2\),)f(A\(1\),)189 2420 y(A\(2\),)g(A\(1\))p Fs(.)143 2509 y Fo(\017)g Fl(A\(10\),)g(5,)g(-1,)h(2,)f(MPI)p 669 2509 V 17 w(INT)g Fs(is)h(the)g(bu\013er)g(comp)q(onen)o(t)g(with)g (en)o(tries)g Fl(A\(10\),)f(A\(11\),)189 2565 y(A\(9\),)g(A\(10\),)g (A\(8\))p Fs(.)166 2647 y Fl(Contiguous)18 b Fs(is)h(a)g(sp)q(ecial)i (case)e(of)g Fl(vector)p Fs(.)30 b(A)19 b(v)o(ector)g(bu\013er)g(comp)q (onen)o(t)g(can)g(b)q(e)h(used)g(to)75 2704 y(extract)14 b(an)i(arbitrary)e(submatrix)h(of)g(a)g(t)o(w)o(o-dimensional)h (matrix.)p eop %%Page: 12 17 12 16 bop 75 -100 a Fs(12)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Ft(Indexed)h(comp)q (onen)o(t)75 135 y Fs(A)e(sequence)i(of)d(elemen)o(ts)i(of)f(the)g (same)g(basic)h(t)o(yp)q(e,)f(sp)q(eci\014ed)i(b)o(y)75 237 y Ft(start)23 b Fs(initial)17 b(elemen)o(t)75 339 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fl(count)c Fo(\025)h Fl(0)p Fs(\))75 442 y Ft(arra)o(y)p 199 442 16 2 v 18 w(of)p 259 442 V 19 w(indices)23 b Fs(Arra)o(y)11 b(of)g(displacemen)o(ts)i(of)e(the)h(elemen)o(ts)g(in)g (the)g(bu\013er)f(comp)q(onen)o(ts,)h(relativ)o(e)189 498 y(to)i(the)h(initial)j(elemen)o(t.)75 600 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 702 y(F)l(or)c(example,)h(if) g Fl(A)f Fs(is)h(an)f(arra)o(y)f(of)h(in)o(tegers,)h(and)f Fl(I)h Fs(is)f(an)h(arra)o(y)e(with)h(v)m(alues)i Fl(2,)23 b(-1,)h(2,)f(1)12 b Fs(then)75 759 y Fl(A\(3\),)23 b(4,)h(I,)f(MPI)p 436 759 15 2 v 17 w(INT)15 b Fs(is)h(the)f(bu\013er)g(comp)q(onen)o(t)g (with)h(en)o(tries)f Fl(A\(5\),)24 b(A\(2\),)f(A\(5\),)g(A\(4\))p Fs(.)166 817 y Fl(Vector)10 b Fs(can)h(b)q(e)h(though)o(t)e(of)h(as)f (a)h(sp)q(ecial)i(case)e(of)f Fl(indexed)g Fs(in)i(whic)o(h)g(the)f (indices)i(are)e(generated)75 874 y(with)16 b(a)e(constan)o(t)h (stride.)166 1015 y Fk(Discussion:)41 b Fj(The)16 b(ab)q(o)o(v)o(e)g (statemen)o(t)f(is)h(not)f(quite)h(true,)h(since)f(there)h(is)f(no)f Ff(lenblk)g Fj(parameter)g(on)75 1071 y(the)f(indexed)f(comp)q(onen)o (t.)k(It)c(w)o(as)g(suggested)h(that)g(indexed)f(will)f(pro)o(vide)g(a) h(list)g(of)f(\(p)q(osition,)g(blo)q(c)o(k)p 1733 1071 13 2 v 15 w(length\))75 1128 y(pairs.)75 1343 y Ft(Heterogeneous)17 b(v)o(ector)g(\(h)o(v)o(ector\))g(comp)q(onen)o(t)75 1432 y Fs(Same)f(as)g Fl(vector)p Fs(,)f(except)i(that)f Fl(stride)f Fs(is)i(giv)o(en)g(in)g(m)o(ultiple)h(of)e(b)o(ytes,)f (rather)h(than)g(m)o(ultiple)i(of)75 1489 y(elemen)o(ts:)75 1606 y Ft(start)23 b Fs(Initial)17 b(elemen)o(t)75 1708 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fl(count)c Fo(\025)h Fl(0)p Fs(\))i(\(in)o(teger\))75 1810 y Ft(stride)23 b Fs(Num)o(b)q(er)15 b(of)g Ft(b)o(ytes)f Fs(b)q(et)o(w)o(een)i(the)f(start)f(of)h(eac)o(h)g(blo)q(c)o(k)h(\(in)o (teger\))75 1912 y Ft(len)o(blk)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(in)g(eac)o(h)g(blo)q(c)o(k)f(\()p Fl(lenblk)d Fr(>)h Fl(0)p Fs(\))i(\(in)o(teger\))75 2014 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)h (\(state\))166 2131 y(Assume)11 b(that)f(an)h(elemen)o(t)h(of)e(t)o(yp) q(e)h Fl(datatype)f Fs(o)q(ccupies)i Fl(m)f Fs(b)o(ytes.)18 b(Then,)12 b(in)g(a)e(b)o(yte)h(addressable)75 2188 y(mac)o(hine,)21 b(the)f(bu\013er)f(comp)q(onen)o(t)h(describ)q(ed)h(b)o(y)e(these)h (\014v)o(e)g(parameters)e(consists)i(of)f(the)h Fl(count)75 2244 y Fs(v)m(ariables)j(at)e(addresses)h Fl(start,)h(start+m,)f(...)48 b(,)24 b(start+m*\(lenblk-1\),)d(start+stride,)75 2300 y(start+stride+m,)h(...,)h(start+stride+m*\(lenblk-1\),)d(...)p Fs(.)166 2359 y(Consider)c(a)f(C)g(arra)o(y)e(declared)k(as)75 2478 y Fl(struct)23 b({)147 2534 y(char)g(info[3];)147 2591 y(short)g(number;)147 2647 y(double)g(val;)242 2704 y(})71 b(a[100])p eop %%Page: 13 18 13 17 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(13)166 45 y(Assume)18 b(that)f(all)i(structure)e (comp)q(onen)o(ts)h(are)g(b)o(yte)g(aligned,)h(a)f Fl(short)f Fs(o)q(ccupies)i(t)o(w)o(o)e(b)o(ytes,)75 102 y(a)e Fl(double)g Fs(o)q(ccupies)i(eigh)o(t)f(b)o(ytes,)f(and)h(no)g(padding)g(is)h (needed.)22 b(Then)16 b(the)g(bu\013er)g(comp)q(onen)o(t)f Fl(&a,)75 158 y(100,)23 b(13,)h(2,)f(MPI)p 436 158 15 2 v 17 w(CHAR)13 b Fs(extracts)f(the)i(\014rst)f(t)o(w)o(o)f(c)o (haracters)g(of)h(eac)o(h)g(structure.)19 b(The)14 b(bu\013er)f(com-)75 214 y(p)q(onen)o(t)i Fl(&\(a.number\),)22 b(100,)i(13,)f(1,)h(MPI)p 898 214 V 17 w(SHORT)14 b Fs(extracts)g(the)h Fl(short)f Fs(\014eld)i(of)f(eac)o(h)g(structure;)75 271 y(and)h(the)f(bu\013er)g (comp)q(onen)o(t)h Fl(&\(a.val\),)22 b(100,)i(13,)f(1)15 b Fs(extracts)g(the)g Fl(double)g Fs(\014eld)i(of)e(eac)o(h)g(struc-)75 327 y(ture.)30 b(Th)o(us,)18 b(heterogeneous)h(v)o(ectors)f(can)g(b)q (e)h(used)g(to)f(extract)g(v)o(ectors)f(of)i(homogeneous)f(v)m(alues)75 384 y(from)c(heterogeneous)i(data.)166 440 y Fl(Vector)e Fs(is)i(a)f(sp)q(ecial)i(case)e(of)g Fl(heterogeneous)22 b(vector)p Fs(.)75 559 y Ft(Heterogeneous)17 b(indexed)h(\(hindexed\))g (comp)q(onen)o(t)75 645 y Fs(Same)e(as)f(indexed,)i(except)g(that)e (displacemen)o(ts)i(are)f(in)g(b)o(ytes,)f(rather)h(than)f(m)o (ultiples)j(of)d(elemen)o(t)75 701 y(size.)75 799 y Ft(start)23 b Fs(initial)17 b(elemen)o(t)75 890 y Ft(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(\()p Fl(count)c Fo(\025)h Fl(0)p Fs(\))75 981 y Ft(arra)o(y)p 199 981 16 2 v 18 w(of)p 259 981 V 19 w(indices)23 b Fs(Arra)o(y)17 b(of)g(b)o(yte)h(displacemen)o(ts)h(of)e(the)h(elemen)o(ts)g(in)h(the)f (bu\013er)f(comp)q(onen)o(ts,)189 1038 y(relativ)o(e)e(to)g(the)g (initial)j(elemen)o(t.)75 1129 y Ft(datat)o(yp)q(e)24 b Fs(T)o(yp)q(e)15 b(of)g(elemen)o(ts)166 1226 y(F)l(or)21 b(example,)k(if)d Fl(a)g Fs(is)h(the)f(C)g(arra)o(y)e(of)i(the)g (previous)h(example,)h(and)e Fl(I)g Fs(is)h(an)f(arra)o(y)f(with)75 1283 y(v)m(alues)14 b Fl(0,)24 b(13,)f(13,)h(27,)f(26)13 b Fs(then)h Fl(&a,)23 b(5,)h(I,)f(MPI)p 1042 1283 15 2 v 17 w(CHAR)13 b Fs(is)g(the)h(bu\013er)f(comp)q(onen)o(t)g(with)h (en)o(tries)75 1339 y Fl(a[0].info[0],)22 b(a[1].info[0],)g (a[1].info[0],)g(a[1].info[1],)g(a[1].info[0])p Fs(.)166 1395 y(Lik)o(e)e Fl(heterogeneous)i(vector)p Fs(,)e Fl(heterogeneous)i (indexed)d Fs(extends)h(the)f Fl(indexed)g Fs(facilit)o(y)75 1452 y(so)c(as)g(to)f(allo)o(w)i(the)f(extraction)g(of)g(homogeneous)g (data)f(from)h(a)g(heterogeneous)g(la)o(y)o(out.)166 1584 y Fk(Discussion:)166 1634 y Fj(The)10 b(use)g(of)f(displacemen)o (ts)g(in)g(indexed)h(bu\013er)h(comp)q(onen)o(ts)e(has)h(the)g(same)f (e\013ect)i(as)e(indirect)h(addressing)75 1684 y(in)o(to)j(an)g(arra)o (y)h(with)f(initial)e(index)j(zero.)19 b(I.e.,)12 b(if)h Ff(a)h Fj(is)f(an)g(arra)o(y)h(with)f(initial)f(index)h(zero,)h(then)g (a)g(bu\013er)g(com-)75 1734 y(p)q(onen)o(t)j(de\014ned)h(b)o(y)e (parameters)g Ff(a[0],)21 b(n,)h(i,)f(type)16 b Fj(extract)h(the)g (elemen)o(ts)g Ff(a[i[0]],)j(a[i[1]],)g(...,)75 1784 y(a[i[n]])p Fj(.)25 b(This)16 b(\014ts)h(w)o(ell)f(C)h(usage)g(but)f (ill-\014ts)g(F)m(ortran)g(usage)h(\(initial)e(index)i(one\).)26 b(W)m(e)16 b(to)q(ok)g(a)h(decision)75 1833 y(of)c(principle)h(to)g (force)g(\\start)g(from)e(zero")j(index)f(arithmetic)f(on)g(F)m(ortran) h(users,)h(rather)f(than)g(ha)o(v)o(e)g(di\013eren)o(t)75 1883 y(index)g(arithmetic)f(in)g(the)i(C)e(and)h(F)m(ortran)g(binding)f (of)g(MPI.)166 1933 y(The)i(use)h(of)e(b)q(oth)h(bu\013er)h(comp)q (onen)o(ts)f(with)g(b)o(yte)g(and)g(elemen)o(t)f(size)i(displacemen)o (ts)e(is)h(motiv)n(ated)e(b)o(y)75 1983 y(the)j(observ)n(ation)f(that)g (some)g(applications)f(require)i(the)g(more)f(general)g(b)o(yte)h (displacemen)o(t,)e(but)i(that)f(most)75 2033 y(applications)d(w)o (ould)h(use)i(elemen)o(t)e(size)h(displacemen)o(t,)e(whic)o(h)i(is)f(m) o(uc)o(h)f(more)h(natural)g(and)g(con)o(v)o(enien)o(t)h(\(and)75 2083 y(mac)o(hine)i(indep)q(enden)o(t\).)29 b(Th)o(us,)18 b(rather)g(than)g(forcing)e(all)g(users)j(to)e(do)g(b)o(yte)h (arithmetics,)e(w)o(e)i(decided)g(to)75 2132 y(pro)o(vide)c(separately) g(the)h(more)e(adv)n(anced)h(function)f(to)h(\\exp)q(ert)h (programmers".)75 2336 y Fi(2.5.2)55 b(Bu\013er)18 b(op)r(erations)75 2421 y Fs(A)e(bu\013er)g(is)h(describ)q(ed)h(b)o(y)e(an)g(opaque)g Ft(bu\013er)i(descriptor)e Fs(ob)s(ject)g(accessed)g(via)h(a)f Ft(bu\013er)h(han-)75 2478 y(dle)p Fs(.)i(Suc)o(h)13 b(an)f(ob)s(ject)f(is)i(created)f(b)o(y)g Fl(MPI)p 809 2478 V 17 w(CREATE)p 970 2478 V 16 w(BUFFER)p Fs(.)f(It)h(is)g(asso)q (ciated)g(with)h(successiv)o(e)g(bu\013er)75 2534 y(comp)q(onen)o(ts)i (b)o(y)g(calling)i(in)f(succession)h(one)e(of)g(the)g(functions)h Fl(MPI)p 1285 2534 V 17 w(APPEND)p 1446 2534 V 16 w(xxx)p Fs(,)e(where)i Fl(xxx)f Fs(iden-)75 2591 y(ti\014es)20 b(the)e(t)o(yp)q(e)h(of)g(bu\013er)g(comp)q(onen)o(t)g(app)q(ended)h (to)f(the)f(bu\013er.)31 b(It)19 b(b)q(ecomes)h(a)o(v)m(ailable)g(for)f (use)75 2647 y(in)h(comm)o(unication)g(op)q(erations)g(once)f (committed)h(b)o(y)f(the)h(function)g Fl(MPI)p 1425 2647 V 17 w(COMMIT)p 1586 2647 V 16 w(BUFFER)p Fs(.)e(Once)75 2704 y(committed,)d(the)g(descriptor)h(cannot)f(b)q(e)h(mo)q(di\014ed)g (an)o(y)f(more.)p eop %%Page: 14 19 14 18 bop 75 -100 a Fs(14)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fs(Note)f(that)f (the)h(commit)g(op)q(eration)g(do)q(es)h(not)f(create)f(y)o(et)h(a)g (message,)f(and)h(do)q(es)h(not)e(bind)j(the)75 102 y(data)11 b(to)h(b)q(e)h(sen)o(t.)18 b(It)12 b(is)h(a)f(commit)g(for)f(the)h (bu\013er)g(descriptor,)h(not)f(a)f(commit)h(for)g(the)g(bu\013er)g (con)o(ten)o(t.)166 159 y(A)19 b(bu\013er)g(descriptor)g(is)h(destro)o (y)o(ed)e(after)h(the)g(completion)h(of)e(the)i(\014rst)e(comm)o (unication)i(op-)75 216 y(eration)e(that)g(uses)h(it,)g(if)g(it)g(is)g (ephemeral,)h(or)e(after)g(it)h(is)g(freed)g(b)o(y)f(a)g(call)i(to)e Fl(MPI)p 1585 216 15 2 v 17 w(FREE)g Fs(and)g(an)o(y)75 272 y(p)q(ending)g(comm)o(unication)f(op)q(eration)f(that)g(uses)g(it)h (has)f(completed,)h(if)g(it)f(is)h(p)q(ersisten)o(t.)23 b(After)16 b(the)75 329 y(bu\013er)f(descriptor)h(ob)s(ject)e(has)h(b)q (een)i(destro)o(y)o(ed,)d(the)h(handle)i(is)f(unde\014ned.)166 422 y Ft(MPI)p 275 422 16 2 v 18 w(CREA)l(TE)p 510 422 V 19 w(BUFFER\()i(bu\013er,)e(p)q(ersistence)i(\))166 515 y Fs(Create)d(a)f(new)i(bu\013er)f(descriptor)h(ob)s(ject.)j(The)c (parameters)g(are:)75 613 y Ft(OUT)j(bu\013er)k Fs(bu\013er)15 b(handle)75 711 y Ft(IN)i(p)q(ersistence)23 b Fs(bu\013er)g(p)q (ersistence)h(\(state)e(t)o(yp)q(e)h(with)g(p)q(ossible)h(v)m(alues)g Fl(MPI)p 1559 711 15 2 v 17 w(PERSISTENT)d Fs(or)189 768 y Fl(MPI)p 264 768 V 16 w(EPHEMERAL)p Fs(\).)166 942 y Fk(Discussion:)166 1000 y Fj(F)m(unction)13 b(really)h(is)f Ff(create)p 630 1000 14 2 v 15 w(buffer)p 777 1000 V 14 w(descriptor)p Fj(.)j(Ma)o(y)e(w)o(an)o(t)f(to)h(c)o(hange)g(the)h (name.)166 1175 y Ft(MPI)p 275 1175 16 2 v 18 w(APPEND)p 520 1175 V 18 w(CONTIGUOUS\()k(bu\013er,)d(start,)h(coun)o(t,)g(datat)o (yp)q(e\))166 1268 y Fs(App)q(end)g(a)e(con)o(tigous)g(comp)q(onen)o(t) g(to)f(bu\013er.)20 b(The)c(parameters)e(are:)75 1367 y Ft(INOUT)k(bu\013er)k Fs(bu\013er)15 b(handle)75 1465 y Ft(IN)i(start)23 b Fs(bu\013er)15 b(comp)q(onen)o(t)h(initial)h (elemen)o(t)f(\(c)o(hoice\))75 1563 y Ft(IN)h(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 1662 y Ft(IN)i(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 1795 y Ft(MPI)p 275 1795 V 18 w(APPEND)p 520 1795 V 18 w(VEC\()h(bu\013er,)f(start,)i(coun) o(t,)f(stride,)g(len)o(blk,)g(datat)o(yp)q(e)h(\))166 1888 y Fs(App)q(end)f(a)e(v)o(ector)f(bu\013er)h(comp)q(onen)o(t)g(to)g (bu\013er.)20 b(The)15 b(parameters)g(are:)75 1987 y Ft(INOUT)j(bu\013er)k Fs(bu\013er)15 b(handle)75 2085 y Ft(IN)i(start)23 b Fs(bu\013er)15 b(comp)q(onen)o(t)h(initial)h (elemen)o(t)f(\(c)o(hoice\))75 2183 y Ft(IN)h(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 2282 y Ft(IN)i(stride)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(b)q (et)o(w)o(een)g(the)f(start)f(of)h(eac)o(h)g(blo)q(c)o(k)h(\(in)o (teger\))75 2380 y Ft(IN)h(len)o(blk)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(in)g(eac)o(h)g(blo)q(c)o(k)g(\(in)o(teger\))75 2478 y Ft(IN)h(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 2653 y Fk(Discussion:)166 2704 y Fj(Whic)o(h)d(order)i(is)f(more)f(natural:)k Ff(stride,)k (lenblk)12 b Fj(or)i Ff(lenblk,)20 b(stride)p Fj(?)p eop %%Page: 15 20 15 19 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(15)166 45 y Ft(MPI)p 275 45 16 2 v 18 w(APPEND)p 520 45 V 18 w(INDEXED\()13 b(bu\013er,)g(start,)g(coun) o(t,)h(arra)o(y)p 1395 45 V 17 w(of)p 1454 45 V 19 w(indices,)h(datat)o (yp)q(e\))166 194 y Fs(App)q(end)i(an)e(indexed)i(bu\013er)e(comp)q (onen)o(t)g(to)g(bu\013er.)20 b(The)15 b(parameters)f(are:)75 288 y Ft(INOUT)k(bu\013er)k Fs(bu\013er)15 b(handle)75 382 y Ft(IN)i(start)23 b Fs(initial)17 b(p)q(osition)g(for)d(indexing)j (\(c)o(hoice\))75 476 y Ft(IN)g(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 570 y Ft(IN)i(arra)o(y)p 277 570 V 18 w(of)p 337 570 V 19 w(indices)24 b Fs(arra)o(y)14 b(of)g(displacemen)o(ts)j(of)e(en)o(tries)g(relativ)o(e)h(to)f(start)f (\(arra)o(y)f(of)i(in)o(tegers\))75 664 y Ft(IN)i(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 758 y(The)j(index)h(v)m(alues)g(used)g(to)e(create)h(this)g(comp)q (onen)o(t)g(are)g(the)g(v)m(alues)h(pro)o(vided)g(at)e(the)h(call;)75 815 y(subsequen)o(t)d(up)q(dates)g(of)e(the)i(arra)o(y)e(of)g(indices)k (will)e(not)f(a\013ect)g(the)g(bu\013er)g(descriptor)h(comp)q(onen)o (t.)166 907 y Ft(MPI)p 275 907 V 18 w(APPEND)p 520 907 V 18 w(HVEC\()h(bu\013er,)f(start,)h(coun)o(t,)h(stride,)f(len)o(blk,)g (datat)o(yp)q(e)h(\))166 999 y Fs(App)q(end)f(a)e(heterogeneous)g(v)o (ector)f(bu\013er)h(comp)q(onen)o(t)h(to)e(bu\013er.)20 b(The)c(parameters)e(are:)75 1093 y Ft(INOUT)k(bu\013er)k Fs(bu\013er)15 b(handle)75 1187 y Ft(IN)i(start)23 b Fs(bu\013er)15 b(comp)q(onen)o(t)h(initial)h(elemen)o(t)f(\(c)o (hoice\))75 1281 y Ft(IN)h(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 1376 y Ft(IN)i(stride)23 b Fs(Num)o(b)q(er)16 b(of)e(b)o(ytes)h(b)q(et)o(w)o(een)h(the)f(start)f (of)h(eac)o(h)g(blo)q(c)o(k)h(\(in)o(teger\))75 1470 y Ft(IN)h(len)o(blk)23 b Fs(Num)o(b)q(er)16 b(of)e(elemen)o(ts)i(in)g (eac)o(h)g(blo)q(c)o(k)g(\(in)o(teger\))75 1564 y Ft(IN)h(datat)o(yp)q (e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 1693 y Ft(MPI)p 275 1693 V 18 w(APPEND)p 520 1693 V 18 w(HINDEXED\()c(bu\013er,)f(start,)i(coun)o(t,)g(arra)o(y)p 1432 1693 V 17 w(of)p 1491 1693 V 19 w(indices,)g(datat)o(yp)q(e\))166 1842 y Fs(App)q(end)k(an)e(heterogeneous)g(indexed)i(bu\013er)e(comp)q (onen)o(t)g(to)g(bu\013er.)20 b(The)15 b(parameters)g(are:)75 1936 y Ft(INOUT)j(bu\013er)k Fs(bu\013er)15 b(handle)75 2030 y Ft(IN)i(start)23 b Fs(initial)17 b(p)q(osition)g(for)d(indexing) j(\(c)o(hoice\))75 2124 y Ft(IN)g(coun)o(t)23 b Fs(Num)o(b)q(er)16 b(of)f(elemen)o(ts)g(\(in)o(teger\))75 2219 y Ft(IN)i(arra)o(y)p 277 2219 V 18 w(of)p 337 2219 V 19 w(indices)24 b Fs(arra)o(y)15 b(of)i(b)o(yte)f(displacemen)o(ts)i(of)f(en)o(tries)g(relativ)o(e)h(to) e(start)f(\(arra)o(y)h(of)g(in-)189 2275 y(tegers\))75 2369 y Ft(IN)h(datat)o(yp)q(e)24 b Fs(datat)o(yp)q(e)15 b(iden)o(ti\014er)h(\(status\))166 2499 y Ft(MPI)p 275 2499 V 18 w(COMMIT)p 526 2499 V 18 w(BUFFER\()j(bu\013er)d(\))166 2591 y Fs(Commit)j(a)g(bu\013er)g(descriptor)h(ob)s(ject.)31 b(The)19 b(ob)s(ject)g(cannot)g(b)q(e)h(c)o(hanged)g(an)o(ymore)e (after)h(it)75 2647 y(is)h(committed.)31 b(A)19 b(bu\013er)g (descriptor)g(ob)s(ject)g(can)g(b)q(e)h(used)f(for)g(comm)o(unication)g (only)h(after)e(it)h(is)75 2704 y(committed.)h(The)15 b(parameters)g(are:)p eop %%Page: 16 21 16 20 bop 75 -100 a Fs(16)613 b Fq(CHAPTER)16 b(2.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Ft(INOUT)i(bu\013er) k Fs(bu\013er)15 b(descriptor)h(handle)166 216 y Fk(Discussion:)166 266 y Fj(The)j(commit)d(function)j(ma)o(y)e(do)h(prepro)q(cessing)j(of) d(the)i(bu\013er)g(descriptor)g(to)f(simplify)d(its)i(use)i(and)75 316 y(commit)12 b(resources)17 b(in)d(the)i(comm)o(uni)o(cation)c (subsystem)j(for)f(its)h(use.)21 b(The)16 b(commit)11 b(function)k(ma)o(y)e(generate)75 365 y(a)h(new)g(ob)r(ject)h(and,)e (hence,)i(mo)q(dify)c(the)k(ob)r(ject)f(handle.)166 416 y(The)i(curren)o(t)i(prop)q(osal)e(do)q(es)g(not)g(allo)o(w)f(to)h (create)h(a)f(bu\013er)h(descriptor,)g(use)g(it)f(in)g(a)f(comm)o (unication)75 465 y(after)k(committed,)f(then)i(mo)q(dify)d(it)i (partially)e(and)i(use)h(it)f(for)g(another)h(comm)o(unicatio)o(n)c({)j (a)g(new)h(bu\013er)75 515 y(descriptor)e(has)f(to)g(b)q(e)h(created.) 29 b(Of)16 b(course,)j(the)f(same)e(bu\013er)i(descriptor)g(can)f(b)q (e)h(reused)g(man)o(y)d(times)h(in)75 565 y(successiv)o(e)k(comm)o (unications)15 b(that)j(send)h(data)f(from)e(the)j(same)e(bu\013ers;)k (e.g.,)d(a)g(new)g(send)h(with)f(a)g(bu\013er)75 615 y(descriptor)d(handle)f(can)g(b)q(e)h(initiated)e(once)h(the)h (previous)f(send)h(with)e(this)h(handle)g(has)g(completed.)166 754 y Fs(Example)i(1:)75 811 y(Consider)g(the)f(follo)o(wing)h(fragmen) o(t)e(of)h(F)l(ortran)f(co)q(de)75 918 y Fl(DOUBLE)23 b(PRECISION)g(A\(10,20\))75 975 y(INTEGER)g(B,)g(C\(5,10\),)g(I\(3\))75 1031 y(INTEGER)g(BH)75 1088 y(DATA)g(\(I\(J\),)g(J=1,3\))g(/9,2,1/)75 1144 y(...)75 1200 y(CALL)g(MPI_CREATE_BUFFER\(BH,)e(MPI_PERSISTENT\)) 75 1257 y(CALL)i(MPI_APPEND_CONTIGUOUS)e(\(BH,)j(B,)f(1,)h (MPI_INTEGER\))75 1313 y(CALL)f(MPI_APPEND_VEC)f(\(BH,)i(A\(1,3\),)e (11,)i(4,)g(2,)f(MPI_DOUBLE\))75 1370 y(CALL)g(MPI_APPEND_INDEXED\(BH,) e(C\(3,7\),)i(3,)h(I,)f(MPI_INTEGER\))166 1477 y Fs(Then)16 b(the)f(bu\013er)g(asso)q(ciated)g(with)h(the)f(handle)i Fl(BH)e Fs(consists)g(of)g(the)g(sequence)h(of)f(v)m(ariables)166 1534 y Fl(B,)23 b(A\(1,3\),)g(A\(2,3\),)g(A\(5,3\),)g(A\(6,3\),)g (A\(9,3\),)g(A\(10,3\),)g(A\(3,4\),)g(A\(4,4\),)g(A\(7,4\),)75 1590 y(A\(8,4\),)g(A\(1,5\),)g(C\(2,8\),)g(C\(5,7\),)g(C\(4,7\))p Fs(.)166 1647 y(A)17 b(message)g(created)g(from)f(this)i(bu\013er)f (will)i(consist)e(of)g(a)g(sequence)h(of)f(one)g(in)o(teger,)h(follo)o (w)o(ed)75 1703 y(b)o(y)d(elev)o(en)h(double)h(precision)g(reals,)e (follo)o(w)o(ed)g(b)o(y)g(three)h(in)o(tegers.)166 1760 y(Example)g(2:)75 1817 y(Supp)q(ose)g(one)g(has)f(an)g(arra)o(y)f(of)h (data)f(declared)i(in)h(C)d(as)75 1924 y Fl(struct)23 b({)147 1981 y(int)g(type;)147 2037 y(double)g(position[3];)147 2093 y(double)g(velocity[3];)242 2150 y(})h(particle[1000])166 2257 y Fs(and)13 b(one)g(w)o(an)o(ts)e(to)i(transfer)f(in)h(one)g (message)g(the)g(n)o(um)o(b)q(er)g(of)f(particles)i(of)e(t)o(yp)q(e)h (1,)g(the)g(n)o(um)o(b)q(er)75 2314 y(of)g(particles)h(of)e(t)o(yp)q(e) i(2,)e(and)i(the)f(p)q(ositions)h(of)f(all)h(the)f(particles)h(of)f (these)g(t)o(w)o(o)f(t)o(yp)q(es.)19 b(The)13 b(follo)o(wing)75 2370 y(co)q(de)j(fragmen)o(t)e(creates)h(a)g(suitable)h(bu\013er)f (descriptor.)75 2478 y Fl(int)23 b(count1)g(=)h(0;)75 2534 y(int)f(count2)g(=)h(0;)75 2591 y(...)75 2647 y (mpi_create_buffer\()d(&handle,)i(MPI_PERSISTENT\);)75 2704 y(mpi_append_contiguous\()e(handle,)i(\(void)g(*\)&count1,)f (MPI_INT\);)p eop %%Page: 17 22 17 21 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(17)75 45 y Fl(mpi_append_contiguous\()21 b(handle,)i(\(void)g(*\)&count2,)f(MPI_INT\);)75 102 y(for)h(\(i=0;)h(i<1000;)f(i++\))g({)147 158 y(if)g(\(particle[i].type) f(==)h(1\))h({)218 214 y(count1++;)218 271 y(mpi_append_contiguous\()d (&handle,)i(\(void)g(*\)&particle[i].position,)e(3,)75 327 y(MPI_DOUBLE\);)h(})147 384 y(elseif)h(\(particle[i].type)e(==)j (2\))f({)218 440 y(count2++;)218 497 y(mpi_append_contiguous\()e (&handle,)i(\(void)g(*\)&particle[i].position,)e(3,)75 553 y(MPI_DOUBLE\);)h(})147 610 y(})75 666 y (mpi_commit_buffer\(handle\))75 723 y(...)166 847 y Fs(The)15 b(bu\013er)g(descriptor)g(handle)h(can)f(no)o(w)f(b)q(e)i(used)f(to)g (send)g(\(rep)q(eatedly\))g(the)g(particle)h(coun)o(ts)75 903 y(for)i(t)o(yp)q(e)h(1)g(and)g(2,)g(and)g(the)g(p)q(ositions)h(of)f (the)g(particles)h(with)f(these)g(t)o(w)o(o)f(t)o(yp)q(es,)i(as)e(long) h(as)g(the)75 960 y(t)o(yp)q(es)c(of)g(the)g(particles)h(is)g(not)f(mo) q(di\014ed.)166 1020 y(Supp)q(ose)h(the)f(incoming)i(data)d(is)i(to)e (b)q(e)i(stored)f(at)f(the)i(receiv)o(er)g(in)o(to)f(the)g(t)o(w)o(o)f (coun)o(t)h(v)m(ariables)75 1076 y Fl(count1)e Fs(and)i Fl(count2)e Fs(and)i(in)o(to)f(the)h Fl(position)e Fs(comp)q(onen)o(ts) h(of)g(the)h(initial)h(en)o(tries)f(of)f(the)g(arra)o(y)f(of)75 1133 y(particles.)20 b(The)13 b(follo)o(wing)h(co)q(de)g(generates)e(a) h(suitable)h(bu\013er)f(descriptor)g(for)g(the)g(receiv)o(e)h(op)q (eration)75 1189 y(\(w)o(e)h(assume)g(4)g(b)o(ytes)f(for)h(in)o(teger)g (and)h(8)f(b)o(ytes)g(for)f(double\).)75 1317 y Fl(...)75 1374 y(mpi_create_buffer\()21 b(&handle,)i(MPI_PERSISTENT\);)75 1430 y(mpi_append_contiguous\()e(&handle,)i(\(void)g(*\)&count1,)f (MPI_INT\);)75 1487 y(mpi_append_contiguous\()f(handle,)i(\(void)g (*\)&count2,)f(MPI_INT\);)75 1543 y(mpi_append_hvec\()g(handle,)h (\(void)g(*\)&particle[0].position,)d(1000,)k(52,)f(3,)75 1600 y(MPI_DOUBLE\);)f(mpi_commit_buffer\()g(handle\))75 1656 y(...)166 1780 y Fs(A)17 b(bu\013er)g(handle)h(can)f(b)q(e)h(used) g(for)e(comm)o(unication,)i(ev)o(en)f(if)g(it)h(is)f(not)g(asso)q (ciated)g(with)g(an)o(y)75 1837 y(v)m(ariables)12 b(\(i.e.,)f(ev)o(en)g (if)h(it)f(w)o(as)f(not)g(set)h(b)o(y)f(an)o(y)h Fl(MPI)p 978 1837 15 2 v 17 w(APPEND)p 1139 1837 V 16 w(xxx)f Fs(call\).)19 b(Suc)o(h)12 b(a)e(handle)i(is)g(asso)q(ciated)75 1893 y(with)19 b(an)g(empt)o(y)f(bu\013er,)i(and)f(a)f(message)h (created)f(from)h(it)g(con)o(tains)g(no)f(data.)30 b(The)19 b(handle)i(still)75 1950 y(need)16 b(b)q(e)g(committed)f(b)q(efore)h (it)f(is)h(used.)166 2010 y(The)e(same)f(en)o(try)g(ma)o(y)g(app)q(ear) h(more)f(than)g(once)h(in)g(a)f(bu\013er)h(comp)q(onen)o(t,)g(and)f (distinct)i(bu\013er)75 2066 y(comp)q(onen)o(ts)h(ma)o(y)f(o)o(v)o (erlap.)23 b(If)16 b(a)g(bu\013er)g(descriptor)g(with)h(replicated)g (en)o(tries)g(is)f(used)h(to)e(receiv)o(e)i(a)75 2123 y(message,)12 b(then)h(the)g(\014nal)g(v)m(alue)h(of)e(an)g(en)o(try)g (that)g(o)q(ccurs)h(more)f(than)g(once)h(in)g(the)g(bu\013er)g (descriptor)75 2179 y(is)j(unde\014ned.)75 2322 y Fi(2.5.3)55 b(Comm)n(unication)17 b(Bu\013ers)h({)g(Extension)1928 2354 y Fg(TO)11 b(DISCUSS)75 2490 y Fk(Discussion:)166 2544 y Fj(James)16 b(Co)o(wnie)g(prop)q(oses)h(the)g(follo)o(wing)d (extension)j(to)f(bu\013er)h(descriptors.)27 b(Its)17 b(main)d(feature)j(is)f(the)75 2594 y(addition)10 b(of)g(a)g(bu\013er)i (comp)q(onen)o(t)e(that)h(allo)o(ws)f(for)g(the)i(e\016cien)o(t)f (represen)o(tation)h(and)f(manipulation)d(of)i(arra)o(ys)75 2644 y(of)j(C)h(structures.)21 b(This)13 b(extension)i(has)f(not)g(b)q (een)h(discussed)g(b)o(y)f(MPIF.)166 2704 y(\(The)g(original)e(prop)q (osal)i(has)g(b)q(een)h(repro)q(cessed)i(b)o(y)d(Rust)o(y)f(and)h (Marc.\))p eop %%Page: 18 23 18 22 bop 75 -100 a Fs(18)613 b Fq(CHAPTER)16 b(2.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fs(F)l(ortran)g (and)g(C)h(b)q(oth)g(allo)o(w)g(to)f(pro)o(vide)h(hierarc)o(hical)i (de\014nitions)f(of)f(comp)q(osite)g(data)f(struc-)75 102 y(tures;)g(on)g(the)h(other)e(hand,)i(bu\013er)f(descriptors)h(are) e(\015at)h(structures.)23 b(Th)o(us,)16 b(it)g(is)h(incon)o(v)o(enien)o (t)h(to)75 158 y(use)e(bu\013er)f(descriptors)g(in)h(order)f(to)g (transfer)f(records.)166 291 y Fk(Discussion:)166 347 y Fj(There)k(also)f(is)g(an)g(e\016ciency)h(consideration.)29 b(Consider)17 b(the)h(transfer)g(of)f(an)g(arra)o(y)g(of)g(structures,) j(as)75 403 y(in)d(the)g(example)f(giv)o(en)h(for)f(an)h Ff(hvector)f Fj(comp)q(onen)o(t.)26 b(The)18 b(arra)o(y)f(is)g(co)o(v)o (ered)h(with)e(strided)i(v)o(ectors)g(with)75 460 y(homogeneous)13 b(comp)q(onen)o(ts;)h(the)h(fact)g(that)f(the)h(en)o(tire)g(data)f (structure)j(is)d(con)o(tiguous)g(in)g(memory)e(is)i(lik)o(ely)75 516 y(to)g(b)q(e)g(lost.)166 655 y Fs(Bu\013er)h(descriptors)h(p)q (erform)f(t)o(w)o(o)e(functions)j(:)131 746 y(1.)22 b(They)17 b(describ)q(e)h(the)f(bu\013er)f Ft(t)o(yp)q(e)p Fs(,)h(i.e.)24 b(the)17 b(data)f(t)o(yp)q(e)h(of)f(eac)o(h)h(elemen)o(t)g(of)f(the)h (bu\013er,)g(and)189 803 y(the)e(relativ)o(e)h(displacemen)o(t)g(of)f (the)g(elemen)o(ts)h(in)g(memory)l(.)131 895 y(2.)22 b(They)15 b(describ)q(e)i(the)e(address)g(in)i(memory)d(of)h(eac)o(h)g (elemen)o(t)h(of)f(the)g(bu\013er.)166 986 y(Consider,)g(for)g (example,)h(the)f(follo)o(wing)h(fragmen)o(t)e(of)h(co)q(de.)75 1088 y Fl(int)23 b(a[10];)75 1145 y(...)75 1201 y (mpi_create_buffer\(&b1,)e(MPI_EPHEMERAL\);)75 1258 y (mpi_append_contiguous\(b1,)f(\(void)k(*\)a,)f(3,)h(MPI_INT\);)75 1314 y(mpi_create_buffer\(&b2,)d(MPI_EPHEMERAL\);)75 1370 y(mpi_append_contiguous\(b2,)f(\(void)k(*\)a,)f(3,)h(MPI_INT\);)75 1427 y(mpi_create_buffer\(&b3,)d(MPI_EPHEMERAL\);)75 1483 y(mpi_append_contiguous\(b3,)f(\(void)k(*\)\(a+2\),)e(3,)i (MPI_INT\);)166 1585 y Fs(Then)14 b Fl(b1)f Fs(and)g Fl(b2)g Fs(b)q(oth)h(p)q(oin)o(t)g(to)f(bu\013er)g(descriptors)h(for)e (the)i(same)f(bu\013er,)g(whereas)g Fl(b3)h Fs(p)q(oin)o(ts)75 1642 y(to)f(a)g(bu\013er)g(descriptor)h(for)f(another)g(bu\013er;)g(ho) o(w)o(ev)o(er,)g(all)h(three)g(bu\013er)f(descriptors)h(ha)o(v)o(e)f (the)g(same)75 1698 y(t)o(yp)q(e.)166 1755 y(In)h(order)e(to)h(pro)o (vide)g(a)g(recursiv)o(e)h(de\014nition)g(of)f(bu\013er)g(descriptors,) g(one)h(needs)f(an)g(ob)s(ject)g(that)75 1811 y(sp)q(eci\014es)j(the)e (bu\013er)g(t)o(yp)q(e,)g(but)g(not)f(its)i(lo)q(cation.)20 b(By)14 b(analogy)l(,)g(in)h(a)e(language)h(suc)o(h)h(as)e(C,)h(a)f (struct)75 1868 y(de\014nition)k(pro)o(vides)f(the)f(structure)g(t)o (yp)q(e,)g(and)g(its)g(declaration)h(pro)o(vides)g(\(implicitly\))i (the)d(address)75 1924 y(of)j(the)h(structure.)31 b(The)19 b(struct)f(de\014nition)i(can)f(b)q(e)h(used)f(recursiv)o(ely)h(as)f(a) f(comp)q(onen)o(t)h(of)f(a)h(new)75 1981 y(struct)c(de\014nition.)166 2037 y(Rather)c(then)h(adding)g(new)g(functions)g(and)f(new)h(ob)s (jects,)f(w)o(e)g(prop)q(ose)g(to)g(use)g(bu\013er)h(descriptors)75 2093 y(in)18 b(b)q(oth)g(roles:)25 b(\\b)q(ound",)18 b(to)f(indicate)i(the)f(t)o(yp)q(e)f(and)h(lo)q(cation)g(of)f(a)h (bu\013er,)f(and)h(\\un)o(b)q(ound",)h(to)75 2150 y(indicate)e(a)e(t)o (yp)q(e,)f(that)h(can)g(b)q(e)h(used)g(as)f(a)g(comp)q(onen)o(t)g(of)g (a)g(new)g(bu\013er)g(de\014nition.)166 2206 y(F)l(ormally)l(,)i(the)g Fl(MPI)p 523 2206 15 2 v 17 w(APPEND)p 684 2206 V 16 w(xxx)g Fs(functions)h(detailed)g(ab)q(o)o(v)o(e)f(tak)o(e)f(a)g (handle)j(to)d(a)h(previously)75 2263 y(de\014ned,)k(uncommitted)e (bu\013er)g(descriptor)h(as)e(their)i Fl(datatype)d Fs(argumen)o(t)h (\(rather)g(than)h(a)g(state)75 2319 y(v)m(ariable\).)25 b(The)17 b(\(comp)q(osite\))g(elemen)o(t)g(describ)q(ed)i(b)o(y)d(this) h(argumen)o(t)f(is)h(an)g(elemen)o(t)g(of)g(the)f(same)75 2376 y(t)o(yp)q(e)f(as)g(the)g(bu\013er)h(argumen)o(t.)166 2432 y(Example)g(2:)75 2534 y Fl(int)23 b(i[2])h(=)f({)h(2,)g(0};)75 2591 y(int)f(a[100];)75 2647 y(...)75 2704 y(mpi_create_buffer\()e (&b1,)j(MPI_PERSISTENT\);)p eop %%Page: 19 24 19 23 bop 75 -100 a Fq(2.5.)29 b(COMMUNICA)l(TION)16 b(BUFFERS)1006 b Fs(19)75 45 y Fl(mpi_append_indexed\()21 b(b1,)j(\(void)f(*\)a,)g(2,)h(i,)f(MPI_INT\);)75 158 y(mpi_create_buffer\()e(&b2,)j(MPI_PERSISTENT\);)75 214 y(mpi_append_contiguous\()d(b2,)i(\(void)g(*\)a,)h(3,)f(b1\);)75 327 y(mpi_create_buffer\()e(&b3,)j(MPI_PERSISTENT\);)75 384 y(mpi_append_indexed\()d(b3,)j(\(void)f(*\)a,)g(2,)h(i,)f(b2\);)75 440 y(mpi_append_vec\()f(b3,)h(\(void)g(*\)\(a+50\),)g(3,)h(3,)f(2,)h (b1\);)166 549 y(b1)15 b Fs(is)i(asso)q(ciated)f(with)g(a)f(bu\013er)h (consisting)g(of)g(the)f(t)o(w)o(o)g(elemen)o(ts)h Fl(a[2],)23 b(a[0])p Fs(.)e(When)16 b(used)75 606 y(un)o(b)q(ound,)25 b(it)d(indicates)h(a)f(comp)q(onen)o(t)g(with)h(t)o(w)o(o)d(in)o (tegers)j(that)e(are)h(one)g(in)o(teger)g(appart;)j(this)75 662 y(comp)q(onen)o(t)15 b(o)q(ccupies)i(three)e(in)o(teger)h(storage)e (units)i(\(12)e(b)o(ytes,)g(if)i(in)o(tegers)f(o)q(ccup)o(y)h(four)f(b) o(ytes\).)166 719 y(The)i(bu\013er)g(asso)q(ciated)g(with)g Fl(b2)g Fs(has)f(3)h(comp)q(onen)o(ts)g(of)f(the)h(t)o(yp)q(e)g (indicated)i(b)o(y)d Fl(b1)p Fs(.)25 b(It)17 b(con-)75 776 y(sists)d(of)g(the)h(arra)o(y)e(en)o(tries)i Fl(a[2],)23 b(a[0],)g(a[5],)g(a[3],)g(a[8],)h(a[6])p Fs(.)19 b(When)14 b(used)h(un)o(b)q(ound,)h(it)75 832 y(indicates)h(a)e(comp)q(onen)o(t)g (with)g(six)h(in)o(tegers)f(that)g(o)q(ccup)o(y)h(nine)g(in)o(teger)g (storage)d(units.)166 889 y(The)g(bu\013er)f(asso)q(ciated)h(with)f Fl(b3)h Fs(has)f(t)o(w)o(o)f(comp)q(onen)o(t)i(of)f(the)g(t)o(yp)q(e)h (indicated)h(b)o(y)e Fl(b2)g Fs(and)h(three)75 946 y(comp)q(onen)o(ts)i (of)g(the)g(t)o(yp)q(e)h(indicated)h(b)o(y)e Fl(b1)p Fs(.)k(It)c(consists)h(of)f(the)g(arra)o(y)f(en)o(tries)75 1042 y Fl(a[20],)23 b(a[18],)g(a[23],)g(a[21],)g(a[26],)g(a[24],)99 1099 y(a[2],)47 b(a[0],)g(a[5],)g(a[3],)g(a[8],)g(a[6],)75 1155 y(a[52],)23 b(a[50],)g(a[55],)g(a[53],)g(a[59],)g(a[57].)166 1251 y Fs(An)o(y)14 b(uncommitted)h(bu\013er)f(descriptor)h(ma)o(y)e(b) q(e)i(used)g(as)e(a)h(data)g(t)o(yp)q(e.)19 b(MPI)14 b(prede\014nes)i(\(han-)75 1307 y(dles)h(to\))f(uncommitted)h(bu\013er) f(descriptors)h Fl(MPI)p 959 1307 15 2 v 17 w(INT)p Fs(,)e Fl(MPI)p 1148 1307 V 17 w(FLOAT)p Fs(,)g(etc.,)h(for)g(eac)o(h)g(basic) h(datat)o(yp)q(e)75 1364 y(of)e(F)l(ortran)e(and)i(C.)g(Eac)o(h)g(of)f (these)h(descriptors)h(can)f(b)q(e)g(though)o(t)g(as)f(describing)j(a)e (bu\013er)g(with)g(one)75 1420 y(elemen)o(t)h(of)f(the)g(corresp)q (onding)h(t)o(yp)q(e.)166 1477 y(When)c(a)g(bu\013er)g(descriptor)g (isd)g(used)h(as)e(a)h(t)o(yp)q(e)g(descriptor)g(then)g(MPI)g(ensures)g (the)g(alignmen)o(ts)75 1534 y(required)k(b)o(y)f(the)h(underlying)h (arc)o(hitecture)e(and)h(add,)f(if)g(required,)h(padding)h(space.)166 1591 y(Example)f(3:)75 1647 y(Consider)g(the)f(follo)o(wing)h(C)f(co)q (de)h(whic)o(h)g(builds)h(a)e(bu\013er)g(descriptor)h(for)e(an)h(arra)o (y)f(of)h(structures.)75 1800 y Fl(/*)24 b(A)f(brain)g(damaged)g (structure)g(which)g(needs)g(both)h(internal)e(and)i(external)99 1857 y(*)g(padding)e(on)i(a)g(32)f(bit)h(machine)f(with)g(strict)g (alignment)g(requirements)99 1913 y(*/)75 1970 y(struct)g(foostruct)g ({)147 2026 y(int)g(i;)147 2083 y(/*)g(four)g(bytes)h(of)f(padding)g (to)h(get)f(back)h(to)f(double)g(alignment)g(*/)147 2139 y(double)g(d[5];)147 2195 y(char)g(b;)147 2252 y(/*)g(trailing)g(7)h (bytes)f(of)g(padding)g(to)h(enforce)f(alignment)f(again)i(*/)75 2308 y(};)75 2421 y(struct)f(foostruct)g(foo[100];)75 2534 y(MPI_BUFFER)f(t;)75 2591 y(MPI_BUFFER)g(fb;)75 2704 y(mpi_create_buffer\(&t,)45 b(MPI_EPHEMERAL\);)p eop %%Page: 20 25 20 24 bop 75 -100 a Fs(20)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fl (mpi_create_buffer\(&fb,)21 b(MPI_PERSISTENT\);)75 158 y(/*)j(Describe)e(the)i(data)f(type)g(in)h(t)g(*/)75 214 y(mpi_append_contiguous\(t,)d(&\(\(struct)h(foostruct)h (*\)0\)->i,1,MPI_INT\);)75 271 y(mpi_append_contiguous\(t,)e (&\(\(struct)h(foostruct)h(*\)0\)->d,5,MPI_DOUBLE\);)75 327 y(mpi_append_contiguous\(t,)e(&\(\(struct)h(foostruct)h (*\)0\)->b,1,MPI_BYTE\);)75 440 y(/*)h(Now)f(describe)g(foo)g(itself,)g (a)h(contiguous)e(array)i(of)f(100)h(elements)99 497 y(*)g(of)f(the)h(derived)f(type.)99 553 y(*/)75 610 y (mpi_append_contiguous\(fb,&)o(foo,100)o(,t\);)75 666 y(mpi_commit\(fb\);)166 764 y Fs(Example)16 b(4:)75 821 y(Consider)21 b(the)f(follo)o(wing)h(co)q(de)f(whic)o(h)h(expresses)g (as)f(a)f(single)j(bu\013er)e(descriptor)g(a)g(subset)g(of)g(an)75 877 y(arbitrary)15 b(dimensional)i(arra)o(y)l(.)170 977 y Fl(real)24 b(a\(100,100,100\))75 1033 y(c)71 b(declare)23 b(the)h(buffer)f(descriptors)170 1090 y(integer)g(onedslice)170 1146 y(integer)g(twodslice)170 1203 y(integer)g(message)75 1316 y(c)71 b(Slice)24 b(out)f(the)h(section)e(a\(1:21:2,1:10,1:11\))g (as)h(a)h(single)f(buffer)g(descriptor)170 1428 y(call)h (mpi_create_buffer\(ondesl)o(ice,)d(MPI_EPHEMERAL\))170 1485 y(call)j(mpi_create_buffer\(twodsl)o(ice,)d(MPI_EPHEMERAL\))170 1541 y(call)j(mpi_create_buffer\(messag)o(e,)69 b(MPI_EPHEMERAL\))170 1654 y(call)24 b(mpi_append_vec)e(\(onedslice,)g(a,)i(1+\(21-1\)/2,)147 1711 y($)501 b(2,)23 b(1,)h(MPI_REAL\))170 1767 y(call)g (mpi_append_hvec\(twodslic)o(e,)d(a,)215 b(10,)147 1824 y($)501 b(100*MPI_REAL_SIZE,)21 b(1,)j(ondeslice\))170 1880 y(call)g(mpi_append_hvec\(message,)68 b(a,)382 b(11,)147 1937 y($)501 b(100*100*MPI_REAL_SIZE,1)o(,twodsl)o(ice\))170 2049 y(call)24 b(mpi_commit\(message\))75 2162 y(c)71 b(all)24 b(done...)166 2337 y Fk(Discussion:)166 2388 y Fj(F)m(or)10 b(p)q(ortabilit)o(y)f(this)i(example)e(requires)j (access)h(to)d(the)h(size)h(in)e(b)o(ytes)h(of)f(an)g(MPI)h(data)g(t)o (yp)q(e.)17 b(\(In)11 b(general)75 2437 y(a)h(bu\013er)i(descriptor\).) k(W)m(e)12 b(should)h(probably)e(pro)o(vide)i(an)f Ff(MPI)p 1096 2437 14 2 v 15 w(SIZEOF\()20 b(bufhandle)g(\))12 b Fj(function)g(to)h(ac)o(hiev)o(e)75 2487 y(this.)166 2704 y Fk(Discussion:)p eop %%Page: 21 26 21 25 bop 75 -100 a Fq(2.5.)29 b(COMMUNICA)l(TION)16 b(BUFFERS)1006 b Fs(21)166 45 y Fj(W)m(e)14 b(should)g(probably)f (restrict)j(a)e(bu\013er)h(descriptor)g(that)g(is)f(used)h(as)f(a)g(t)o (yp)q(e)g(parameter)g(to)g(describ)q(e)i(a)75 95 y(bu\013er)g(that)f (is)g(con)o(tained)g(within)f(one)i(\(comp)q(osite\))e(v)n(ariable)g (\(arra)o(y)m(,)g(structure,)j(etc.\).)23 b(The)15 b(w)o(orry)g(is)g (that)75 145 y(on)c(a)g(mac)o(hine)g(with)g(a)g(segmen)o(ted)h(address) h(space)f(di\013eren)o(t)h(v)n(ariables)d(ma)o(y)g(not)h(b)q(elong)h (to)f(the)h(same)f(address)75 195 y(segmen)o(t,)16 b(so)g(that)g(their) g(relativ)o(e)g(displacemen)o(t)f(is)h(not)g(de\014ned)h(\(think)f(of)f (a)h(bu\013er)h(that)f(con)o(tains)g(a)g(heap)75 244 y(v)n(ariable,)c(a)i(stac)o(k)g(v)n(ariable)f(and)h(a)f(static)i(v)n (ariable\).)75 446 y Ft(Alternativ)o(e)j(within)g(alternativ)o(e)75 532 y Fs(A)k(v)m(arian)o(t)h(on)f(the)h(previous)g(prop)q(osal)g(is)g (to)e(k)o(eep)i(un)o(b)q(ound)h(bu\013er)e(descriptors)h(syn)o (tactically)75 588 y(distinct)12 b(from)e(b)q(ound)i(bu\013er)f (descriptors.)18 b(A)11 b(bu\013er)g(descriptor)g(is)h Ft(un)o(b)q(ound)f Fs(if)g(it)g(is)g(not)g(asso)q(ciated)75 645 y(with)18 b(a)g(\014xed)h(memory)f(address.)28 b(A)19 b(bu\013er)f(descriptor)g(can)h(b)q(e)g(used)f(as)g(a)g(datat)o(yp)q(e) f(if)i(and)f(only)75 701 y(if)g(it)g(is)g(un)o(b)q(ound.)28 b(This)18 b(prev)o(en)o(ts)f(the)h(p)q(ossible)h(confusion)f(arising)h (from)d(using)j(the)e(same)h(ob)s(ject)75 758 y(for)d(t)o(w)o(o)f (di\013eren)o(t)i(functions,)f(one)h(where)g(the)f(starting)g(address)h (is)g(signi\014can)o(t)g(and)g(one)f(where)h(the)75 814 y(starting)f(address)g(should)h(b)q(e)g(ignored.)166 871 y(An)k(un)o(b)q(ound)g(bu\013er)f(descriptor)h(is)g(created)f(b)o (y)g(using)h(the)g(sp)q(ecial)h Fl(MPI)p 1502 871 15 2 v 17 w(NULL)d Fs(v)m(alue)j(as)e(the)75 927 y Fl(start)f Fs(parameter)f(in)i Fl(MPI)p 564 927 V 17 w(APPEND)p 725 927 V 16 w(xxx)f Fs(op)q(erations)h(that)e(create)i(this)f (descriptor.)30 b(An)19 b(un)o(b)q(ound)75 984 y(bu\013er)c(descriptor) h(can)f(also)g(b)q(e)h(created)f(b)o(y)h Ft(un)o(binding)g Fs(a)f(b)q(ound)h(bu\013er)f(descriptor.)166 1075 y Ft(MPI)p 275 1075 16 2 v 18 w(UNBIND\(handlein,)k(handleout\))75 1210 y(IN)e(handlein)24 b Fs(handle)17 b(to)d(b)q(ound)j(bu\013er)e (descriptor)75 1302 y Ft(OUT)j(handleout)25 b Fs(handle)e(to)f(un)o(b)q (ound)i(bu\013er)f(descriptor)g(with)g(the)g(datat)o(yp)q(e)f(of)g(the) h(input)189 1358 y(bu\013er)15 b(descriptor)166 1533 y Fk(Discussion:)166 1583 y Fj(The)j Ff(MPI)p 324 1583 14 2 v 15 w(UNBIND)e Fj(function)h(allo)o(ws)f(to)h(create)i(a)e (bu\013er)i(datat)o(yp)q(e)e(after)h(the)g(pattern)g(of)f(an)g (existing)75 1633 y(bu\013er)e(and,)e(therefor,)i(after)f(the)g (pattern)h(of)e(an)h(existing)f(data)h(structure)i(in)d(memory)m(.)166 1772 y Fs(W)l(e)i(rep)q(eat)g(the)h(last)f(three)g(examples,)h(using)g (this)f(alternativ)o(e)h(notation.)166 1828 y(Example)g(2':)75 1927 y Fl(int)23 b(i[2])h(=)f({)h(2,)g(0};)75 1984 y(int)f(a[100];)75 2040 y(...)75 2097 y(mpi_create_buffer\()e(&b1,)j(MPI_PERSISTENT\);)75 2153 y(mpi_append_indexed\()d(b1,)j(MPI_NULL,)e(2,)i(i,)g(MPI_INT\);)75 2266 y(mpi_create_buffer\()d(&b2,)j(MPI_PERSISTENT\);)75 2322 y(mpi_append_contiguous\()d(b2,)i(MPI_NULL,)g(3,)h(b1\);)75 2435 y(mpi_create_buffer\()d(&b3,)j(MPI_PERSISTENT\);)75 2492 y(mpi_append_indexed\()d(b3,)j(\(void)f(*\)a,)g(2,)h(i,)f(b2\);)75 2548 y(mpi_append_vec\()f(b3,)h(\(void)g(*\)\(a+50\),)g(3,)h(3,)f(2,)h (b1\);)166 2647 y Fs(Example)16 b(3:)p eop %%Page: 22 27 22 26 bop 75 -100 a Fs(22)613 b Fq(CHAPTER)16 b(2.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 102 y Fl(struct)23 b(foostruct)g({)147 158 y(int)g(i;)147 214 y(double)g(d[5];)147 271 y(char)g(b;)75 327 y(};)75 440 y(struct)g(foostruct)g(foo[100];)75 553 y(MPI_BUFFER)f(t,)i(tt;)75 610 y(MPI_BUFFER)e(fb;)75 723 y(mpi_create_buffer\(&t,)45 b(MPI_EPHEMERAL\);)75 779 y(mpi_create_buffer\(&fb,)21 b(MPI_PERSISTENT\);)75 892 y(/*)j(Describe)e(the)i(data)f(type)g(in)h(t)g(*/)75 948 y(mpi_append_contiguous\(t,)d(&\(\(struct)h(foostruct)h (*\)0\)->i,1,MPI_INT\);)75 1005 y(mpi_append_contiguous\(t,)e (&\(\(struct)h(foostruct)h(*\)0\)->d,5,MPI_DOUBLE\);)75 1061 y(mpi_append_contiguous\(t,)e(&\(\(struct)h(foostruct)h (*\)0\)->b,1,MPI_BYTE\);)75 1174 y(/*)h(Abstract)e(buffer)h(datatype)g (*/)75 1231 y(mpi_unbind\(t,)f(tt\);)75 1344 y(/*)i(Now)f(describe)g (foo)g(itself,)g(a)h(contiguous)e(array)i(of)f(100)h(elements)99 1400 y(*)g(of)f(the)h(derived)f(type.)99 1456 y(*/)75 1513 y(mpi_append_contiguous\(fb,&)o(foo,100)o(,tt\);)75 1569 y(mpi_commit\(fb\);)166 1657 y Fs(Example)16 b(4:)170 1800 y Fl(real)24 b(a\(100,100,100\))75 1857 y(c)71 b(declare)23 b(the)h(buffer)f(descriptors)170 1913 y(integer)g(onedslice)170 1970 y(integer)g(twodslice)170 2026 y(integer)g(message)75 2139 y(c)71 b(Slice)24 b(out)f(the)h(section)e(a\(1:21:2,1:10,1:11\))g (as)h(a)h(single)f(buffer)g(descriptor)170 2252 y(call)h (mpi_create_buffer\(ondesl)o(ice,)d(MPI_EPHEMERAL\))170 2308 y(call)j(mpi_create_buffer\(twodsl)o(ice,)d(MPI_EPHEMERAL\))170 2365 y(call)j(mpi_create_buffer\(messag)o(e,)69 b(MPI_EPHEMERAL\))170 2478 y(call)24 b(mpi_append_vec)e(\(onedslice,)g(MPI_NULL,)h (1+\(21-1\)/2,)147 2534 y($)501 b(2,)23 b(1,)h(MPI_REAL\))170 2591 y(call)g(mpi_append_hvec\(twodslic)o(e,)d(MPI_NULL,)47 b(10,)147 2647 y($)501 b(100*MPI_REAL_SIZE,)21 b(1,)j(ondeslice\))170 2704 y(call)g(mpi_append_hvec\(message,)c(a,)263 b(11,)p eop %%Page: 23 28 23 27 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(23)147 45 y Fl($)501 b(100*100*MPI_REAL_SIZE,1)o (,twodsl)o(ice\))170 158 y(call)24 b(mpi_commit\(message\))75 271 y(c)71 b(all)24 b(done...)75 388 y Ft(T)o(yp)q(e)17 b(matc)o(hing)75 473 y Fs(The)h(rules)i(for)d(bu\013er)h(t)o(yp)q(e)h (matc)o(hing)f(are)g(extended)h(to)f(comp)q(osite)g(bu\013er)h (descriptors:)26 b(A)18 b(send)75 530 y(bu\013er)f(descriptor)g(matc)o (hes)f(a)g(receiv)o(e)h(bu\013er)g(descriptor)g(is)g(the)g(successiv)o (e)g(basic)h(en)o(tries)e(in)i(eac)o(h)75 586 y(descriptor)e(ha)o(v)o (e)e(iden)o(tical)k(names.)166 643 y(Example:)75 719 y Fl(struct)23 b({)147 775 y(int)g(a[5];)147 832 y(char)g(c;)242 888 y(})h(st[100];)75 1001 y(...)75 1058 y(mpi_create_buffer\(&b1,)d (MPI_PERSISTENT\);)75 1114 y(mpi_append_contiguous\(b1,)f(\(void)k (*\)st,)f(5,)g(MPI_INT\);)75 1170 y(mpi_append_contiguous\(b1,)d (\(void)k(*\)&\(st.c\),)e(1,)i(MPI_CHAR\);)75 1227 y (mpi_append_contiguous\(b1,)c(\(void)k(*\)st,)f(5,)g(MPI_INT\);)75 1283 y(mpi_append_contiguous\(b1,)d(\(void)k(*\)&\(st.c\),)e(1,)i (MPI_CHAR\);)75 1340 y(...)75 1396 y(mpi_create_buffer\(&b2,)d (MPI_PERSISTENT\);)75 1453 y(mpi_append_contiguous\(b2,)f(\(void)k (*\)st,)f(5,)g(MPI_INT\);)75 1509 y(mpi_append_contiguous\(b2,)d (\(void)k(*\)&\(st.c\),)e(1,)i(MPI_CHAR\);)75 1622 y (mpi_create_buffer\(&b3,)d(MPI_PERSISTENT\);)75 1679 y(mpi_append_contiguous\(b3,)f(\(void)k(*\)st,)f(2,)g(b2\);)166 1755 y Fs(The)d(bu\013er)h(descriptors)f(asso)q(ciated)h(with)f Fl(b1)g Fs(and)g Fl(b3)g Fs(ha)o(v)o(e)g(matc)o(hing)g(t)o(yp)q(es.)35 b(They)21 b(b)q(oth)75 1811 y(corresp)q(ond)h(to)f(a)g(sequence)i(of)f Fl(int,)h(int,)g(char,)g(int,)h(int,)f(char)p Fs(.)39 b(\(in)22 b(fact,)g(b)q(oth)g(bu\013ers)75 1867 y(consist)15 b(of)g(the)h(same)e(sequence)j(of)e(v)m(ariables\).)166 2000 y Fk(Discussion:)166 2050 y Fj(A)i(more)g(restrictiv)o(e)i (de\014nition)e(of)f(t)o(yp)q(e)i(matc)o(hing)e(is)h(to)h(require)g (\\structural)g(matc)o(hing":)23 b(I.e.,)17 b(t)o(w)o(o)75 2100 y(descriptors)12 b(matc)o(h)e(if)g(all)f(corresp)q(onding)j(top)f (\(comp)q(osite\))f(comp)q(onen)o(ts)g(recursiv)o(ely)i(matc)o(h)e (pairwise.)17 b(With)75 2149 y(suc)o(h)e(de\014nition,)e(the)i(t)o(w)o (o)f(bu\013er)h(descriptors)h(in)d(the)i(last)f(example)f(don't)g(ha)o (v)o(e)h(matc)o(hing)e(t)o(yp)q(es.)20 b(I)14 b(\(Marc\))75 2199 y(b)q(eliev)o(e)f(that)f(the)h(w)o(eak)o(er)g(de\014nition)f(mak)o (es)f(more)h(sense)i(in)e(our)g(en)o(vironmen)o(t,)f(since)i(w)o(e)g (do)f(not,)g(in)g(general,)75 2249 y(require)j(an)o(y)e(corrsp)q (ondance)j(b)q(et)o(w)o(een)f(data)f(alignemen)o(t)e(at)h(sender)j(and) e(data)f(alignmen)o(t)f(at)i(receiv)o(er.)166 2388 y Fs(T)o(yp)q(e)h(matc)o(hing)h(of)e(bu\013er)i(descriptor)f(and)h(host)e (language)i(is)f(dealt)h(similarly)l(.)75 2505 y Ft(Message)h(length)75 2591 y Fs(The)d(length)h(of)e(a)h(message)f(is)h(de\014ned)i(as)d(the)h (n)o(um)o(b)q(er)g(of)f(basic)i(elemen)o(ts)g(\(comp)q(onen)o(ts)e (with)h(basic)75 2647 y(datat)o(yp)q(es\))i(in)i(the)e(message.)25 b(This)17 b(is)g(the)g(v)m(alue)h(returned)g(in)f(the)g(status)f(of)h (a)f(receiv)o(e)i(op)q(eration)75 2704 y(for)d(that)f(message.)p eop %%Page: 24 29 24 28 bop 75 -100 a Fs(24)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fk(Discussion:)166 100 y Fj(An)g(alternativ)o(e)f(de\014nition)g(is)h(to)f(coun)o(t)h (message)g(length)f(as)h(the)g(n)o(um)o(b)q(er)f(of)g(top)h(comp)q (onen)o(ts)f(in)g(the)75 150 y(message.)h(If)10 b(w)o(e)h(accept)g(the) g(w)o(eak)o(er)f(de\014nition)g(of)f(t)o(yp)q(e)i(compatibilit)o(y)l(,) d(this)i(has)g(the)h(e\013ect)h(that)e(the)g(length)h(of)75 199 y(a)f(message)g(sen)o(t)g(ma)o(y)f(b)q(e)h(v)o(ery)h(di\013eren)o (t)g(from)d(the)j(length)f(of)f(the)i(message)f(receiv)o(ed)h({)f(ev)o (en)h(in)e(a)h(homogeneous)75 249 y(en)o(vironmen)o(t.)17 b(Consider)d(in)f(the)i(previous)f(example)e(the)j(case)g(of)e(message) g(sen)o(t)i(with)e Ff(b1)h Fj(and)f(receiv)o(ed)i(with)75 299 y Ff(b3)e Fj({)h(the)h(length)e(has)h(halv)o(ed.)166 360 y(On)19 b(the)g(other)g(hand,)g(if)f(w)o(e)g(accept)i(the)f (stronger)h(de\014nition)e(of)f(t)o(yp)q(e)i(compatibilit)o(y)m(,)d (coun)o(ting)i(top)75 417 y(elemen)o(ts)d(is)g(plausible.)21 b(Since)16 b(top)f(comp)q(onen)o(ts)g(are)g(required)h(to)f(matc)o(h)f (pairwise,)h(the)h(message)f(length)g(is)75 473 y(in)o(teger,)f(and)g (iden)o(tical)f(at)h(sending)g(and)f(receiving)i(sites.)75 704 y Ft(Bu\013er)i(descriptors)g(as)g(datat)o(yp)q(es)h(in)g(comm)o (unication)h(op)q(erations)75 799 y Fs(W)l(e)d(prop)q(ose)f(to)g(allo)o (w)h(the)g(use)g(of)f(bu\013er)h(descriptor)g(handles)h(as)e(datat)o (yp)q(es)g(in)h(con)o(tiguous)g(bu\013er)75 855 y(comm)o(unication)g (op)q(erations)f(\(e.g.)k(in)d Fl(MPI)p 847 855 15 2 v 17 w(SENDC)p Fs(\))166 917 y(If)g(w)o(e)g(do)h(so,)e(then)i(w)o(e)f (don't)f(need)j(an)o(ymore)d(the)h(more)g(general)h(comm)o(unication)g (op)q(erations)75 973 y(with)f(a)e Fl(buffer)p 363 973 V 17 w(handle)g Fs(argumen)o(t!!!)75 1030 y Fl(MPI)p 150 1030 V 17 w(SEND\(buffer)p 431 1030 V 15 w(handle,)23 b(dest,)h(tag,)f(context\))75 1086 y Fs(is)16 b(replaced)g(b)o(y)75 1143 y Fl(MPI)p 150 1143 V 17 w(SENDC\(start,)22 b(1,)i(buffer)p 693 1143 V 16 w(handle,)f(dest,)g(tag,)g(context\))p Fs(,)75 1199 y(where)12 b Fl(start)e Fs(is)i(the)f(\014rst)g(bu\013er)h (lo)q(cation.)19 b(\(If)11 b(un)o(b)q(ound)i(bu\013er)e(descriptors)h (are)f(k)o(ept)g(distinct)i(from)75 1255 y(b)q(ound)j(bu\013er)e (descriptors,)h(then)g(the)g(argumen)o(t)e Fl(buffer)p 1117 1255 V 17 w(handle)h Fs(gets)g(replaced)h(b)o(y)g(the)g(un)o(b)q (ound)75 1312 y(datat)o(yp)q(e)g(of)f(the)i(bu\013er.\))166 1373 y(Suc)o(h)21 b(simpli\014cation)j(w)o(ould)d(b)q(e)g(a)g(big)g (win,)i(as)d(comm)o(unication)i(with)f(general)g(bu\013ers)g(b)q(e-)75 1430 y(comes)13 b(a)h(sp)q(ecial)h(case)f(of)f(comm)o(unication)h(with) g(con)o(tiguous)f(bu\013ers.)20 b(One)14 b(ca)o(v)o(eat:)k(When)c(a)f (bu\013er)75 1486 y(descriptor)i(is)f(used)h(as)f(a)g(datat)o(yp)q(e,)f (w)o(e)h(ma)o(y)g(ha)o(v)o(e)g(restrictions)g(on)h(where)f(from)f(the)i (di\013eren)o(t)f(v)m(ari-)75 1543 y(ables)j(app)q(earing)g(in)g(the)f (bu\013er)h(descriptor)f(come)h(from:)k(They)16 b(need)h(to)f(all)h (come)f(from)g(the)g(same)75 1599 y(con)o(tiguous)f(address)g(sap)q (ce,)g(so)f(that)h(their)g(relativ)o(e)g(displacemen)o(t)i(is)e(w)o (ell)h(de\014ned.)21 b(It)15 b(means)g(that)75 1656 y(w)o(e)g(shall)i (only)f(b)q(e)g(able)h(to)d(pac)o(k)i(in)g(one)g(message)f(v)m (ariables)i(that)e(b)q(elong)h(to)f(the)h(same)f(con)o(tiguous)75 1712 y(address)i(space.)27 b(W)l(e)17 b(ha)o(v)o(e)g(to)g(think)h(ab)q (out)f(syn)o(tactic)g(rules)i(to)d(enforce)i(this)g(restriction:)24 b(E.g.,)16 b(is)75 1768 y(it)i(su\016cien)o(t)g(to)e(sa)o(y)h(in)h(C)g (that)e(a)h(bu\013er)h(is)g(either)g(all)g(in)g(stac)o(k,)f(or)g(all)h (in)g(heap,)g(or)f(all)h(in)h(static?)75 1825 y(What)c(is)g(the)h (equiv)m(alen)o(t)g(F)l(ortran)e(rule?)22 b(Are)15 b(suc)o(h)h(rules)g (acceptable?)166 1969 y Fk(Discussion:)34 b Fj(OPINIONS?)75 2199 y Ft(Other)18 b(issues)75 2295 y Fs(An)13 b(ephemeral)h(bu\013er)e (descriptor)h(can)g(b)q(e)g(freed)g(as)f(so)q(on)g(as)h(it)f(has)h(b)q (een)h(used)f(as)f(a)g(t)o(yp)q(e)h(parameter)75 2351 y(in)j(a)f(bu\013er)g(descriptor)h(creation)f(function.)166 2412 y(A)d(bu\013er)g(descriptor)g(handle)i(cannot)d(b)q(e)i(passed)f (to)g(the)g(same)f Fl(MPI)p 1350 2412 V 17 w(APPEND)p 1511 2412 V 16 w(xxx)h Fs(function)h(b)q(oth)75 2469 y(as)i(a)g(bu\013er)g(descriptor)g(handle)i(and)e(as)g(a)g(datat)o(yp)q (e)g(\(no)f(self-references\).)166 2613 y Fk(Discussion:)34 b Fj(The)14 b(last)g(constrain)o(t)g(can)g(b)q(e)h(remo)o(v)o(ed)e(if)g (someb)q(o)q(dy)g(cares.)p eop %%Page: 25 30 25 29 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(25)75 45 y Fi(2.5.4)55 b(T)n(yp)r(e)18 b(Matc)n(hing)75 131 y Fs(One)e(can)f(think)h(of)f(message)g (transmission)g(as)g(consisting)h(of)f(three)g(phases:)131 221 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i(send)f(bu\013er) h(and)f(a)g(message)g(is)g(assem)o(bled)131 313 y(2.)22 b(A)15 b(message)g(is)g(transferred)g(from)f(sender)i(to)f(receiv)o(er) 131 405 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h(incoming)i (message)e(and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o(e)h (bu\013er)166 495 y(T)o(yp)q(e)k(matc)o(hing)f(has)h(to)e(b)q(e)j (observ)o(ed)e(at)g(eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f(eac)o(h)75 552 y(v)m(ariable)d(in)f(the)f (sender)h(bu\013er)f(has)g(to)f(matc)o(h)h(the)g(t)o(yp)q(e)h(of)e(the) i(corresp)q(onding)g(en)o(try)f(in)h(the)f(bu\013er)75 608 y(descriptor;)f(the)g(t)o(yp)q(e)f(of)g(an)g(en)o(try)g(in)h(the)g (bu\013er)f(descriptor)h(used)f(to)g(send)h(a)f(message)g(has)g(to)g (matc)o(h)75 665 y(the)g(t)o(yp)q(e)h(of)f(the)g(corresp)q(onding)h(en) o(try)f(in)h(the)f(bu\013er)h(descriptor)f(used)h(to)f(receiv)o(e)h (the)f(message;)h(and)75 721 y(the)j(t)o(yp)q(e)f(of)g(eac)o(h)h(v)m (ariable)h(in)f(the)f(receiv)o(e)i(bu\013er)e(has)h(to)f(matc)o(h)f (the)i(corresp)q(onding)g(en)o(try)g(in)g(the)75 778 y(receiv)o(er)g(bu\013er)f(descriptor.)20 b(A)c(program)e(that)g(fails) i(to)f(observ)o(e)g(these)g(three)g(rules)h(is)g(erroneous.)166 834 y(T)l(o)f(de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l (,)j(w)o(e)d(need)i(to)e(deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 890 y(t)o(yp)q(es)h(of)g(the)h(host)f (language)g(with)h(bu\013er)g(descriptor)f(t)o(yp)q(es;)h(and)g(matc)o (hing)f(of)g(bu\013er)h(descriptor)75 947 y(t)o(yp)q(es)22 b(at)f(sender)h(and)g(receiv)o(er.)40 b(W)l(e)22 b(restrict)f(ourselv)o (es)i(to)e(the)g(case)h(where)g(the)g(sending)h(and)75 1003 y(receiving)17 b(programs)d(are)h(written)g(in)h(the)f(same)g (language.)166 1060 y(The)e(t)o(yp)q(e)g(of)f(an)h(en)o(try)f(in)h(the) g(sender)g(bu\013er)g(descriptor)g(matc)o(hes)g(the)f(t)o(yp)q(e)h(of)f (an)h(en)o(try)f(in)i(the)75 1116 y(receiv)o(er)g(bu\013er)g (descriptor)g(if)h(they)f(ha)o(v)o(e)f(iden)o(tical)j(names:)j Fl(MPI)p 1238 1116 15 2 v 16 w(INTEGER)13 b Fs(matc)o(hes)h Fl(MPI)p 1682 1116 V 16 w(INTEGER)p Fs(,)75 1173 y Fl(MPI)p 150 1173 V 17 w(REAL)g Fs(matc)o(hes)h Fl(MPI)p 525 1173 V 17 w(REAL)p Fs(,)f(and)i(so)e(on.)166 1229 y(The)19 b(t)o(yp)q(e)h(of)f(a)g(v)m(ariable)i(in)f(a)f(host)g(program)f(matc)o (hes)h(the)g(t)o(yp)q(e)h(of)e(the)i(en)o(try)f(in)h(a)f(bu\013er)75 1286 y(descriptor)h(if)h(the)f(name)g(of)g(the)g(bu\013er)g(descriptor) g(en)o(try)g(t)o(yp)q(e)g(corresp)q(onds)h(to)e(the)h(basic)h(t)o(yp)q (e)75 1342 y(of)e(the)g(host)g(program)f(v)m(ariable:)30 b(an)19 b(en)o(try)g(with)h(t)o(yp)q(e)f(name)g Fl(MPI)p 1303 1342 V 17 w(INTEGER)f Fs(matc)o(hes)h(a)g(F)l(ortran)75 1399 y(v)m(ariable)h(of)d(t)o(yp)q(e)h Fl(INTEGER)p Fs(,)f(an)h(en)o (try)g(with)g(t)o(yp)q(e)h(name)f Fl(MPI)p 1205 1399 V 17 w(REAL)f Fs(matc)o(hes)h(a)g(F)l(ortran)f(v)m(ariable)75 1455 y(of)j(t)o(yp)q(e)h Fl(REAL)p Fs(,)f(and)h(so)f(on.)37 b(There)21 b(is)g(one)g(exception)h(to)e(this)h(last)g(rule:)31 b(An)21 b(en)o(try)g(with)g(t)o(yp)q(e)75 1511 y(name)14 b Fl(MPI)p 270 1511 V 17 w(BYTE)g Fs(can)g(b)q(e)h(used)f(to)g(matc)o (h)g(an)o(y)f(b)o(yte)h(of)g(storage)f(\(on)h(a)g(b)o(yte-addressable)h (mac)o(hine\),)75 1568 y(irresp)q(ectiv)o(e)e(of)f(the)g(datat)o(yp)q (e)f(of)h(the)f(v)m(ariable)j(that)d(con)o(tains)h(this)g(b)o(yte.)19 b(The)12 b(v)m(alue)h(of)e(the)h(message)75 1624 y(en)o(try)j(will)i(b) q(e)f(the)f(binary)h(v)m(alue)g(of)f(the)g(corresp)q(onding)h(b)o(yte)f (in)h(memory)l(.)166 1681 y(W)l(e)f(th)o(us)g(ha)o(v)o(e)g(t)o(w)o(o)f (cases:)143 1771 y Fo(\017)23 b Fs(Comm)o(unication)15 b(of)g(t)o(yp)q(ed)h(v)m(alues)g(\(e.g.,)e(with)i(datat)o(yp)q(e)f (di\013eren)o(t)g(from)g Fl(MPI)p 1604 1771 V 17 w(BYTE)p Fs(\),)f(where)189 1827 y(the)f(datat)o(yp)q(es)g(of)g(the)g(corresp)q (onding)h(en)o(tries)g(in)g(the)f(sender)h(program,)f(in)h(the)f (sender)h(bu\013er)189 1884 y(descriptor,)21 b(in)g(the)f(receiv)o(er)h (bu\013er)f(descriptor)g(and)g(in)h(the)f(receiv)o(er)h(program)e (should)i(all)189 1940 y(matc)o(h.)143 2032 y Fo(\017)i Fs(Comm)o(unication)16 b(of)f(un)o(t)o(yp)q(ed)h(v)m(alues)h(\(e.g.,)d (of)h(datat)o(yp)q(e)h Fl(MPI)p 1317 2032 V 16 w(BYTE)p Fs(\),)f(where)h(the)f(the)h(corre-)189 2089 y(sp)q(onding)f(en)o (tries)g(in)h(the)e(sender)i(and)e(the)h(receiv)o(er)g(bu\013er)g (descriptors)g(ha)o(v)o(e)f(b)q(oth)g(datat)o(yp)q(e)189 2145 y Fl(MPI)p 264 2145 V 16 w(BYTE)p Fs(.)f(In)i(this)f(case,)g (there)g(are)f(no)h(requiremen)o(ts)g(on)g(the)g(datat)o(yp)q(es)f(of)h (the)f(corresp)q(ond-)189 2202 y(ing)g(en)o(tries)h(in)g(the)g(sender)f (and)h(the)f(receiv)o(er)h(programs,)e(nor)h(is)h(it)g(required)g(that) f(they)g(b)q(e)h(the)189 2258 y(same.)75 2379 y Fi(2.5.5)55 b(Data)19 b(con)n(v)n(ersion)75 2465 y Fs(W)l(e)c(use)h(the)f(follo)o (wing)h(terminology:)75 2555 y Ft(t)o(yp)q(e)h(con)o(v)o(ersion)22 b Fs(c)o(hanges)13 b(the)g(datat)o(yp)q(e)f(of)h(a)g(v)m(alue,)h(e.g.)k (b)o(y)13 b(rounding)h(a)f Fl(REAL)f Fs(to)g(an)h Fl(INTEGER)p Fs(.)75 2647 y Ft(represen)o(tation)k(con)o(v)o(ersion)22 b Fs(c)o(hanges)16 b(the)h(binary)g(represen)o(tation)f(of)g(a)g(v)m (alue,)h(e.g.)23 b(from)16 b(Hex)189 2704 y(\015oating)f(p)q(oin)o(t)g (to)g(IEEE)g(\015oating)h(p)q(oin)o(t.)p eop %%Page: 26 31 26 30 bop 75 -100 a Fs(26)613 b Fq(CHAPTER)16 b(2.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fs(Represen)o (tation)23 b(con)o(v)o(ersion)f(is)g(needed)i(when)e(data)f(is)i(mo)o (v)o(ed)e(across)h(mac)o(hines)g(that)g(use)75 102 y(di\013eren)o(t)17 b(binary)g(enco)q(dings)h(for)e(the)g(same)h(datat)o(yp)q(e.)23 b(Represen)o(tation)17 b(con)o(v)o(ersion)g(preserv)o(es,)g(to)75 158 y(the)c(greatest)f(p)q(ossible)j(extend,)f(the)g Ft(v)m(alue)f Fs(represen)o(ted.)20 b(Ho)o(w)o(ev)o(er,)12 b(rounding)i(errors)f(and)g(o)o(v)o(er\015o)o(w)75 214 y(and)e(under\015o)o(w)h(exceptions)g(ma)o(y)e(o)q(ccur)h(during)h (\015oating)f(p)q(oin)o(t)h(con)o(v)o(ersions;)g(con)o(v)o(ersion)f(of) g(in)o(tegers)75 271 y(ma)o(y)18 b(also)h(lead)h(to)e(exceptions)i (when)g(a)f(v)m(alue)h(that)e(can)h(represen)o(ted)h(in)g(one)f(system) f(cannot)h(b)q(e)75 327 y(represen)o(ted)d(in)g(the)f(other)g(system.)k (MPI)c(do)q(es)h(not)f(sp)q(ecify)h(rules)g(for)f(represen)o(tation)g (con)o(v)o(ersion.)166 384 y(The)e(t)o(yp)q(e)g(matc)o(hing)g(rules)g (imply)h(that)e(MPI)h(comm)o(unication)h(nev)o(er)f(en)o(tails)g(t)o (yp)q(e)g(con)o(v)o(ersion.)75 440 y(On)20 b(the)f(other)f(hand,)i(MPI) f(requires)h(that)e(a)h(represen)o(tation)g(con)o(v)o(ersion)g(b)q(e)h (p)q(erformed)f(when)g(a)75 497 y(t)o(yp)q(ed)i(v)m(alue)h(is)f (transferred)f(across)g(en)o(vironmen)o(ts)h(that)f(use)h(di\013eren)o (t)f(represen)o(tations)h(for)f(the)75 553 y(datat)o(yp)q(e)15 b(of)h(this)g(v)m(alue.)24 b(An)16 b(exception)h(o)q(ccurring)g(during) g(represen)o(tation)f(con)o(v)o(ersion)g(results)g(in)75 610 y(a)f(failure)h(of)f(the)g(comm)o(unication;)g(an)h(error)e(o)q (ccurs)h(either)h(in)g(the)f(send)h(op)q(eration,)f(or)g(the)g(receiv)o (e)75 666 y(op)q(eration,)g(or)g(b)q(oth.)166 723 y(If)g(a)g(v)m(alue)h (sen)o(t)f(in)h(a)e(message)h(is)g(un)o(t)o(yp)q(ed)h(\(i.e.,)e(of)h(t) o(yp)q(e)g Fl(MPI)p 1280 723 15 2 v 17 w(BYTE)p Fs(\),)f(then)h(the)g (binary)h(repre-)75 779 y(sen)o(tation)g(of)g(the)h(b)o(yte)f(stored)g (at)f(the)i(receiv)o(er)g(is)g(iden)o(tical)h(to)e(the)g(binary)h (represen)o(tation)g(of)f(the)75 835 y(b)o(yte)e(loaded)i(at)d(the)i (sender.)20 b(This)15 b(holds)h(true,)e(whether)h(sender)g(and)f (receiv)o(er)i(run)e(in)i(the)e(same)g(or)75 892 y(in)i(distinct)g(en)o (vironmen)o(ts;)f(no)h(represen)o(tation)f(con)o(v)o(ersion)g(is)h (required.)166 948 y(Note)d(that)g(no)g(con)o(v)o(ersions)h(ev)o(er)f (o)q(ccur)h(when)g(an)f(MPI)g(program)g(executes)h(in)g(a)f (homogeneous)75 1005 y(system,)f(where)g(all)g(pro)q(cesses)g(run)g(in) h(the)f(same)f(en)o(vironmen)o(t.)19 b(Also)12 b(note)f(the)h (di\013eren)o(t)g(b)q(eha)o(vior)g(of)75 1061 y Fl(MPI)p 150 1061 V 17 w(BYTE)g Fs(and)g(of)g Fl(MPI)p 481 1061 V 17 w(CHARACTER)p Fs(.)e(A)j(bu\013er)f(descriptor)h(en)o(try)f(with)h (datat)o(yp)q(e)f(of)f Fl(MPI)p 1646 1061 V 17 w(CHARACTER)75 1118 y Fs(can)17 b(only)g(matc)o(h)f(a)h(F)l(ortran)f(v)m(ariable)i(of) e(t)o(yp)q(e)h Fl(CHARACTER)p Fs(;)e(and)i(represen)o(tation)g(con)o(v) o(ersion)g(ma)o(y)75 1174 y(o)q(ccur)d(when)g(v)m(alues)h(of)e(t)o(yp)q (e)g Fl(MPI)p 672 1174 V 17 w(CHARACTER)f Fs(are)i(transferred.,)e (e.g.,)h(from)g(an)g(EBCDIC)g(enco)q(ding)75 1231 y(to)i(an)g(ASCI)q(I) h(enco)q(ding.)166 1287 y(Consider)g(the)f(follo)o(wing)h(examples.)75 1390 y Fl(!)48 b(FIRST)23 b(EXAMPLE)75 1446 y(INTEGER)g(A\(100\))75 1503 y(...)75 1559 y(CALL)g(MPI_CREATE_BUFFER\(BH,)e(MPI_PERSISTENT\)) 75 1616 y(CALL)i(MPI_APPEND_CONTIGUOUS)e(\(BH,)j(A\(1\),)f(1,)g (MPI_CHARACTER\))166 1718 y Fs(This)16 b(program)f(is)i(erroneous,)e (since)j(the)e(t)o(yp)q(e)g(of)f(the)h(bu\013er)g(descriptor)h(do)q(es) f(not)f(matc)o(h)h(the)75 1775 y(t)o(yp)q(e)g(of)g(the)g(v)m(ariable)i (in)f(the)f(bu\013er.)23 b(The)16 b(b)q(eha)o(vior)h(or)e(this)i (program)e(is)i(unde\014ned.)24 b(A)16 b(desirable)75 1831 y(b)q(eha)o(vior)d(is)h(for)e(that)g(error)g(to)h(b)q(e)g (detected)h(\(at)e(compile)i(time)f(or)g(run)g(time\))g(and)g(result)g (in)h(an)f(error)75 1888 y(condition.)75 1990 y Fl(!)48 b(SECOND)23 b(EXAMPLE)75 2047 y(INTEGER)g(A\(100\))75 2103 y(...)75 2160 y(CALL)g(MPI_CREATE_BUFFER\(BH,)e(MPI_PERSISTENT\)) 75 2216 y(CALL)i(MPI_APPEND_CONTIGUOUS)e(\(BH,)j(A\(1\),)f(1,)g (MPI_BYTE\))166 2319 y Fs(This)16 b(program)f(is)h(correct.)k(The)c(b)o (yte)g(that)e(will)k(b)q(e)e(transmitted)f(in)i(a)e(message)g(created)h (with)75 2375 y(this)i(bu\013er)g(descriptor)g(is)g(the)g(\014rst)g(b)o (yte)f(in)i(the)e(storage)g(o)q(ccupied)j(b)o(y)d Fl(A\(1\))p Fs(.)27 b(The)18 b(v)m(alue)h(of)f(this)75 2432 y(b)o(yte)d(is)h (implemen)o(tation)h(dep)q(enden)o(t,)f(and)g(will)h(v)m(ary)e (according)h(to)f(the)g(n)o(um)o(b)q(er)h(of)f(b)o(ytes)g(used)h(to)75 2488 y(store)e(F)l(ortran)h(v)m(ariables)h(of)f(t)o(yp)q(e)g Fl(INTEGER)f Fs(and)i(the)f(b)o(yte)g(ordering)g(on)h(the)f(host)g(mac) o(hine.)75 2591 y Fl(!)48 b(THIRD)23 b(EXAMPLE)75 2647 y(INTEGER)g(A\(100\))75 2704 y(...)p eop %%Page: 27 32 27 31 bop 75 -100 a Fq(2.5.)34 b(COMMUNICA)l(TION)16 b(BUFFERS)1001 b Fs(27)75 45 y Fl(CALL)23 b(MPI_CREATE_BUFFER\(BH1,)e (MPI_PERSISTENT\))75 102 y(CALL)i(MPI_APPEND_CONTIGUOUS)e(\(BH1,)i (A\(1\),)h(1,)f(MPI_INTEGER\))75 158 y(...)75 214 y(CALL)g (MPI_CREATE_BUFFER\(BH2,)e(MPI_PERSISTENT\))75 271 y(CALL)i (MPI_APPEND_CONTIGUOUS)e(\(BH2,)i(A\(1\),)h(4,)f(MPI_BYTE\))166 368 y Fs(Consider)16 b(the)f(follo)o(wing)h(three)f(cases:)131 456 y(1.)22 b(A)d(message)g(is)h(sen)o(t)f(using)h(bu\013er)f (descriptor)h(handle)h Fl(BH1)e Fs(and)g(receiv)o(ed)i(using)f (bu\013er)f(de-)189 512 y(scriptor)e(handle)h Fl(BH2)p Fs(.)25 b(The)18 b(program)e(is)h(erroneous,)h(since)g(the)f(sender)h (and)f(receiv)o(er)h(bu\013er)189 568 y(descriptors)g(don't)g(matc)o (h.)28 b(The)19 b(b)q(eha)o(vior)f(of)g(the)h(program)e(is)h (unde\014ned,)j(and)d(a)g(p)q(ossible)189 625 y(\(desirable)e(but)f (not)g(required\))h(b)q(eha)o(vior)g(is)g(for)e(an)h(error)g(condition) h(to)f(o)q(ccur.)131 716 y(2.)22 b(Both)17 b(sender)h(and)g(receiv)o (er)g(use)g(bu\013er)g(descriptor)g(handle)h Fl(BH1)p Fs(.)26 b(The)18 b(program)f(is)h(correct,)189 773 y(and)h(an)f(in)o (teger)h(v)m(alue)h(is)f(transmitted)g(from)f(sender)h(to)f(receiv)o (er.)31 b(The)19 b(v)m(alue)h(assigned)g(to)189 829 y Fl(A\(1\))e Fs(at)g(the)g(receiv)o(er)i(is)f(iden)o(tical)h(to)e(the)h (v)m(alue)h(of)e Fl(A\(1\))g Fs(at)g(the)g(sender,)i(but)f(the)f (binary)189 886 y(represen)o(tation)g(of)g(this)g(v)m(alue)i(ma)o(y)d (b)q(e)i(di\013eren)o(t,)g(if)f(sender)h(and)f(receiv)o(er)h(run)g(in)g (di\013eren)o(t)189 942 y(en)o(vironmen)o(ts.)131 1033 y(3.)j(Both)17 b(sender)h(and)g(receiv)o(er)g(use)g(bu\013er)g (descriptor)g(handle)h Fl(BH2)p Fs(.)26 b(The)18 b(program)f(is)h (correct.)189 1090 y(Assume)13 b(that)f(b)q(oth)h(sender)h(and)f (receiv)o(er)h(use)f(four)g(b)o(ytes)f(to)h(store)f(in)o(tegers.)19 b(Then)14 b(the)f(v)m(alue)189 1146 y(stored)h(in)i Fl(A\(1\))e Fs(at)h(the)g(receiv)o(er)h(has)e(the)i(same)e(binary)i(represen)o (tation)f(as)g(the)g(v)m(alue)h(of)e Fl(A\(1)189 1203 y Fs(at)i(the)i(sender.)26 b(If)18 b(the)f(same)g(b)o(yte)g(ordering)h (and)g(the)f(same)g(enco)q(ding)i(of)e(in)o(teger)g(v)m(alues)h(is)189 1259 y(used)d(at)g(b)q(oth)g(ends)h(then)f Fl(A\(1\))f Fs(at)h(the)g(receiv)o(er)h(is)f(set)g(to)g(the)g(v)m(alue)h(of)f Fl(A\(1\))f Fs(at)h(the)g(sender;)189 1315 y(otherwise)g(the)g(same)g (32)g(bits)g(ma)o(y)g(enco)q(de)h(di\013eren)o(t)g(v)m(alues.)166 1403 y(Data)e(represen)o(tation)i(con)o(v)o(ersion)f(also)h(applies)h (to)e(the)g(en)o(v)o(elop)q(e)i(of)e(a)g(message:)20 b(source,)15 b(des-)75 1459 y(tination)h(and)f(tag)f(are)h(all)h(in)o (tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i(con)o(v)o(erted.)166 1592 y Fk(Implemen)o(tati)o(on)c(note:)166 1642 y Fj(The)i(curren)o(t)h (de\014nition)e(do)q(es)h(not)g(require)g(messages)g(to)f(carry)i(t)o (yp)q(e)f(information.)h(A)e(message)h(can)f(b)q(e)75 1691 y(comp)q(osed)h(and)h(sen)o(t)h(using)e(only)g(the)h(information)d (in)i(the)i(sender)g(bu\013er)g(descriptor,)g(and)e(can)h(b)q(e)h (receiv)o(ed)75 1741 y(and)c(stored)h(using)f(only)g(the)h(information) c(in)j(the)h(receiv)o(er)h(bu\013er)f(descriptor.)19 b(If)11 b(messages)i(are)f(sen)o(t)i(b)q(et)o(w)o(een)75 1791 y(di\013eren)o(t)g(mac)o(hines)f(then)g(one)h(can)f(either)h(use)g (a)f(\\univ)o(ersal")g(data)g(enco)q(ding)g(for)g(messages,)g(use)h (kno)o(wledge)75 1841 y(of)g(the)i(receiv)o(er)h(en)o(vironmen)o(t)d (in)g(order)i(to)f(con)o(v)o(ert)h(data)e(at)h(the)h(sender,)h(or)e (use)g(kno)o(wledge)g(of)g(the)g(sender)75 1891 y(en)o(vironmen)o(t)f (in)g(order)i(to)f(con)o(v)o(ert)g(data)g(at)g(the)g(receiv)o(er.)23 b(In)15 b(either)h(case)g(the)f(lo)q(cal)g(bu\013er)g(descriptor)i(can) 75 1940 y(b)q(e)e(used)g(to)f(deriv)o(e)h(the)g(t)o(yp)q(es)g(of)f(the) g(v)n(alues)g(transferred.)21 b(Ho)o(w)o(ev)o(er,)15 b(additional)d(t)o(yp)q(e)j(information)c(carried)75 1990 y(b)o(y)j(messages)g(can)g(pro)o(vide)f(b)q(etter)j(error)f (detection.)166 2205 y Fk(Discussion:)166 2255 y Fj(There)f(is)f(no)g (agreemen)o(t)f(y)o(et)i(on)f(the)g(need)h(for)f(a)f(de\014nition)h (and,)g(if)f(needed,)i(on)f(the)g(righ)o(t)g(de\014nition)g(of)75 2305 y(t)o(yp)q(e)i(matc)o(hing)d(and)i(con)o(v)o(ersion)g(b)q(et)o(w)o (een)h(programs)e(written)i(in)f(di\013eren)o(t)h(languages.)j(One)c (prop)q(osal)g(is)g(to)53 b Fg(TO)11 b(DISCUSS)75 2355 y Fj(allo)o(w)h(b)q(oth)h(F)m(ortran)g(routines)h(to)f(sp)q(ecify)g(C)h (datat)o(yp)q(es)f(in)g(their)h(bu\013er)g(descriptors,)g(and)f(for)g (C)g(routines)h(to)75 2405 y(sp)q(ecify)h(F)m(ortran)g(data)o(yp)q(es.) 20 b(The)15 b(draft)g(will)e(sp)q(ecify)i(rules)g(for)f(t)o(yp)q(e)i (matc)o(hing)c(b)q(et)o(w)o(een)k(F)m(ortran)f(v)n(ariables)75 2455 y(and)i(C)g(datat)o(yp)q(es)h(in)f(the)h(bu\013er)g(descriptors)h (and)e(b)q(et)o(w)o(een)i(C)e(v)n(ariables)f(and)i(F)m(ortran)f(datat)o (yp)q(es)h(in)e(the)75 2504 y(bu\013er)g(descriptor.)21 b(\(E.g.,)14 b(a)g(F)m(ortran)g(v)n(ariable)g(of)g(t)o(yp)q(e)h Ff(REAL)f Fj(ma)o(y)f(matc)o(h)g(an)h(en)o(try)i(of)e(t)o(yp)q(e)h Ff(MPI)p 1705 2504 14 2 v 15 w(FLOAT)e Fj(in)75 2554 y(a)g(bu\013er)i(descriptor,)f(but)g(not)g(an)f(en)o(try)h(of)f(t)o(yp) q(e)h Ff(MPI)p 957 2554 V 15 w(INTEGER)p Fj(.)e(Data)h(represen)o (tation)i(con)o(v)o(ersion)f(o)q(ccurs)h(if)75 2604 y(corresp)q(onding) h(C)f(and)g(F)m(ortran)g(t)o(yp)q(es)h(ha)o(v)o(e)f(di\013eren)o(t)i (represen)o(tations)g(on)e(the)g(same)g(pro)q(cess.)24 b(There)16 b(still)75 2654 y(is)f(the)g(requiremen)o(t)g(that)g (bu\013er)g(descriptor)h(en)o(tries)g(on)f(the)g(send)h(side)f(matc)o (h)f(v)o(erbatim)f(bu\013er)j(descriptor)75 2704 y(en)o(tries)f(on)f (the)g(receiv)o(e)h(side,)f(and)g(there)h(is)f(no)f(c)o(hange)i(in)e (in)o(terpro)q(cess)j(comm)o(unication.)p eop %%Page: 28 33 28 32 bop 75 -100 a Fs(28)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fj(This)d(prop)q (osal)h(giv)o(es)f(a)h(precise)h(meaning)d(to)h(in)o(terlanguage)g (comm)o(unication.)h(It)g(has)g(t)o(w)o(o)f(dra)o(wbac)o(ks:)75 95 y(It)18 b(ma)o(y)d(require)k(additional)d(data)h(con)o(v)o(ersions)h (when)h(messages)e(are)h(assem)o(bled)g(or)f(disassem)o(bled,)h(and)f (it)75 145 y(imp)q(oses)12 b(a)h(\014xed)g(corresp)q(ondence)j(b)q(et)o (w)o(een)e(C)f(and)f(F)m(ortran)h(t)o(yp)q(e)g(whic)o(h)g(ma)o(y)e(not) i(b)q(e)g(the)g(preferred)i(one)e(on)75 195 y(the)h(target)g(mac)o (hine.)j(\(Some)c(mac)o(hines)f(ma)o(y)g(w)o(an)o(t)h(F)m(ortran)h Ff(REAL)f Fj(to)g(matc)o(h)f(C)i Ff(FLOAT)p Fj(;)e(other)i(ma)o(y)e(w)o (an)o(t)h(it)75 244 y(to)h(matc)o(h)e(C)i Ff(double)p Fj(;)e(MPI)i(will)f(enforce)i(one)f(of)f(the)i(t)o(w)o(o.\))166 296 y(Another)c(\(non\)solution)f(is)h(to)g(ignore)f(the)h(issue)h(and) e(let)h(the)g(b)q(eha)o(vior)g(of)f(C)g(to)h(F)m(ortran)f(comm)o (unication)75 346 y(b)q(e)h(unde\014ned.)19 b(The)11 b(user)h(can)f(alw)o(a)o(ys)e(matc)o(h)h(language)g(at)g(sending)h(and) g(receiving)g(end)g(if)f(he)h(or)g(she)g(is)g(willing)75 396 y(to)j(struggle)g(with)f(the)i(issue)g(of)e(calling)f(MPI)i(C)g (functions)g(from)e(F)m(ortran)i(or)g(vice-v)o(ersa.)75 632 y Fm(2.6)70 b(Receiv)n(e)20 b(Criteria)75 737 y Fs(The)d(selection) h(of)f(a)f(message)h(b)o(y)f(a)h(receiv)o(e)h(op)q(eration)f(is)g(done) g(uniquely)i(according)f(to)e(the)h(v)m(alue)75 793 y(of)e(the)g (message)g(en)o(v)o(elop)q(e.)21 b(The)15 b(receiv)o(e)h(op)q(eration)f (sp)q(eci\014es)i(an)e Ft(en)o(v)o(elop)q(e)i(pattern)p Fs(;)f(a)e(message)75 850 y(can)21 b(b)q(e)g(receiv)o(ed)h(b)o(y)f (that)f(receiv)o(e)i(op)q(eration)f(only)g(if)g(its)g(en)o(v)o(elop)q (e)h(matc)o(hes)f(that)f(pattern.)36 b(A)75 906 y(pattern)18 b(sp)q(eci\014es)j(v)m(alues)f(for)e(the)h Fl(source)p Fs(,)f Fl(tag)h Fs(and)g Fl(context)f Fs(\014elds)i(of)e(the)h(message) f(en)o(v)o(elop)q(e.)75 963 y(In)k(addition,)j(the)c(v)m(alue)i(for)f (the)f Fl(dest)h Fs(\014eld)h(is)f(set,)h(implicitly)l(,)j(to)21 b(b)q(e)i(equal)f(to)f(the)h(receiving)75 1019 y(pro)q(cess)13 b(id.)20 b(The)13 b(receiv)o(er)h(ma)o(y)e(sp)q(ecify)i(a)e(wildcard)i Fl(MPI)p 1093 1019 15 2 v 17 w(ANY)p 1182 1019 V 17 w(SOURCE)e Fs(v)m(alue)i(for)e Fl(source)p Fs(,)g(and/or)g(a)75 1076 y(wildcard)17 b Fl(MPI)p 335 1076 V 17 w(ANY)p 424 1076 V 16 w(TAG)e Fs(v)m(alue)i(for)e Fl(tag)p Fs(,)g(indicating)i (that)e(an)o(y)g(source)h(and/or)f(tag)f(are)h(acceptable.)75 1132 y(It)i(cannot)g(sp)q(ecify)h(a)f(wildcard)h(v)m(alue)g(for)f Fl(context)f Fs(or)g Fl(dest)p Fs(.)25 b(Th)o(us,)17 b(a)f(message)h(can)g(b)q(e)h(receiv)o(ed)75 1188 y(b)o(y)d(a)f(receiv) o(e)i(op)q(eration)f(only)h(if)f(it)g(is)h(addressed)f(to)f(the)h (receiving)i(task,)d(has)g(a)h(matc)o(hing)g(con)o(text,)75 1245 y(has)e(matc)o(hing)g(source)h(unless)g(source=)p Fl(MPI)p 854 1245 V 17 w(ANY)p 943 1245 V 17 w(SOURCE)f Fs(in)h(the)f(pattern,)g(and)g(has)h(a)f(matc)o(hing)g(tag)75 1301 y(unless)j(tag=)p Fl(MPI)p 383 1301 V 17 w(ANY)p 472 1301 V 16 w(TAG)f Fs(in)h(the)f(pattern.)166 1360 y(The)f(length)h(of)e(the)h(receiv)o(ed)h(message)e(m)o(ust)h(b)q(e)g (less)h(then)f(or)f(equal)i(the)f(length)h(of)e(the)h(receiv)o(e)75 1416 y(bu\013er.)31 b(I.e.,)19 b(all)g(incoming)h(data)e(m)o(ust)h (\014t,)g(without)f(truncation,)i(in)o(to)f(the)f(receiv)o(e)i (bu\013er.)31 b(It)18 b(is)75 1473 y(erroneous)e(to)f(receiv)o(e)h(a)g (message)f(whic)o(h)i(length)f(exceed)h(the)f(receiv)o(e)h(bu\013er,)e (and)h(the)g(outcome)f(of)75 1529 y(program)f(where)i(this)f(o)q(ccurs) h(is)f(undetermined.)75 1683 y Fm(2.7)70 b(Comm)n(unication)20 b(Mo)r(de)75 1788 y Fs(A)15 b(send)h(op)q(eration)f(can)h(o)q(ccur)f (in)h(one)g(of)f(three)g(mo)q(des:)75 1889 y Ft(ST)l(AND)o(ARD)22 b Fs(The)10 b(send)h(ma)o(y)f(b)q(e)h(started)f(whether)h(or)f(not)g(a) g(matc)o(hing)h(receiv)o(e)g(has)f(b)q(een)i(p)q(osted.)75 1990 y Ft(READ)o(Y)22 b Fs(The)15 b(send)h(ma)o(y)f(b)q(e)h(started)e (only)i(if)f(a)g(matc)o(hing)g(receiv)o(e)i(has)e(b)q(een)h(p)q(osted.) 75 2091 y Ft(SYNCHR)o(ONOUS)22 b Fs(The)17 b(send)g(op)q(eration)f (will)i(not)e(complete)h(successfully)h(un)o(til)f(it)g(is)f(guaran-) 189 2147 y(teed)f(that)g(the)g(message)g(sen)o(t)g(will)h(b)q(e)g (receiv)o(ed.)166 2248 y(The)23 b(start)e(and)h(completion)i(of)e(a)g Ft(standard)k(send)c Fs(do)g(not)g(dep)q(end)i(on)e(the)h(status)e(of)h (a)75 2305 y(matc)o(hing)15 b(receiv)o(e.)166 2363 y(A)d Ft(ready)i(send)d Fs(should)j(b)q(e)e(started)g(only)h(if)f(a)g(matc)o (hing)g(receiv)o(e)i(is)e(already)h(p)q(osted;)g(otherwise)75 2420 y(the)22 b(op)q(eration)g(is)h(erroneous)e(and)h(its)h(outcome)e (is)i(unde\014ned.)42 b(In)22 b(some)g(systems,)g(this)h(allo)o(ws)75 2476 y(the)16 b(remo)o(v)m(al)h(of)f(a)g(hand-shak)o(e)h(op)q(eration)f (that)g(is)h(otherwise)f(required,)h(and)g(results)g(in)g(impro)o(v)o (ed)75 2532 y(p)q(erformance.)j(Its)15 b(completion)i(do)q(es)e(not)g (dep)q(end)i(on)e(the)g(status)f(of)h(a)g(matc)o(hing)g(receiv)o(e.)166 2591 y(A)g Ft(sync)o(hronous)i(send)e Fs(can)h(b)q(e)g(started)e (irresp)q(ectiv)o(e)j(of)e(the)h(status)f(of)g(a)g(matc)o(hing)g (receiv)o(e.)75 2647 y(It)e(will)h(complete)g(only)f(if)h(a)e(matc)o (hing)h(receiv)o(e)h(is)f(p)q(osted,)h(and)f(the)g(receiv)o(e)g(op)q (eration)h(is)f(committed)75 2704 y(to)20 b(receiv)o(e)i(the)g(message) e(sen)o(t)h(b)o(y)g(the)g(secure)h(send.)38 b(\(I.e.,)22 b(the)f(receiv)o(e)h(has)f(b)q(een)h(p)q(osted,)h(the)p eop %%Page: 29 34 29 33 bop 75 -100 a Fq(2.8.)34 b(COMMUNICA)l(TION)16 b(OBJECTS)1004 b Fs(29)75 45 y(incoming)18 b(message)f(has)f(b)q(een)i (matc)o(hed)f(to)g(this)g(p)q(osted)g(receiv)o(e,)h(and)f(the)g(receiv) o(e)h(op)q(eration)f(has)75 102 y(already)e(started)g(and)g(cannot)g(b) q(e)h(cancelled)i(an)o(ymore.\))166 160 y(There)d(is)h(only)g(one)f (receiv)o(e)h(mo)q(de,)f(whic)o(h)h(can)g(matc)o(h)e(an)o(y)h(of)g(the) g(send)h(mo)q(des.)166 219 y(Comm)o(unication)d(imp)q(oses)g(an)f (order)h(on)f(the)h(ev)o(en)o(ts)f(o)q(ccurring)h(at)f(the)h(comm)o (unicating)g(no)q(des.)75 275 y(It)i(is)g(alw)o(a)o(ys)e(the)i(case)g (that)e(the)i(completion)h(of)e(a)g(receiv)o(e)i(o)q(ccurs)e(after)g (the)h(start)e(of)h(the)h(matc)o(hing)75 332 y(send.)29 b(If)19 b(the)f(sync)o(hronous)g(send)h(mo)q(de)f(is)h(used,)g(then)g (it)f(is)h(also)f(the)g(case)g(that)g(the)g(completion)75 388 y(of)i(a)h(send)g(o)q(ccurs)g(after)f(the)g(start)g(of)g(the)h (matc)o(hing)f(receiv)o(e.)38 b(Of)20 b(course,)i(on)f(eac)o(h)f(pro)q (cess,)i(a)75 445 y(comm)o(unication)16 b(completes)f(after)f(it)h(is)g (started.)k(No)c(other)f(order)h(is)g(imp)q(osed)h(b)o(y)f(MPI.)f (I.e.,)h(if)g(the)75 501 y(standard)g(send)g(mo)q(de)h(is)f(used,)h (the)f(send)h(op)q(eration)f(ma)o(y)f(complete)i(b)q(efore)g(the)f (matc)o(hing)g(receiv)o(e)75 557 y(w)o(as)f(started.)166 692 y Fk(Implemen)o(tati)o(on)h(note:)45 b Fj(A)16 b(ready)h(send)g (can)f(b)q(e)h(implemen)o(ted)d(as)i(a)g(regular)h(send;)g(in)f(suc)o (h)h(case)75 742 y(there)k(will)d(b)q(e)j(no)e(p)q(erformance)h(adv)n (an)o(tage)f(\(or)h(disadv)n(an)o(tage\))f(for)h(the)g(use)h(of)e (ready)h(send;)j(and)d(some)75 792 y(erroneous)15 b(programs)e(ma)o(y)f (go)h(undetected)j(\(and)e(ma)o(y)e(still)h(pro)q(duce)i(correct)h (results\).)166 844 y(A)e(p)q(ossible)g(comm)o(unication)c(proto)q(col) k(for)g(the)g(v)n(arious)f(comm)o(unication)e(mo)q(des)i(is)h(outlined) f(b)q(elo)o(w:)166 896 y Ff(ready)21 b(send)p Fj(:)e(The)c(message)f (is)h(sen)o(t)g(asap;)g(an)g(ac)o(kno)o(wledge)f(is)h(returned)h(b)o(y) f(receiv)o(er.)22 b(The)15 b(proto)q(col)75 945 y(is)f(\(send,)g(ac)o (k\).)166 997 y Ff(synchronous)19 b(send:)29 b Fj(The)20 b(sender)g(sends)h(a)e(request-to-send)j(message.)34 b(The)19 b(receiv)o(er)i(stores)g(this)75 1047 y(request.)f(When)15 b(a)f(matc)o(hing)e(receiv)o(e)j(is)f(p)q(osted,)h(the)g(receiv)o(er)h (sends)f(bac)o(k)f(a)g(p)q(ermission-to-send)g(message,)75 1097 y(and)g(the)g(sender)i(no)o(w)d(sends)i(the)g(message.)j(The)c (proto)q(col)g(is)f(\(req,)h(ac)o(k-req,)g(send,)h(ac)o(k\).)166 1149 y Ff(standard)20 b(send:)e Fj(First)d(proto)q(col)f(ma)o(y)e(b)q (e)j(used)h(for)d(short)i(messages,)f(and)g(second)i(proto)q(col)e(for) g(long)75 1199 y(messages.)166 1257 y(Of)g(course,)g(there)i(are)e(man) o(y)e(other)i(p)q(ossible)h(c)o(hoices.)75 1495 y Fm(2.8)70 b(Comm)n(unication)20 b(Ob)t(jects)75 1601 y Fs(An)g(opaque)g(comm)o (unication)h(ob)s(ject)e(iden)o(ti\014es)i(v)m(arious)g(prop)q(erties)f (of)g(a)f(comm)o(unication)i(op)q(er-)75 1657 y(ation,)i(suc)o(h)f(as)f (the)h(bu\013er)g(descriptor)g(that)f(is)h(asso)q(ciated)g(with)g(it,)h (its)f(con)o(text,)g(the)g(tag)f(and)75 1714 y(destination)16 b(parameters)f(to)g(b)q(e)h(used)g(for)e(a)h(send,)h(or)f(the)g(tag)g (and)h(source)f(parameters)g(to)f(b)q(e)i(used)75 1770 y(for)g(a)g(receiv)o(e.)24 b(In)17 b(addition,)g(this)g(ob)s(ject)f (stores)g(information)g(ab)q(out)g(the)h(status)e(of)h(the)h(last)f (com-)75 1827 y(m)o(unication)g(op)q(eration)g(that)e(w)o(as)h(p)q (erformed)g(with)h(this)g(ob)s(ject.)j(This)d(ob)s(ject)f(is)g (accessed)h(using)g(a)75 1883 y(comm)o(unication)g(handle.)166 1942 y(One)g(can)f(consider)h(comm)o(unication)g(op)q(erations)f(to)f (consist)i(of)f(the)g(follo)o(wing)g(sub)q(op)q(erations:)75 2058 y Ft(INIT\(op)q(eration,)k(params,)d(handle\))24 b Fs(Pro)q(cess)15 b(pro)o(vides)f(all)i(relev)m(an)o(t)e(parameters)g (for)g(its)g(par-)189 2115 y(ticipation)k(in)g(the)g(comm)o(unication)g (op)q(eration)f(\(t)o(yp)q(e)g(of)g(op)q(eration,)h(data)e(bu\013er,)i (tag,)e(par-)189 2171 y(ticipan)o(ts,)f(etc.\).)k(An)d(ob)s(ject)e(is)i (created)f(that)g(iden)o(ti\014es)i(the)e(op)q(eration.)75 2274 y Ft(ST)l(AR)l(T\(handle\))24 b Fs(The)16 b(comm)o(unication)g(op) q(eration)f(is)h(started)75 2376 y Ft(COMPLETE\(handle\))25 b Fs(The)16 b(comm)o(unication)g(op)q(eration)f(is)h(completed.)75 2478 y Ft(FREE\(handle\))25 b Fs(The)16 b(comm)o(unication)g(ob)s (ject,)e(and)h(asso)q(ciated)h(resources)f(are)g(freed.)75 2595 y(Correct)f(in)o(v)o(o)q(cation)i(of)f(these)g(sub)q(op)q (erations)h(is)g(a)f(sequence)h(of)f(the)g(form)551 2704 y Ft(INIT)i Fs(\()p Ft(ST)l(AR)l(T)g(COMPLETE)p Fs(\))1208 2685 y Fe(\003)1246 2704 y Ft(FREE)p Fr(:)p eop %%Page: 30 35 30 34 bop 75 -100 a Fs(30)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fs(I.e.,)c(an)g(ob)s (ject)g(needs)h(b)q(e)f(created)h(b)q(efore)f(comm)o(unication)h(o)q (ccurs;)g(it)f(can)g(b)q(e)h(reused)g(only)g(after)e(the)75 102 y(previous)16 b(use)f(has)f(completed;)i(and)f(it)g(needs)h(to)e(b) q(e)h(freed)h(ev)o(en)o(tually)f(\(of)g(course,)f(one)h(can)g(assume)75 158 y(that)f(all)j(ob)s(jects)d(are)h(freed)h(at)e(program)g (termination,)h(b)o(y)h(default\).)166 215 y(The)h(ab)q(o)o(v)o(e)f (scenario)i(p)q(ertains)f(to)f Fh(p)n(ersistent)g Fs(ob)s(jects.)24 b(One)18 b(can)f(also)f(create)h Fh(ephemer)n(al)g Fs(ob-)75 272 y(jects.)h(Suc)o(h)12 b(an)f(ob)s(ject)g(p)q(ersists)h(only)f(un)o (til)i(the)e(comm)o(unication)h(op)q(eration)f(is)h(completed,)h(at)d (whic)o(h)75 328 y(p)q(oin)o(t)i(it)g(is)g(destro)o(y)o(ed.)18 b(Th)o(us)11 b(the)h(correct)f(in)o(v)o(o)q(cation)h(of)f(sub)q(op)q (erations)i(with)e(an)h(ephemeral)h(ob)s(ject)75 385 y(is)j Ft(INIT)h(ST)l(AR)l(T)g(COMPLETE)p Fs(.)166 442 y(A)i(user)g(ma)o(y)f(directly)j(in)o(v)o(ok)o(e)e(these)g(sub)q(op)q (erations.)32 b(This)20 b(w)o(ould)f(allo)o(w)g(the)g(amortization)75 498 y(of)j(the)g(o)o(v)o(erhead)g(of)g(setting)h(up)g(a)f(comm)o (unication)h(o)o(v)o(er)f(man)o(y)g(successiv)o(e)h(uses)g(of)f(the)g (same)75 555 y(handle,)14 b(and)e(allo)o(ws)g(the)h(o)o(v)o(erlap)f(of) f(comm)o(unication)i(and)g(computation.)18 b(Simpler)c(comm)o (unication)75 611 y(op)q(erations)i(com)o(bine)g(sev)o(eral)g(of)f (these)h(sub)q(op)q(erations)h(in)o(to)e(one)h(op)q(eration,)f(th)o(us) h(simplifying)i(the)75 668 y(use)13 b(of)g(comm)o(unication)h(primitiv) o(es.)20 b(Th)o(us,)13 b(one)h(only)f(needs)h(to)f(sp)q(ecify)h (precisely)h(the)e(seman)o(tics)g(of)75 724 y(these)h(sub)q(op)q (erations)g(in)h(order)e(to)g(sp)q(ecify)i(the)f(seman)o(tics)f(of)h (MPI)f(p)q(oin)o(t)h(to)f(p)q(oin)o(t)h(comm)o(unication)75 781 y(op)q(erations.)166 838 y(W)l(e)h(sa)o(y)f(that)f(a)i(comm)o (unication)g(op)q(eration)g(\(send)g(or)f(receiv)o(e\))h(is)g Ft(p)q(osted)g Fs(once)g(a)g Ft(start)f Fs(sub-)75 894 y(op)q(eration)h(has)f(b)q(een)i(in)o(v)o(ok)o(ed;)f(the)g(op)q (eration)f(is)i Ft(completed)f Fs(once)g(the)g Ft(complete)g Fs(sub)q(op)q(eration)75 951 y(has)i(completed.)28 b(A)17 b(send)h(and)g(a)f(receiv)o(e)h(op)q(eration)g Ft(matc)o(h)f Fs(if)h(the)f(receiv)o(e)i(pattern)e(sp)q(eci\014ed)i(b)o(y)75 1007 y(the)c(receiv)o(e)h(matc)o(hes)f(the)g(message)g(en)o(v)o(elop)q (e)i(created)e(b)o(y)g(the)g(send.)75 1133 y Fi(2.8.1)55 b(Comm)n(unication)17 b(Ob)s(ject)h(Creation)75 1220 y Fs(An)i(ob)s(ject)f(for)g(a)h(send)g(op)q(eration)g(is)g(created)g(b) o(y)f(a)h(call)h(to)e Ft(MPI)p 1310 1220 16 2 v 18 w(INIT)p 1445 1220 V 18 w(xSEND)p Fs(.)h(The)f(pre\014x)75 1277 y(to)g Fl(SEND)h Fs(indicates)h(the)f(comm)o(unication)h(mo)q(de:)30 b(none)20 b(for)g Fl(standard)p Fs(,)f Fl(R)h Fs(for)f Fl(ready)p Fs(,)i(and)f Fl(S)g Fs(for)75 1333 y Fl(synchronous)p Fs(.)i(A)16 b(call)i(to)e Ft(MPI)p 675 1333 V 18 w(INIT)p 810 1333 V 19 w(RECV)g Fs(is)h(similarly)h(used)f(for)f(creating)g(an)h (ob)s(ject)e(for)h(a)75 1390 y(receiv)o(e)i(op)q(eration.)27 b(The)18 b(creation)g(of)f(a)g(comm)o(unication)h(ob)s(ject)f(is)h(a)f (lo)q(cal)i(op)q(eration)e(that)g(need)75 1446 y(not)e(in)o(v)o(olv)o (e)h(comm)o(unication)g(with)f(a)g(remote)g(pro)q(cess.)166 1539 y Ft(MPI)p 275 1539 V 18 w(INIT)p 410 1539 V 19 w(SEND)i(\(handle,)i(bu\013er)p 929 1539 V 18 w(handle,)f(dest,)e(tag,) i(con)o(text,)f(p)q(ersistence\))166 1632 y Fs(Creates)d(a)h(comm)o (unication)h(ob)s(ject)f(for)f(standard)h(mo)q(de)h(send.)k(P)o (arameters)14 b(are)75 1728 y Ft(OUT)k(handle)24 b Fs(message)11 b(handle.)20 b(The)12 b(handle)h(should)f(not)f(b)q(e)i(asso)q(ciated)f (with)f(an)o(y)h(ob)s(ject)f(b)q(efore)189 1785 y(the)k(call.)75 1882 y Ft(IN)i(bu\013er)p 291 1882 V 18 w(handle)24 b Fs(handle)16 b(to)f(send)h(bu\013er)f(descriptor)75 1978 y Ft(IN)i(dest)23 b Fs(rank)15 b(in)h(con)o(text)e(of)h(destination)h (\(in)o(teger\))75 2075 y Ft(IN)h(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(sen)o(t)g(with)g(this)h(handle)g(\(in)o (teger\))75 2172 y Ft(IN)h(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(sen)o(t)g(with)h(this)f(handle)75 2269 y Ft(IN)i(p)q(ersistence)23 b Fs(handle)h(p)q(ersistence)g(\(state)d(t) o(yp)q(e,)j(with)f(t)o(w)o(o)e(v)m(alues:)35 b Fl(MPI)p 1527 2269 15 2 v 17 w(PERSISTENT)21 b Fs(and)189 2325 y Fl(MPI)p 264 2325 V 16 w(EPHEMERAL)p Fs(\))166 2458 y Ft(MPI)p 275 2458 16 2 v 18 w(INIT)p 410 2458 V 19 w(RSEND)16 b(\(handle,)h(bu\013er)p 965 2458 V 18 w(handle,)g(dest,)e (tag,)i(con)o(text,)f(p)q(ersistence\))166 2550 y Fs(Creates)e(a)h (comm)o(unication)h(ob)s(ject)f(for)f(standard)h(mo)q(de)h(send.)k(P)o (arameters)14 b(are)75 2647 y Ft(OUT)k(handle)24 b Fs(message)11 b(handle.)20 b(The)12 b(handle)h(should)f(not)f(b)q(e)i(asso)q(ciated)f (with)f(an)o(y)h(ob)s(ject)f(b)q(efore)189 2704 y(the)k(call.)p eop %%Page: 31 36 31 35 bop 75 -100 a Fq(2.8.)34 b(COMMUNICA)l(TION)16 b(OBJECTS)1004 b Fs(31)75 45 y Ft(IN)17 b(bu\013er)p 291 45 16 2 v 18 w(handle)24 b Fs(handle)16 b(to)f(send)h(bu\013er)f (descriptor)75 137 y Ft(IN)i(dest)23 b Fs(rank)15 b(in)h(con)o(text)e (of)h(destination)h(\(in)o(teger\))75 230 y Ft(IN)h(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(sen)o(t)g(with)g(this)h(handle)g (\(in)o(teger\))75 322 y Ft(IN)h(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(sen)o(t)g(with)h(this)f(handle)75 414 y Ft(IN)i(p)q(ersistence)23 b Fs(handle)h(p)q(ersistence)g(\(state)d(t) o(yp)q(e,)j(with)f(t)o(w)o(o)e(v)m(alues:)35 b Fl(MPI)p 1527 414 15 2 v 17 w(PERSISTENT)21 b Fs(and)189 470 y Fl(MPI)p 264 470 V 16 w(EPHEMERAL)p Fs(\))166 596 y Ft(MPI)p 275 596 16 2 v 18 w(INIT)p 410 596 V 19 w(SSEND)c(\(handle,)i(bu\013er) p 958 596 V 18 w(handle,)f(dest,)e(tag,)i(con)o(text,)f(p)q (ersistence\))166 688 y Fs(Creates)d(a)h(comm)o(unication)h(ob)s(ject)f (for)f(sync)o(hronous)i(mo)q(de)f(send.)21 b(P)o(arameters)13 b(are)75 777 y Ft(OUT)18 b(handle)24 b Fs(message)11 b(handle.)20 b(The)12 b(handle)h(should)f(not)f(b)q(e)i(asso)q(ciated)f (with)f(an)o(y)h(ob)s(ject)f(b)q(efore)189 834 y(the)k(call.)75 926 y Ft(IN)i(bu\013er)p 291 926 V 18 w(handle)24 b Fs(handle)16 b(to)f(send)h(bu\013er)f(descriptor)75 1018 y Ft(IN)i(dest)23 b Fs(rank)15 b(in)h(con)o(text)e(of)h(destination)h(\(in)o(teger\))75 1110 y Ft(IN)h(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(sen)o(t)g (with)g(this)h(handle)g(\(in)o(teger\))75 1203 y Ft(IN)h(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(sen)o(t)g(with)h(this)f(handle)75 1295 y Ft(IN)i(p)q(ersistence)23 b Fs(handle)h(p)q(ersistence)g (\(state)d(t)o(yp)q(e,)j(with)f(t)o(w)o(o)e(v)m(alues:)35 b Fl(MPI)p 1527 1295 15 2 v 17 w(PERSISTENT)21 b Fs(and)189 1351 y Fl(MPI)p 264 1351 V 16 w(EPHEMERAL)p Fs(\))166 1477 y Ft(MPI)p 275 1477 16 2 v 18 w(INIT)p 410 1477 V 19 w(RECV)14 b(\(handle,)j(bu\013er)p 931 1477 V 18 w(handle,)f(source,)e(tag,)i(con)o(text,)f(p)q(ersistence\))166 1625 y Fs(Creates)f(a)h(receiv)o(e)h(handle.)22 b(P)o(arameters)13 b(are)75 1715 y Ft(OUT)18 b(handle)24 b Fs(message)11 b(handle.)20 b(The)12 b(handle)h(should)f(not)f(b)q(e)i(asso)q(ciated)f (with)f(an)o(y)h(ob)s(ject)f(b)q(efore)189 1771 y(the)k(call.)75 1863 y Ft(IN)i(bu\013er)p 291 1863 V 18 w(handle)24 b Fs(handle)16 b(to)f(receiv)o(e)h(bu\013er)f(descriptor.)75 1956 y Ft(IN)i(source)22 b Fs(rank)15 b(in)h(con)o(text)f(of)g(source,) g(or)f Fl(MPI)p 974 1956 15 2 v 17 w(ANY)p 1063 1956 V 17 w(SOURCE)g Fs(\(in)o(teger\).)75 2048 y Ft(IN)j(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(receiv)o(ed)h(with)g(this)f (handle,)h(or)f Fl(MPI)p 1341 2048 V 17 w(ANY)p 1430 2048 V 16 w(TAG)g Fs(\(in)o(teger\).)75 2140 y Ft(IN)i(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(receiv)o(ed)h(with)g(this)g (handle.)75 2232 y Ft(IN)h(p)q(ersistence)23 b Fs(handle)h(p)q (ersistence)g(\(state)d(t)o(yp)q(e,)j(with)f(t)o(w)o(o)e(v)m(alues:)35 b Fl(MPI)p 1527 2232 V 17 w(PERSISTENT)21 b Fs(and)189 2289 y Fl(MPI)p 264 2289 V 16 w(EPHEMERAL)p Fs(\))166 2378 y(See)16 b(Section)g(2.4.2)d(for)i(a)g(discussion)i(of)d(source,)h (tag)g(and)g(con)o(text.)166 2511 y Fk(Discussion:)166 2561 y Fj(Ephemeral)e(\\p)q(ersisten)o(t)h(handles")g(are)f(not)h (terribly)f(useful)g(as)h(suc)o(h.)k(But)c(their)g(existence)h(allo)o (w)c(us)j(to)75 2611 y(describ)q(e)i(all)c(the)j(higher)f(lev)o(el)f (send/receiv)o(e)j(op)q(erations)e(in)g(terms)f(of)h(basic)g(op)q (erations)g(on)f(handles.)p eop %%Page: 32 37 32 36 bop 75 -100 a Fs(32)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fi(2.8.2)55 b(Comm)n(unication)17 b(Start)75 166 y Ft(MPI)p 184 166 16 2 v 18 w(ST)l(AR)l(T\(handle\))75 291 y(IN)g(handle)24 b Fs(comm)o(unication)16 b(handle)166 380 y(The)i Fl(MPI)p 337 380 15 2 v 17 w(START)g Fs(function)h(starts)e(the)h(execution)i (of)d(a)h(comm)o(unication)h(op)q(eration)g(\(send)f(or)75 436 y(receiv)o(e\).)34 b(A)20 b(send)g(can)g(b)q(e)g(started)f(pro)o (vided)i(that)e(the)g(previous)i(send)f(using)h(the)e(same)h(handle)75 493 y(has)c(completed,)h(or)f(as)g(so)q(on)g(as)g(the)g(handle)i(has)e (b)q(een)h(created,)f(if)h(there)f(w)o(as)g(no)g(preceding)i(send.)75 549 y(If)f(the)g(send)g(mo)q(de)g(is)g Fl(ready)f Fs(then,)h(in)g (addition,)h(a)e(matc)o(hing)h(receiv)o(e)g(should)h(ha)o(v)o(e)e(b)q (een)i(already)75 606 y(p)q(osted.)h(The)14 b(send)f(bu\013er)g(should) h(not)f(b)q(e)g(up)q(dated)h(after)e(the)h(send)h(is)f(started,)g(un)o (til)h(the)f(op)q(eration)75 662 y(completes.)166 719 y(A)18 b(receiv)o(e)g(can)g(b)q(e)h(started)e(as)g(so)q(on)h(as)f(the)h (preceding)h(receiv)o(e)g(using)g(the)f(same)f(handle)i(has)75 775 y(completed,)i(or)e(as)f(so)q(on)i(as)e(the)i(handle)g(has)f(b)q (een)i(created)e(is)h(there)f(w)o(as)g(no)g(preceding)h(receiv)o(e.)75 832 y(The)e(receiv)o(e)h(bu\013er)f(should)h(not)e(b)q(e)i(accessed)g (after)e(the)h(receiv)o(e)h(is)f(started,)g(un)o(til)h(the)f(op)q (eration)75 888 y(completes.)166 945 y(A)f(program)f(that)g(do)q(es)h (not)g(satisfy)f(these)i(conditions)g(is)f(erroneous)g(and)g(its)g (outcome)g(is)g(un-)75 1001 y(determined.)166 1057 y(The)h(call)h (returns)f(as)f(so)q(on)h(as)f(p)q(ossible.)30 b(A)18 b(call)h(that)e(starts)g(a)g(send)i(can)f(return)g(b)q(efore)g(the)75 1114 y(data)13 b(to)g(b)q(e)h(sen)o(t)f(has)h(b)q(een)h(cleared)f(out)f (of)h(the)f(send)h(bu\013er)g(and)g(b)q(efore)g(an)o(y)f(matc)o(hing)h (receiv)o(e)g(has)75 1170 y(b)q(een)j(p)q(osted)g(\(this)f(is)h(true)f (for)g(all)h(send)g(mo)q(des,)g(including)i Fl(synchronous)p Fs(\).)i(A)16 b(call)i(that)d(starts)g(a)75 1227 y(receiv)o(e)h(can)f (return)h(b)q(efore)f(andy)g(data)g(has)g(b)q(een)h(receiv)o(ed)h(in)f (the)f(receiv)o(e)h(bu\013er.)75 1348 y Fi(2.8.3)55 b(Comm)n(unication) 17 b(Completion)75 1433 y Fs(The)j(completion)h(of)f(a)f(send)i(op)q (eration)f(indicates)h(that)e(the)h(sender)h(is)f(no)o(w)f(free)h(to)f (up)q(date)i(the)75 1490 y(lo)q(cations)i(in)g(the)f(send)h(bu\013er,)h (or)e(an)o(y)g(other)g(lo)q(cation)h(that)e(can)i(b)q(e)g(referenced)g (b)o(y)f(the)g(send)75 1546 y(op)q(eration.)32 b(It)19 b(do)q(es)g(not)g(indicate)i(that)d(the)h(message)g(has)g(b)q(een)h (receiv)o(ed;)i(rather,)d(it)g(ma)o(y)f(ha)o(v)o(e)75 1603 y(b)q(een)c(bu\013ered)f(b)o(y)f(the)h(comm)o(unication)g (subsystem.)19 b(Ho)o(w)o(ev)o(er,)12 b(if)h(a)f Fl(synchronous)f Fs(mo)q(de)i(send)g(w)o(as)75 1659 y(used,)i(the)g(completion)h(of)f (the)f(send)i(op)q(eration)f(indicates)h(that)e(a)h(matc)o(hing)g (receiv)o(e)h(w)o(as)e(initiated,)75 1716 y(and)h(that)g(the)g(message) g(will)i(ev)o(en)o(tually)f(b)q(e)g(receiv)o(ed)g(b)o(y)f(this)h(matc)o (hing)f(receiv)o(e.)166 1772 y(The)h(completion)g(of)g(a)f(receiv)o(e)h (op)q(eration)g(indicates)h(that)e(the)h(receiv)o(er)g(is)g(no)o(w)f (free)h(to)f(access)75 1829 y(the)d(lo)q(cations)h(in)g(the)f(receiv)o (e)i(bu\013er,)e(whic)o(h)h(con)o(tain)f(the)h(receiv)o(ed)g(message,)f (or)g(an)o(y)f(other)h(lo)q(cation)75 1885 y(that)g(can)g(b)q(e)h (referenced)h(b)o(y)e(the)h(receiv)o(e)g(op)q(eration.)19 b(It)13 b(do)q(es)g(not)f(indicate)i(that)d(the)i(matc)o(hing)f(send)75 1942 y(op)q(eration)j(has)g(completed)i(\(but)e(indicates,)h(of)f (course,)g(that)f(the)h(send)h(w)o(as)f(initiated\).)166 2033 y Ft(MPI)p 275 2033 16 2 v 18 w(W)-6 b(AIT)17 b(\()h(handle,)g (status\))75 2158 y(IN)f(handle)24 b Fs(comm)o(unication)16 b(handle)75 2250 y Ft(OUT)i(status)23 b Fs(handle)15 b(that)d(is)i(asso)q(ciated)f(with)h(return)f(status)f(ob)s(ject.)19 b(The)13 b(return)g(status)g(ob)s(ject)189 2306 y(is)i(up)q(dated)h(b)o (y)g(call.)166 2396 y(A)g(call)i(to)e Ft(MPI)p 467 2396 V 18 w(W)-6 b(AIT)16 b Fs(returns)g(when)h(the)f(send)h(op)q(eration)g (iden)o(ti\014ed)h(b)o(y)f Fl(handle)e Fs(is)i(com-)75 2452 y(plete.)166 2508 y(The)g(call)g(returns)g(a)f(handle)i(to)e(an)g (opaque)h(ob)s(ject)f(that)g(con)o(tains)g(information)h(on)f(the)h (com-)75 2565 y(pleted)f(op)q(eration)g({)f(the)g Ft(return)i(status)e Fs(ob)s(ject.)166 2657 y Ft(MPI)p 275 2657 V 18 w(TEST)j(\(handle,)h (\015ag,)e(status\))p eop %%Page: 33 38 33 37 bop 75 -100 a Fq(2.8.)34 b(COMMUNICA)l(TION)16 b(OBJECTS)1004 b Fs(33)75 45 y Ft(IN)17 b(handle)24 b Fs(comm)o(unication)16 b(handle)75 142 y Ft(OUT)i(\015ag)24 b Fs(logical)75 238 y Ft(OUT)18 b(status)23 b Fs(handle)17 b(that)d(is)i(asso)q(ciated)f(with)h(return)f(status)f(ob)s(ject.)166 335 y(A)f(call)h(to)f Ft(MPI)p 457 335 16 2 v 18 w(TEST)h Fs(returns)f Fl(flag=true)f Fs(if)i(the)f(op)q(eration)h(iden)o (ti\014ed)h(b)o(y)e Fl(handle)f Fs(is)i(com-)75 391 y(plete.)20 b(In)15 b(suc)o(h)f(case,)g(the)g(return)g(handle)h(p)q(oin)o(ts)f(to)f (an)h(opaque)g(ob)s(ject)g(that)f(con)o(tains)h(information)75 447 y(on)j(the)g(completed)g(op)q(eration.)25 b(It)17 b(returns)f Fl(flag=false)p Fs(,)g(otherwise.)24 b(In)18 b(suc)o(h)f(case,)g(the)f(v)m(alue)i(of)75 504 y(the)d(return)g(handle) i(is)f(unde\014ned.)166 561 y(A)g(call)i(to)e Fl(MPI)p 433 561 15 2 v 17 w(WAIT)f Fs(blo)q(c)o(ks)i(only)g(the)g(executing)h (thread.)23 b(If)17 b(the)f(executing)i(pro)q(cess)e(is)h(m)o(ul-)75 618 y(tithreaded,)f(then)g(other)f(threads)h(within)g(the)g(pro)q(cess) g(can)g(b)q(e)g(sc)o(heduled)i(for)d(execution.)22 b(The)16 b(use)75 674 y(of)h(a)h(blo)q(c)o(king)h(receiv)o(e)f(op)q(eration)g (\()p Fl(MPI)p 803 674 V 17 w(WAIT)p Fs(\))e(allo)o(ws)i(the)g(op)q (erating)g(system)f(to)g(desc)o(hedule)j(the)75 730 y(blo)q(c)o(k)o(ed) 15 b(thread)f(and)g(sc)o(hedule)i(another)e(thread)g(for)g(execution,)h (if)f(suc)o(h)h(is)f(a)o(v)m(ailable.)22 b(The)14 b(use)h(of)e(a)75 787 y(non)o(blo)q(c)o(king)k(receiv)o(e)g(op)q(eration)g(\()p Fl(MPI)p 776 787 V 16 w(TEST)p Fs(\))e(allo)o(ws)h(the)g(user)g(to)g (sc)o(hedule)h(alternativ)o(e)g(activities)75 843 y(within)f(a)f (single)i(thread)e(of)g(execution.)166 900 y(The)j(in)o(tended)i (implemen)o(tation)g(of)e Fl(MPI)p 909 900 V 17 w(TEST)f Fs(is)i(for)f(that)g(op)q(eration)g(to)g(return)g(as)g(so)q(on)g(as)75 957 y(p)q(ossible.)k(Ho)o(w)o(ev)o(er,)14 b(if)i(rep)q(eatedly)h (called)g(for)d(an)i(op)q(eration)f(that)g(is)h(enabled,)h(it)e(m)o (ust)g(ev)o(en)o(tually)75 1013 y(succeed.)166 1070 y(The)23 b(return)f(status)f(ob)s(ject)h(for)g(a)g(send)h(op)q(eration)g (carries)f(no)h(information.)41 b(The)23 b(return)75 1127 y(status)17 b(ob)s(ject)h(for)f(a)h(receiv)o(e)h(op)q(eration)g (carries)f(information)g(on)g(the)g(source,)h(tag)e(and)i(length)f(of) 75 1183 y(the)i(receiv)o(ed)h(message.)34 b(These)20 b(\014elds)h(are)e(required)i(b)q(ecause)g(the)f(receiv)o(e)h(op)q (eration)f(ma)o(y)f(ha)o(v)o(e)75 1240 y(sp)q(eci\014ed)i Fl(DONTCARE)d Fs(in)i(either)f(source)g(or)g(tag)f(\014eld,)i(and)f (the)g(message)g(ma)o(y)f(ha)o(v)o(e)h(b)q(een)h(shorter)75 1296 y(than)15 b(the)g(receiv)o(e)h(bu\013er.)166 1389 y Ft(MPI)p 275 1389 16 2 v 18 w(RETURN)p 522 1389 V 20 w(ST)l(A)l(TUS\()i(handle,)f(coun)o(t,)h(source,)e(tag\))75 1521 y(IN)h(handle)24 b Fs(handle)16 b(to)f(return)g(status)f(ob)s (ject)75 1617 y Ft(OUT)k(coun)o(t)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(elemen)o(ts)g(receiv)o(ed.)75 1714 y Ft(OUT)j(source)23 b Fs(rank)14 b(of)h(message)g(sender)h(in)g(message)f(con)o(text)f (\(in)o(teger\).)75 1810 y Ft(OUT)k(tag)24 b Fs(tag)14 b(of)h(receiv)o(ed)h(message)f(\(in)o(teger\).)166 1942 y Ft(MPI)p 275 1942 V 18 w(CREA)l(TE)p 510 1942 V 19 w(ST)l(A)l(TUS\()i(handle\))166 2035 y Fs(Creates)d(a)h(new)h(return)f (status)f(ob)s(ject.)75 2131 y Ft(OUT)k(handle)24 b Fs(handle)16 b(to)f(return)g(status)g(ob)s(ject.)1928 2179 y Fg(TO)c(DISCUSS)166 2304 y Fk(Discussion:)166 2354 y Fj(Should)i(w)o(e)h(ha)o(v)o(e)g(p)q (ersisten)o(t/ephemeral)h(status)g(ob)r(jects?)166 2405 y(The)e(use)g(of)f(a)h(return)g(status)h(ob)r(ject,)f(rather)g(than)g (a)f(list)g(of)g(parameters)h(ma)o(y)d(simplify)g(the)j(use)h(of)e(MPI) 75 2455 y(routines,)17 b(if)f(the)g(v)n(alues)g(stored)i(in)e(the)h(ob) r(ject)g(are)f(seldom)f(c)o(hec)o(k)o(ed.)27 b(A)16 b(prede\014ned)j (return)e(status)g(ob)r(ject)75 2504 y(should)d(b)q(e)h(pro)o(vided,)f (to)g(ease)h(programming.)h(The)f(main)d(reason)j(for)f(the)h(use)g(of) f(a)g(return-status)i(ob)r(ject)f(is)75 2554 y(that)i(one)g(w)o(an)o (ts)f(to)h(b)q(e)g(able)g(to)f(use)i(the)f(same)f Ff(MPI)p 949 2554 14 2 v 15 w(WAIT)g Fj(and)g Ff(MPI)p 1217 2554 V 15 w(TEST)g Fj(calls)g(for)h(c)o(hec)o(king)g(on)f(di\013eren)o(t)75 2604 y(t)o(yp)q(es)f(of)f(op)q(erations,)f(e.g.)19 b(b)q(oth)14 b(sends)i(and)d(receiv)o(es,)j(and)e(p)q(erhaps)h(new)g(op)q(erations)f (in)g(the)g(future.)20 b(These)75 2654 y(di\013eren)o(t)12 b(op)q(erations)g(return)g(di\013eren)o(t)g(information)c(\(alb)q(eit)j (w)o(e)h(could)f(decree)i(that)e(a)g(send)h(returns)h(the)f(same)75 2704 y(information)f(as)j(a)g(receiv)o(e\).)p eop %%Page: 34 39 34 38 bop 75 -100 a Fs(34)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fj(The)d(issue)h (of)f(memory)d(allo)q(cation)i(for)g(status)i(is)f(not)g(y)o(et)h(solv) o(ed.)j(There)e(is)d(a)h(desire)h(that)g(the)f(user)h(will)75 95 y(b)q(e)h(able)e(to)h(allo)q(cate)f(space)i(for)f(them)f(\(e.g.,)g (on)g(the)i(stac)o(k\).)166 145 y(W)m(e)i(return)i(n)o(um)o(b)q(er)d (of)h(elemen)o(ts)g(receiv)o(ed,)j(rather)e(than)f(b)o(ytes)i(receiv)o (ed)f(b)q(ecause)h(\\n)o(um)o(b)q(er)e(of)g(ele-)75 195 y(men)o(ts")g(is)h(closer)h(to)e(the)i(application)d(seman)o(tic)i(lev) o(el)f(\(user)i(do)q(es)g(not)f(need)h(to)f(b)q(e)g(a)o(w)o(are)g(of)f (the)i(size)g(of)75 244 y(elemen)o(ts\),)g(and)f(is)f(more)h(in)o(v)n (arian)o(t.)28 b(The)19 b(return)g(status)g(ob)r(ject)g(returns)h(the)e (n)o(um)o(b)q(er)g(of)f Fk(elemen)o(ts)e Fj(re-)75 294 y(ceiv)o(ed.)k(If)14 b(there)h(w)o(as)f(no)g(truncation,)g(then)h(this) f(is)g(equal)g(to)f(the)i(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(sen)o (t.)19 b(On)c(the)f(other)75 344 y(hand,)k(the)f(n)o(um)o(b)q(er)g(of)g (b)o(ytes)h(collected)g(from)d(the)j(sender)h(memory)m(,)c(the)j(n)o (um)o(b)q(er)e(of)h(b)o(ytes)h(sen)o(t)g(o)o(v)o(er)f(the)75 394 y(wire,)12 b(and)g(the)g(n)o(um)o(b)q(er)f(of)g(b)o(ytes)i(stored)f (in)g(the)g(receiv)o(er)h(memory)d(ma)o(y)g(all)g(b)q(e)i(di\013eren)o (t,)h(in)e(a)h(heterogeneous)75 444 y(en)o(vironmen)o(t.)23 b(The)17 b(n)o(um)o(b)q(er)e(of)g(elemen)o(ts)h(sen)o(t)h(can)f(b)q(e)g (computed)g(form)e(the)j(send)g(bu\013er)g(descriptor;)g(the)75 493 y(n)o(um)o(b)q(er)c(of)g(elemen)o(ts)h(receiv)o(ed)h(can)f(b)q(e)h (computed)e(from)g(the)h(receiv)o(e)h(bu\013er)g(descriptor)g(and)f (the)g(length)g(\(in)75 543 y(b)o(ytes\))h(of)e(the)i(receiv)o(ed)g (message.)75 747 y Fi(2.8.4)55 b(Multiple)17 b(Completions)75 833 y Fs(It)f(is)h(con)o(v)o(enien)o(t)f(to)g(b)q(e)g(able)h(to)f(w)o (ait)f(for)h(the)g(completion)h(of)e(an)o(y)h(or)g(all)h(the)f(op)q (erations)g(in)h(a)f(set,)75 889 y(rather)11 b(than)g(ha)o(ving)g(to)g (w)o(ait)g(for)f(a)h(sp)q(eci\014c)j(message.)k(A)11 b(call)h(to)f Fl(MPI)p 1296 889 15 2 v 17 w(WAITANY)f Fs(or)h Fl(MPI)p 1615 889 V 17 w(TESTANY)f Fs(can)75 946 y(b)q(e)17 b(used)g(to)e(w)o(ait)h(for)f(the)h(completion)i(of)d (one)i(out)e(of)h(sev)o(eral)g(op)q(erations;)h(a)e(call)j(to)d Fl(MPI)p 1694 946 V 17 w(WAITALL)75 1002 y Fs(can)g(b)q(e)h(used)g(to)f (w)o(ait)f(for)h(all)h(p)q(ending)h(op)q(erations)e(in)h(a)f(list.)166 1094 y Ft(MPI)p 275 1094 16 2 v 18 w(W)-6 b(AIT)l(ANY)16 b(\()i(list)p 677 1094 V 19 w(of)p 738 1094 V 19 w(handles,)g(coun)o (t,)f(index,)g(status\))166 1186 y Fs(Blo)q(c)o(ks)h(un)o(til)g(one)g (of)f(the)g(op)q(erations)h(asso)q(ciated)g(with)f(the)h(comm)o (unication)g(handles)h(in)f(the)75 1242 y(arra)o(y)13 b(has)i(completed.)21 b(Returns)15 b(the)g(index)h(of)e(that)g(handle)i (in)f(the)g(arra)o(y)l(,)f(and)g(returns)h(the)g(status)75 1299 y(of)g(that)f(op)q(eration)i(in)g(the)f(ob)s(ject)g(asso)q(ciated) g(with)h(the)f(status.)k(The)c(parameters)g(are:)75 1399 y Ft(IN)i(list)p 225 1399 V 20 w(of)p 287 1399 V 19 w(handles)23 b Fs(list)16 b(of)f(handles)h(to)f(comm)o(unication)h(ob)s(jects.)75 1491 y Ft(IN)h(coun)o(t)23 b Fs(list)16 b(length)g(\(in)o(teger\))75 1583 y Ft(OUT)i(index)23 b Fs(index)17 b(of)d(handle)j(for)d(op)q (eration)i(that)e(completed)i(\(in)o(teger\).)75 1676 y Ft(OUT)i(status)23 b Fs(handle)18 b(that)e(is)i(asso)q(ciated)f(with) g(return)f(status)g(ob)s(ject.)24 b(Ob)s(ject)17 b(is)g(set)g(to)f (return)189 1732 y(status)e(of)h(op)q(eration)g(that)g(completed.)166 1832 y(The)e(successful)i(execution)g(of)d Fl(MPI)p 788 1832 15 2 v 17 w(WAITANY\(list)p 1093 1832 V 16 w(of)p 1157 1832 V 17 w(handles,)22 b(index,)h(status\))13 b Fs(has)g(the)75 1889 y(same)k(e\013ect)g(as)h(the)f(successful)i (execution)g(of)e Fl(MPI)p 996 1889 V 16 w(WAIT\(handle[i],)22 b(status\))p Fs(,)17 b(where)h Fl(i)f Fs(is)h(the)75 1945 y(v)m(alue)e(returned)f(b)o(y)g Fl(index)f Fs(and)h Fl(handle[i])e Fs(is)i(the)g Fl(i)p Fs(-th)g(handle)h(in)f(the)g(list,) g(and)g(the)g(cancellation)75 2002 y(of)g(all)h(remaining)g(w)o(ait)f (op)q(erations.)166 2058 y(If)21 b(more)g(then)g(one)g(op)q(eration)h (is)f(enabled)i(and)e(can)g(terminate,)h(one)f(is)h(arbitrarily)g(c)o (hosen)75 2115 y(\(sub)s(ject)15 b(to)f(the)i(restrictions)f(on)g(op)q (eration)h(termination)f(order,)g(and)g(fairness,)g(see)h(Chapter)f (6\).)166 2171 y Fl(MPI)p 241 2171 V 17 w(WAITANY)23 b(\()g(list)p 592 2171 V 17 w(of)p 657 2171 V 17 w(handles,)g(count,)g (index,)g(status\))14 b Fs(is)147 2261 y Fl({MPI_WAIT)22 b(\(handle[0],)h(return_handle\);)e(index)j(=)f(0})h(||)f(...)75 2317 y(||)147 2374 y({MPI_WAIT)f(\(handle[count-1],)g(return_handle\);) g(index)h(=)g(count-1})166 2463 y Fs(\(\\)p Fo(jj)p Fs(")13 b(indicates)k(c)o(hoice;)f(one)f(of)g(the)g(alternativ)o(es)h(is)f(c)o (hosen,)g(nondeterministically)m(.\))166 2555 y Ft(MPI)p 275 2555 16 2 v 18 w(TEST)l(ANY)i(\()h(list)p 669 2555 V 20 w(of)p 731 2555 V 19 w(handles,)f(coun)o(t,)g(index,)g(status\)) 166 2647 y Fs(Causes)12 b(either)h(one)f(or)g(none)h(of)f(the)g(op)q (erations)g(asso)q(ciated)h(with)f(the)h(comm)o(unication)g(handles)75 2704 y(to)f(return.)19 b(In)13 b(the)g(former)f(case,)h(it)g(has)f(the) h(same)f(return)h(seman)o(tics)g(as)f(a)g(call)i(to)e Fl(MPI)p 1610 2704 15 2 v 17 w(WAIT)p 1723 2704 V 17 w(ANY)p Fs(.)f(In)p eop %%Page: 35 40 35 39 bop 75 -100 a Fq(2.9.)29 b(BLOCKING)16 b(COMMUNICA)l(TION)972 b Fs(35)75 45 y(the)15 b(latter)g(case,)g(it)h(returns)f(a)g(v)m(alue)h (of)f({1)f(in)j Fl(index)d Fs(and)h Fl(status)g Fs(is)h(unde\014ned.)21 b(The)16 b(parameters)75 102 y(are:)75 186 y Ft(IN)h(list)p 225 186 16 2 v 20 w(of)p 287 186 V 19 w(handles)23 b Fs(list)16 b(of)f(handles)h(to)f(comm)o(unication)h(ob)s(jects.)75 274 y Ft(IN)h(coun)o(t)23 b Fs(list)16 b(length)g(\(in)o(teger\))75 362 y Ft(OUT)i(index)23 b Fs(index)18 b(of)e(handle)h(for)f(op)q (eration)h(that)e(completed,)j(or)e(-1)g(if)h(none)f(completed)i(\(in)o (te-)189 418 y(ger\).)75 506 y Ft(OUT)g(status)23 b Fs(handle)c(that)d (is)h(asso)q(ciated)g(with)h(return)f(status)f(ob)s(ject.)24 b(Set)17 b(to)f(return)h(status)f(of)189 562 y(op)q(eration)f(that)g (completed,)g(if)h(an)o(y;)e(unde\014ned)k(when)d Fl(index)23 b(=)h(-1)p Fs(.)166 683 y Ft(MPI)p 275 683 V 18 w(W)-6 b(AIT)l(ALL\(list)p 624 683 V 20 w(of)p 686 683 V 19 w(handles,)18 b(coun)o(t,)f(list)p 1136 683 V 20 w(of)p 1198 683 V 19 w(statuss\))166 775 y Fs(Blo)q(c)o(ks)c(un)o(til)h(all)g (comm)o(unication)g(op)q(erations)f(asso)q(ciated)g(with)g(handles)h (in)g(the)f(list)h(complete,)75 831 y(and)h(return)h(the)f(status)f(of) h(all)h(these)f(op)q(erations.)20 b(The)c(parameters)e(are:)75 916 y Ft(IN)j(list)p 225 916 V 20 w(of)p 287 916 V 19 w(handles)23 b Fs(list)16 b(of)f(handles)h(to)f(comm)o(unication)h(ob)s (jects.)75 1004 y Ft(IN)h(coun)o(t)23 b Fs(lists)16 b(length)g(\(in)o (teger\))75 1091 y Ft(OUT)i(list)p 280 1091 V 20 w(of)p 342 1091 V 19 w(status)23 b Fs(Must)12 b(ha)o(v)o(e)h(the)g(same)f (length)i(as)f(the)g(\014rst)f(list.)20 b(Eac)o(h)13 b(return)f(status)h(ob)s(ject)189 1148 y(is)i(set)g(to)g(the)g(return)g (status)g(of)f(the)i(corresp)q(onding)g(op)q(eration)f(in)h(the)g (\014rst)e(list.)166 1309 y Fk(Discussion:)166 1358 y Fj(The)i(fairness)g(requiremen)o(t)g(giv)o(en)f(in)g(Chapter)h(6)g (implies)d(that)j(the)g(use)h(of)e Ff(WAIT)p 1518 1358 14 2 v 15 w(ANY)g Fj(cannot)h(lead)f(to)75 1408 y(starv)n(ation:)21 b(If)15 b(the)h(sending)g(pro)q(cess)h(has)f(issued)g(a)f(send)i (complete)d(op)q(eration,)i(and)f(the)h(receiving)g(pro)q(cess)75 1458 y(rep)q(eatedly)21 b(p)q(ost)f(a)g(receiv)o(e)h(for)e(the)h (message)g(sen)o(t,)h(then)g(that)e(message)h(m)o(ust)f(b)q(e)h(ev)o (en)o(tually)f(receiv)o(ed.)75 1508 y(Section)f(6)f(has)h(not)g(y)o(et) g(b)q(een)g(discussed.)31 b(The)18 b(fairness)h(requiremen)o(t)e(can)h (b)q(e)g(attac)o(k)o(ed)g(either)h(for)e(b)q(eing)75 1558 y(to)q(o)g(w)o(eak)h(\(ev)o(en)o(tually)f(is)g(not)g(go)q(o)q(d)g (enough\),)h(or)g(to)q(o)f(strong)h(\(hard)f(to)g(implemen)o(t\).)26 b(In)18 b(the)g(later)f(case,)75 1608 y(if)g(the)g(requiremen)o(t)h(of) f(fairness)g(is)h(dropp)q(ed)g(from)d(MPI)j(implemen)o(tatio)o(ns,)d (then)j(some)f(mec)o(hanism)e(need)75 1657 y(b)q(e)i(pro)o(vided)f(to)f (the)i(user)g(to)f(ac)o(hiev)o(e)g(fairness)h(b)o(y)f(his)f(or)h(her)h (o)o(wn)f(devices.)25 b(One)17 b(suc)o(h)g(prop)q(osal)f(is)f(to)h(b)q (e)75 1707 y(able)e(to)g(sp)q(ecify)g(a)g(rotating)g(priorit)o(y)f (order)i(on)f(the)g(op)q(erations)h(p)q(osted)g(b)o(y)f(a)f Ff(MPI)p 1431 1707 V 15 w(WAITANY)p Fj(:)g(searc)o(h)i(the)f(list)75 1757 y(sequen)o(tially)m(,)e(starting)i(from)e(a)i(user)h(sp)q (eci\014ed)g(p)q(osition.)918 b Fg(TO)11 b(DISCUSS)166 1807 y Fj(A)k(prop)q(osal)f(w)o(as)h(made)e(for)h(an)h Ff(MPI)p 764 1807 V 15 w(TESTALL)e Fj(function)h(that)h(returns)h (successfully)g(\()p Ff(flag)21 b(=)h(true)p Fj(\))14 b(if)75 1857 y(all)f(op)q(erations)h(in)f(the)i(list)e(ha)o(v)o(e)h (terminated,)f(unsuccessfully)i(\()p Ff(flag)21 b(=)h(false)p Fj(\),)12 b(otherwise.)75 2080 y Fm(2.9)70 b(Blo)r(c)n(king)21 b(Comm)n(unication)75 2181 y Fs(Blo)q(c)o(king)g(send)e(and)h(receiv)o (e)g(op)q(erations)f(com)o(bine)h(all)h(comm)o(unication)f(sub)q(op)q (erations)g(in)o(to)f(one)75 2238 y(call.)h(The)12 b(op)q(eration)h (returns)f(only)g(when)h(the)f(comm)o(unication)h(completes)g(and)f(no) g(comm)o(unication)75 2294 y(ob)s(ject)19 b(p)q(ersists)i(after)e(the)i (call)g(completed.)35 b(Ho)o(w)o(ev)o(er,)20 b(the)g(bu\013er)g (descriptor)h(ob)s(ject)e(needs)i(b)q(e)75 2350 y(created)15 b(ahead)h(of)e(the)i(call.)166 2407 y(W)l(e)e(use)h(the)f(same)g (naming)h(con)o(v)o(en)o(tion)f(as)g(for)g(comm)o(unication)h(handle)g (creation:)20 b(a)14 b(pre\014x)g(of)75 2463 y Fl(R)h Fs(indicates)i(the)e Fl(ready)f Fs(mo)q(de,)i(and)f(a)g(pre\014x)g(of)g Fl(S)g Fs(indicates)i(the)e(sync)o(hronous)g(mo)q(de.)166 2555 y Ft(MPI)p 275 2555 16 2 v 18 w(SEND)j(\(bu\013er)p 610 2555 V 18 w(handle,)g(dest,)f(tag,)h(con)o(text\))166 2647 y Fs(Blo)q(c)o(king)e(standard)f(send.)166 2704 y Fl(MPI)p 241 2704 15 2 v 17 w(SEND)23 b(\(buffer)p 545 2704 V 16 w(handle,)g(dest,)g(tag,)h(context\))14 b Fs(is)p eop %%Page: 36 41 36 40 bop 75 -100 a Fs(36)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fl (MPI_INIT_SEND\(handle,)21 b(buffer_handle,)h(dest,)h(tag,)g(context,) 75 102 y(MPI_EPHEMERAL\))75 158 y(MPI_START\(handle\))75 214 y(MPI_WAIT\(handle,)f(null\))166 343 y Ft(MPI)p 275 343 16 2 v 18 w(RSEND)c(\(bu\013er)p 649 343 V 18 w(handle,)g(dest,)f (tag,)h(con)o(text\))166 435 y Fs(blo)q(c)o(king)e(ready)g(send)166 491 y Fl(MPI)p 241 491 15 2 v 17 w(RSEND)23 b(\(buffer)p 569 491 V 16 w(handle,)g(dest,)g(tag,)g(context\))15 b Fs(is)75 584 y Fl(MPI_INIT_RSEND\(handle,)21 b(buffer_handle,)h (dest,)h(tag,)g(context,)g(MPI_EPHEMERAL\))75 641 y (MPI_START\(handle\))75 697 y(MPI_WAIT\(handle,)f(null\))166 826 y Ft(MPI)p 275 826 16 2 v 18 w(SSEND)c(\(bu\013er)p 639 826 V 18 w(handle,)g(dest,)f(tag,)h(con)o(text\))166 918 y Fs(Blo)q(c)o(king)e(sync)o(hronous)g(send.)166 974 y Fl(MPI)p 241 974 15 2 v 17 w(SSEND)23 b(\(buffer)p 569 974 V 16 w(handle,)g(dest,)g(tag,)g(context\))15 b Fs(is)75 1067 y Fl(MPI_INIT_SSEND\(handle,)21 b(buffer_handle,)h (dest,)h(tag,)g(context,)g(MPI_EPHEMERAL\))75 1123 y (MPI_START\(handle\))75 1180 y(MPI_WAIT\(handle,)f(null\))166 1308 y Ft(MPI)p 275 1308 16 2 v 18 w(RECV\(bu\013er)p 599 1308 V 18 w(handle,)c(source,)e(tag,)i(con)o(text,)f(status\))166 1400 y Fs(Blo)q(c)o(king)f(receiv)o(e)166 1457 y Fl(MPI)p 241 1457 15 2 v 17 w(RECV\(buffer)p 522 1457 V 15 w(handle,)23 b(source,)g(tag,)g(context,)g(status\))14 b Fs(is)75 1550 y Fl(MPI_INIT_RECV\(handle,)21 b(buffer_handle,)h(source,)h(tag,)g (context,)g(MPI_EPHEMERAL\))75 1606 y(MPI_START\(handle\))75 1663 y(MPI_WAIT\(handle,)f(status\))166 1832 y Fk(Implemen)o(tati)o(on) 12 b(note:)166 1882 y Fj(While)f(these)i(functions)f(can)h(b)q(e)f (implemen)o(ted)e(via)h(calls)h(to)f(functions)h(that)g(implemen)o(t)e (sub)q(op)q(erations,)75 1931 y(as)k(describ)q(ed)h(in)e(this)g (subsection,)i(an)e(e\016cien)o(t)h(implemen)o(tatio)o(n)d(ma)o(y)g (optimize)h(a)o(w)o(a)o(y)h(these)i(m)o(ultiple)c(calls,)75 1981 y(pro)o(vided)j(it)f(do)q(es)i(not)f(c)o(hange)g(the)g(b)q(eha)o (vior)g(of)f(correct)j(programs.)75 2207 y Fm(2.10)70 b(Non)n(blo)r(c)n(king)22 b(Comm)n(unication)75 2308 y Fs(Non)o(blo)q(c)o(king)f(send)f(and)f(receiv)o(e)i(op)q(erations)f (com)o(bine)g(the)g(\014rst)f(t)o(w)o(o)f(sub)q(op)q(erations)i(\()p Fl(INIT)f Fs(and)75 2365 y Fl(START)p Fs(\))c(in)o(to)g(one)h(call.)22 b(They)16 b(use)g(ephemeral)h(comm)o(unication)f(ob)s(jects,)f(so)g (that)g(the)h(op)q(eration)f(is)75 2421 y(completed,)g(and)g(the)g (asso)q(ciated)f(resources)h(are)f(freed,)h(b)o(y)f(using)h(one)g(of)f (the)h(functions)g Fl(MPI)p 1742 2421 V 17 w(WAIT,)75 2478 y(MPI)p 150 2478 V 17 w(TEST,)23 b(MPI)p 382 2478 V 17 w(WAITANY,)f(MPI)p 685 2478 V 17 w(TESTANY)p Fs(,)14 b(or)h Fl(MPI)p 1025 2478 V 17 w(WAITALL)p Fs(.)f(Here,)h(to)q(o,)g(a)g (bu\013er)g(ob)s(ject)g(has)h(to)75 2534 y(b)q(e)g(created)f(ahead)g (of)g(the)g(comm)o(unication)h(initiation)h(op)q(eration.)166 2591 y(W)l(e)23 b(use)f(the)h(same)f(naming)h(con)o(v)o(en)o(tion)g(as) f(for)g(blo)q(c)o(king)i(op)q(erations:)35 b(a)22 b(pre\014x)h(of)f Fl(R)h Fs(\()p Fl(S)p Fs(\))75 2647 y(indicates)g(the)f Fl(READY)f Fs(\()p Ft(SYNCHR)o(ONOUS)p Fs(\))g(mo)q(de.)40 b(In)23 b(addition,)h(a)d(pre\014x)i(of)e Ft(I)h Fs(is)g(used)h(to)75 2704 y(indicate)17 b Fh(imme)n(diate)e Fs(\(i.e.,)g(non)o(blo)q(c)o (king\))h(execution.)p eop %%Page: 37 42 37 41 bop 75 -100 a Fq(2.11.)29 b(CONTIGUOUS)16 b(BUFFER)f(COMMUNICA)l (TION)h(OPERA)l(TIONS)359 b Fs(37)166 45 y Ft(MPI)p 275 45 16 2 v 18 w(ISEND)18 b(\(handle,)g(bu\013er)p 813 45 V 18 w(handle,)g(dest,)f(tag,)g(con)o(text\))166 137 y Fs(Non)o(blo)q(c)o(king)f(standard)f(send.)166 193 y Fl(MPI)p 241 193 15 2 v 17 w(ISEND)23 b(\(handle,)g(buffer)p 760 193 V 16 w(handle,)g(dest,)g(tag,)g(context\))14 b Fs(is)75 274 y Fl(MPI_INIT_SEND\(handle,)21 b(buffer_handle,)h(dest,) h(tag,)g(context,)g(MPI_EPHEMERAL\))75 331 y(MPI_START\(handle\))166 447 y Ft(MPI)p 275 447 16 2 v 18 w(IRSEND)18 b(\(handle,)g(bu\013er)p 852 447 V 18 w(handle,)g(dest,)f(tag,)h(con)o(text\))166 539 y Fs(Non)o(blo)q(c)o(king)e(ready)f(send.)166 595 y Fl(MPI)p 241 595 15 2 v 17 w(IRSEND)23 b(\(handle,)f(buffer)p 783 595 V 17 w(handle,)h(dest,)g(tag,)g(context\))14 b Fs(is)75 676 y Fl(MPI_INIT_RSEND\(handle,)21 b(buffer_handle,)h (dest,)h(tag,)g(context,)g(MPI_EPHEMERAL\))75 733 y (MPI_START\(handle\))166 849 y Ft(MPI)p 275 849 16 2 v 18 w(ISSEND)18 b(\(handle,)g(bu\013er)p 842 849 V 18 w(handle,)g(dest,)f(tag,)g(con)o(text\))166 941 y Fs(Non)o(blo)q(c)o (king)f(sync)o(hronous)f(send.)166 997 y Fl(MPI)p 241 997 15 2 v 17 w(ISSEND)23 b(\(handle,)f(buffer)p 783 997 V 17 w(handle,)h(dest,)g(tag,)g(context\))14 b Fs(is)75 1078 y Fl(MPI_INIT_SSEND\(handle,)21 b(buffer_handle,)h(dest,)h(tag,)g (context,)g(MPI_EPHEMERAL\))75 1135 y(MPI_START\(handle\))166 1251 y Ft(MPI)p 275 1251 16 2 v 18 w(IRECV\(handle,)18 b(bu\013er)p 802 1251 V 18 w(handle,)g(source,)e(tag,)i(con)o(text\)) 166 1343 y Fs(Non)o(blo)q(c)o(king)e(receiv)o(e)166 1399 y Fl(MPI)p 241 1399 15 2 v 17 w(IRECV\(handle,)22 b(buffer)p 736 1399 V 16 w(handle,)h(source,)g(tag,)g(context\))14 b Fs(is)75 1480 y Fl(MPI_INIT_RECV\(handle,)21 b(buffer_handle,)h (source,)h(tag,)g(context,)g(MPI_EPHEMERAL\))75 1537 y(MPI_START\(handle\))75 1678 y Fm(2.11)70 b(Con)n(tiguous)23 b(Bu\013er)g(Comm)n(unication)e(Op)r(erations)75 1779 y Fs(The)g(most)f(frequen)o(t)h(t)o(yp)q(e)f(of)h(bu\013er)g(used)g(is) g(a)g(bu\013er)f(with)h(one)g(con)o(tiguous)g(comp)q(onen)o(t.)37 b(W)l(e)75 1836 y(sp)q(ecialize)17 b(the)d(functions)h(in)g(the)f(t)o (w)o(o)f(previous)i(subsections)g(to)f(this)g(case,)g(th)o(us)g(a)o(v)o (oiding)h(the)f(need)75 1892 y(for)h(the)g(creation)h(of)f(a)g (bu\013er)g(descriptor)h(ob)s(ject.)k(W)l(e)c(use)f(the)h(same)f (naming)h(sc)o(heme)g(used)g(in)g(the)75 1948 y(previous)g (subsections,)g(and)f(app)q(end)h(a)f Ft(C)g Fs(in)h(the)g(function)g (name,)e(for)h Fl(CONTIGUOUS)p Fs(.)166 2040 y Ft(MPI)p 275 2040 16 2 v 18 w(SENDC)i(\(start,)h(coun)o(t,)f(datat)o(yp)q(e,)h (dest,)f(tag,)h(con)o(text\))166 2132 y Fs(Blo)q(c)o(king)e(send)g(of)f (con)o(tiguous)g(bu\013er)g(in)h(standard)f(mo)q(de.)166 2189 y Fl(MPI)p 241 2189 15 2 v 17 w(SENDC)23 b(\(start,)g(count,)g (datatype,)f(dest,)i(tag,)f(context\))14 b Fs(is)75 2270 y Fl(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 2326 y(MPI_APPEND_CONTIGUOUS\()f(buffer_handle,)h(start,)h(count,)g (datatype\))75 2382 y(MPI_COMMIT_BUFFER\()e(buffer_handle\))75 2439 y(MPI_SEND)i(\(buffer_handle,)f(count,)h(dest,)g(tag,)g(context\)) 166 2555 y Ft(MPI)p 275 2555 16 2 v 18 w(RSENDC)18 b(\(start,)f(coun)o (t,)g(datat)o(yp)q(e,)h(dest,)f(tag,)h(con)o(text\))166 2647 y Fs(Blo)q(c)o(king)e(send)g(of)f(con)o(tiguous)g(bu\013er)g(in)h (ready)g(mo)q(de.)166 2704 y Fl(MPI)p 241 2704 15 2 v 17 w(RSENDC)23 b(\(start,)g(count,)g(datatype,)f(dest,)h(tag,)h (context\))14 b Fs(is)p eop %%Page: 38 43 38 42 bop 75 -100 a Fs(38)613 b Fq(CHAPTER)16 b(2.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fl (MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 102 y(MPI_APPEND_CONTIGUOUS\()f(buffer_handle,)h(start,)h(count,)g (datatype\))75 158 y(MPI_COMMIT_BUFFER\()e(buffer_handle\))75 214 y(MPI_RSEND\()h(buffer_handle,)g(count,)h(dest,)h(tag,)f(context\)) 166 348 y Ft(MPI)p 275 348 16 2 v 18 w(SSENDC)18 b(\(start,)f(coun)o (t,)g(datat)o(yp)q(e,)h(dest,)f(tag,)h(con)o(text\))166 440 y Fs(Blo)q(c)o(king)e(send)g(of)f(con)o(tiguous)g(bu\013er)g(in)h (sync)o(hron)o(uous)g(mo)q(de.)166 498 y Fl(MPI)p 241 498 15 2 v 17 w(SSENDC)23 b(\(start,)g(count,)g(datatype,)f(dest,)h (tag,)h(context\))14 b Fs(is)75 597 y Fl(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 653 y(MPI_APPEND_CONTIGUOUS\()f (buffer_handle,)h(start,)h(count,)g(datatype\))75 709 y(MPI_COMMIT_BUFFER\()e(buffer_handle\))75 766 y(MPI_SSEND\()h (buffer_handle,)g(count,)h(dest,)h(tag,)f(context\))166 899 y Ft(MPI)p 275 899 16 2 v 18 w(RECV)o(C)16 b(\(start,)i(coun)o(t,)f (datat)o(yp)q(e,)h(source,)f(tag,)h(con)o(text,)f(status\))166 992 y Fs(Blo)q(c)o(king)f(receiv)o(e)h(of)d(con)o(tiguous)i(bu\013er.) 166 1049 y Fl(MPI)p 241 1049 15 2 v 17 w(RECVC)23 b(\(start,)g(count,)g (datatype,)f(source,)h(tag,)h(context,)e(status\))15 b Fs(is)75 1148 y Fl(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h (MPI_EPHEMERAL\))75 1204 y(MPI_APPEND_CONTIGUOUS\()f(buffer_handle,)h (start,)h(count,)g(datatype\))75 1261 y(MPI_COMMIT_BUFFER\()e (buffer_handle\))75 1317 y(MPI_RECV\()i(buffer_handle,)f(source,)g (tag,)i(context,)f(return_handle\))166 1450 y Ft(MPI)p 275 1450 16 2 v 18 w(ISENDC)17 b(\(handle,)i(start,)e(coun)o(t,)g (datat)o(yp)q(e,)h(dest,)f(tag,)h(con)o(text\))166 1543 y Fs(Non)o(blo)q(c)o(king)e(send)g(of)f(con)o(tiguous)g(bu\013er)g(in)h (standard)f(mo)q(de.)166 1601 y Fl(MPI)p 241 1601 15 2 v 17 w(ISENDC)23 b(\(handle,)f(start,)i(count,)f(datatype,)f(dest,)h (tag,)h(context\))14 b Fs(is)75 1699 y Fl(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 1756 y(MPI_APPEND_CONTIGUOUS\()f (buffer_handle,)h(start,)h(count,)g(datatype\))75 1812 y(MPI_COMMIT_BUFFER\()e(buffer_handle\))75 1869 y(MPI_ISEND\()h (handle,)h(buffer_handle,)f(dest,)i(tag,)f(context\))166 2002 y Ft(MPI)p 275 2002 16 2 v 18 w(IRSENDC)17 b(\(handle,)i(start,)e (coun)o(t,)g(datat)o(yp)q(e,)h(dest,)f(tag,)h(con)o(text\))166 2095 y Fs(Non)o(blo)q(c)o(king)e(send)g(of)f(con)o(tiguous)g(bu\013er)g (in)h(ready)f(mo)q(de.)166 2152 y Fl(MPI)p 241 2152 15 2 v 17 w(IRSENDC)23 b(\(handle,)f(start,)h(count,)g(datatype,)g(dest,)g (tag,)h(context\))14 b Fs(is)75 2251 y Fl(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h(MPI_EPHEMERAL\))75 2307 y(MPI_APPEND_CONTIGUOUS\()f (buffer_handle,)h(start,)h(count,)g(datatype\))75 2364 y(MPI_COMMIT_BUFFER\()e(buffer_handle\))75 2420 y(MPI_IRSEND\()h (handle,)h(buffer_handle,)f(dest,)h(tag,)h(context\))166 2553 y Ft(MPI)p 275 2553 16 2 v 18 w(ISSENDC)17 b(\(handle,)i(start,)e (coun)o(t,)g(datat)o(yp)q(e,)h(dest,)f(tag,)h(con)o(text\))166 2646 y Fs(Non)o(blo)q(c)o(king)e(send)g(of)f(con)o(tiguous)g(bu\013er)g (in)h(sync)o(hronous)g(mo)q(de.)166 2704 y Fl(MPI)p 241 2704 15 2 v 17 w(ISSENDC)23 b(\(handle,)f(start,)h(count,)g(datatype,)g (dest,)g(tag,)h(context\))14 b Fs(is)p eop %%Page: 39 44 39 43 bop 75 -100 a Fq(2.12.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1138 b Fs(39)75 45 y Fl(MPI_CREATE_BUFFER\()21 b(buffer_handle,)h (MPI_EPHEMERAL\))75 102 y(MPI_APPEND_CONTIGUOUS\()f(buffer_handle,)h (start,)h(count,)g(datatype\))75 158 y(MPI_COMMIT_BUFFER\()e (buffer_handle\))75 214 y(MPI_ISSEND\()h(handle,)h(buffer_handle,)f (dest,)h(tag,)h(context\))166 347 y Ft(MPI)p 275 347 16 2 v 18 w(IRECV)o(C\(handle,)17 b(start,)g(coun)o(t,)h(datat)o(yp)q (e,)g(source,)e(tag,)i(con)o(text\))166 440 y Fs(Non)o(blo)q(c)o(king)e (receiv)o(e)g(of)f(con)o(tiguous)g(bu\013er.)166 498 y Fl(MPI)p 241 498 15 2 v 17 w(IRECVC\(handle,)22 b(start,)h(count,)g (datatype,)f(source,)h(tag,)h(context\))14 b Fs(is)75 596 y Fl(MPI_CREATE_BUFFER)22 b(\(buffer_handle,)f(MPI_EPHEMERAL\))75 653 y(MPI_APPEND_CONTIGUOUS\()g(buffer_handle,)h(start,)h(count,)g (datatype\))75 709 y(MPI_COMMIT_BUFFER\()e(buffer_handle\))75 766 y(MPI_IRECV\()h(handle,)h(buffer_handle,)f(source,)h(tag,)h (context\))75 914 y Fm(2.12)70 b(Prob)r(e)23 b(and)h(Cancel)75 1018 y Fs(The)d Fl(MPI)p 249 1018 V 17 w(PROBE)f Fs(op)q(eration)h (allo)o(ws)g(incoming)h(messages)f(to)f(b)q(e)i(c)o(hec)o(k)o(ed)f (for,)g(without)g(actually)75 1074 y(receiving)16 b(them.)k(The)15 b(user)g(can)g(then)h(decide)g(where)f(to)g(receiv)o(e)g(them,)g(based) g(on)g(the)g(information)75 1130 y(returned)h(b)o(y)f(the)g(prob)q(e)g (\(basically)l(,)i(the)e(information)g(on)g(the)h(message)e(en)o(v)o (elop)q(e\).)21 b(An)16 b(additional)75 1187 y(function,)k Fl(MPI)p 345 1187 V 17 w(GET)p 434 1187 V 17 w(LEN)e Fs(allo)o(ws)h(the)f(amoun)o(t)g(of)g(storage)g(needed)i(to)e(receiv)o (e)h(the)g(message)f(to)g(b)q(e)75 1243 y(computed,)23 b(when)g(this)f(length)g(is)h(not)e(readily)i(computed)f(from)f(the)h (information)g(returned)g(b)o(y)75 1300 y Fl(MPI)p 150 1300 V 17 w(PROBE)p Fs(.)166 1357 y(The)e Fl(MPI)p 339 1357 V 17 w(CANCEL)f Fs(op)q(eration)h(allo)o(ws)h(p)q(ending)g(comm)o (unications)g(to)e(b)q(e)i(cancelled.)37 b(This)20 b(is)75 1414 y(required)13 b(for)f(clean)o(up.)20 b(P)o(osting)12 b(a)f(send)i(or)f(a)g(receiv)o(e)h(ties)f(user)g(resources)h(\(send)f (or)g(receiv)o(e)h(bu\013ers\),)75 1470 y(and)i(a)g(cancel)i(ma)o(y)d (b)q(e)i(needed)g(to)f(free)g(these)h(resources)f(gracefully)l(.)166 1563 y Ft(MPI)p 275 1563 16 2 v 18 w(IPR)o(OBE\()j(source,)e(tag,)i (con)o(text,)f(\015ag,)h(t)o(yp)q(e,)f(status\))75 1709 y(IN)g(source)22 b Fs(rank)15 b(in)h(con)o(text)f(of)g(source,)g(or)f Fl(MPI)p 974 1709 15 2 v 17 w(ANY)p 1063 1709 V 17 w(SOURCE)g Fs(\(in)o(teger\).)75 1807 y Ft(IN)j(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(receiv)o(ed)h(with)g(this)f(handle,)h(or)f Fl(MPI)p 1341 1807 V 17 w(ANY)p 1430 1807 V 16 w(TAG)g Fs(\(in)o(teger\).)75 1905 y Ft(IN)i(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(receiv)o(ed)h(with)g(this)g(handle.) 75 2002 y Ft(OUT)i(\015ag)24 b Fs(\(logical\))75 2100 y Ft(IN)17 b(t)o(yp)q(e)23 b Fs(assumed)15 b(t)o(yp)q(e)g(of)g(data)g (in)h(message)e(\(status)g(v)m(ariable\).)75 2197 y Ft(OUT)k(status)23 b Fs(handle)17 b(that)d(is)i(asso)q(ciated)f(with)h(return)f(status)f (ob)s(ject.)166 2308 y Fl(MPI)p 241 2308 V 17 w(IPROBE)i Fs(returns)g Fl(flag)24 b(=)f(true)17 b Fs(if)g(there)g(is)g(a)g (message)f(that)g(can)h(b)q(e)h(receiv)o(ed)g(and)f(that)75 2365 y(matc)o(hes)j(the)g(pattern)g(sp)q(eci\014ed)j(b)o(y)d(the)g (parameters)g Fl(source)p Fs(,)g Fl(tag)p Fs(,)h(and)f Fl(context)p Fs(.)35 b(It)20 b(returns)75 2421 y Fl(flag)j(=)h(false)p Fs(,)15 b(otherwise.)21 b(If)15 b Fl(MPI)p 731 2421 V 17 w(PROBE)g Fs(returns)g Fl(flag)24 b(=)f(true)p Fs(,)15 b(then)h(the)f(length,)h(source)g(and)75 2478 y(tag)g(of)g(the)h (message)g(matc)o(hed)g(are)f(returned)h(in)h(the)f(return)g(status)f (ob)s(ject.)24 b(These)18 b(are)e(the)h(same)75 2534 y(v)m(alues)i(that)d(w)o(ould)i(ha)o(v)o(e)f(b)q(een)i(returned)f(b)o (y)f(a)h(call)g(to)f Fl(MPI)p 1176 2534 V 17 w(RECV)g Fs(or)g(to)g Fl(MPI)p 1494 2534 V 16 w(SRECV)g Fs(executed)i(at)75 2591 y(the)d(same)f(p)q(oin)o(t)h(in)h(the)f(program)e(\(with)i(a)f(ca) o(v)o(eat)g(concerning)i(length;)f(see)g(b)q(elo)o(w\).)22 b(These)16 b(v)m(alues)75 2647 y(can)j(b)q(e)h(deco)q(ded)g(from)e(the) h(return)g(status)f(ob)s(ject)g(using)i(the)f Fl(MPI)p 1302 2647 V 16 w(RETURN)p 1462 2647 V 17 w(STAT)f Fs(function.)32 b(The)75 2704 y(return)15 b(status)f(ob)s(ject)h(is)h(unde\014ned)h(if) f Fl(flag=false)p Fs(.)p eop %%Page: 40 45 40 44 bop 75 -100 a Fs(40)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fs(The)21 b(length)g(v)m(alue)h(returned)g(b)o(y)e(the)h(return)g(status)f(ob)s (ject)g(is)h(\(correctly\))f(the)h(n)o(um)o(b)q(er)g(of)75 102 y(elemen)o(ts)14 b(in)h(the)f(message,)f(pro)o(vided)i(that)e(all)h (elemen)o(ts)h(in)f(the)g(message)f(are)h(of)f(the)h(t)o(yp)q(e)g(sp)q (eci\014ed)75 158 y(b)o(y)h(the)g Fl(type)g Fs(parameter;)f(otherwise)h (the)h(length)g(v)m(alue)g(returned)g(is)f(unde\014ned.)166 214 y(A)e(subsequen)o(t)i(receiv)o(e)f(executed)g(with)g(the)g(same)f (con)o(text,)g(and)g(the)h(source)f(and)h(tag)f(returned)75 271 y(b)o(y)18 b(the)g(call)h(to)e Fl(MPI)p 441 271 15 2 v 17 w(IPROBE)g Fs(will)j(receiv)o(e)e(the)g(message)g(that)f(w)o(as) g(matc)o(hed)h(b)o(y)g(the)g(prob)q(e,)h(if)f(no)75 327 y(other)12 b(in)o(terv)o(ening)h(receiv)o(e)h(o)q(ccurred)f(after)f (the)g(prob)q(e.)19 b(If)13 b(the)g(receiving)h(pro)q(cess)e(is)h(m)o (ultithreaded,)75 384 y(it)i(is)h(the)f(user)h(resp)q(onsibilit)o(y)h (to)e(ensure)h(that)e(the)i(last)f(condition)h(holds.)166 476 y Ft(MPI)p 275 476 16 2 v 18 w(PR)o(OBE\()i(source,)e(tag,)i(con)o (text,)f(t)o(yp)q(e,)g(status\))75 598 y(IN)g(source)22 b Fs(rank)15 b(in)h(con)o(text)f(of)g(source,)g(or)f Fl(MPI)p 974 598 15 2 v 17 w(ANY)p 1063 598 V 17 w(SOURCE)g Fs(\(in)o(teger\).)75 687 y Ft(IN)j(tag)24 b Fs(user)15 b(tag)f(for)h(messages)g(receiv)o(ed)h(with)g(this)f(handle,)h(or)f Fl(MPI)p 1341 687 V 17 w(ANY)p 1430 687 V 16 w(TAG)g Fs(\(in)o(teger\).)75 775 y Ft(IN)i(con)o(text)23 b Fs(con)o(text)15 b(of)f(messages)h(receiv)o(ed)h(with)g(this)g(handle.)75 864 y Ft(IN)h(t)o(yp)q(e)23 b Fs(assumed)15 b(t)o(yp)q(e)g(of)g(data)g (in)h(message)e(\(status)g(v)m(ariable\).)75 952 y Ft(OUT)k(status)23 b Fs(handle)17 b(that)d(is)i(asso)q(ciated)f(with)h(return)f(status)f (ob)s(ject.)166 1039 y Fl(MPI)p 241 1039 V 17 w(PROBE)f Fs(b)q(eha)o(v)o(es)h(lik)o(e)g Fl(MPI)p 712 1039 V 17 w(IPROBE)f Fs(except)h(that)f(it)h(is)g(a)f(blo)q(c)o(king)j(call)e (whic)o(h)h(returns)e(only)75 1096 y(after)h(a)h(matc)o(hing)h(message) e(has)h(b)q(een)i(found.)166 1228 y Fk(Discussion:)166 1278 y Fj(MPI)e(guaran)o(tees)g(that)g(successiv)o(e)i(messages)e(sen)o (t)h(from)d(a)h(source)i(to)f(a)f(destination)h(within)e(the)j(same)75 1328 y(con)o(text)21 b(are)g(receiv)o(ed)h(in)e(the)h(order)g(they)g (are)g(sen)o(t.)39 b(Th)o(us,)22 b(MPI)e(m)o(ust)g(supp)q(ort,)i (either)g(explicitly)d(or)75 1378 y(implicitly)l(,)13 b(a)i(FIF)o(O)g(structure)j(to)d(manage)e(messages)j(b)q(et)o(w)o(een)h (eac)o(h)e(pair)g(of)g(pro)q(cesses,)j(for)d(eac)o(h)h(con)o(text.)75 1428 y Ff(MPI)p 144 1428 14 2 v 15 w(PROBE)h Fj(returns)i(information)c (on)j(the)h(\014rst)f(matc)o(hing)e(message)i(in)f(this)h(FIF)o(O;)g (this)g(will)f(also)g(b)q(e)h(the)75 1477 y(message)i(receiv)o(ed)i(b)o (y)e(the)h(\014rst)h(subsequen)o(t)g(receiv)o(e)g(with)e(the)h(same)e (source,)k(tag)d(and)h(con)o(text)g(as)f(the)75 1527 y(message)14 b(matc)o(hed)f(b)o(y)g Ff(MPI)p 528 1527 V 16 w(PROBE)p Fj(.)166 1577 y(Message)g(passing)f(in)g(MPI)g(can)h(b)q (e)g(implem)o(en)o(ted)d(without)i(app)q(ending)g(t)o(yp)q(e)h (information)c(to)j(messages.)75 1627 y(A)h(message)h(is)f(merely)f(a)h (string)h(of)f(b)o(ytes)h(and)f(the)h(in)o(terpretation)g(of)e(these)j (b)o(ytes)f(in)o(to)f(a)g(sequence)j(of)c(t)o(yp)q(ed)75 1677 y(elemen)o(ts)18 b(is)g(done)g(using)g(the)g(information)d(in)j (the)g(bu\013er)h(descriptors)h(at)e(eac)o(h)g(end.)31 b(The)18 b(abilit)o(y)e(to)i(use)75 1726 y(suc)o(h)g(implem)o(en)o (tation)c(strategy)j(is)g(deemed)g(to)g(b)q(e)g(an)g(imp)q(ortan)o(t)e (goal.)25 b(In)17 b(suc)o(h)h(implemen)o(tatio)o(n,)c(when)75 1776 y(a)k(message)f(arriv)o(es,)i(it)f(is)g(not)g(kno)o(wn)g(ho)o(w)f (man)o(y)f(elemen)o(ts)i(it)g(con)o(tains,)g(or)g(ev)o(en)h(ho)o(w)f(m) o(uc)o(h)f(storage)h(is)75 1826 y(needed)h(to)e(receiv)o(e)i(that)e (message)g(\(b)q(ecause)i(of)e(p)q(ossible)h(represen)o(tation)g(con)o (v)o(ersion)g(in)f(a)g(heterogeneous)75 1876 y(en)o(vironmen)o(t\).)f (The)c(prob)q(e)f(function)g(cannot)g(use)h(a)f(bu\013er)h(descriptor;) h(this)e(defeats)h(the)g(purp)q(ose)g(of)f(probing)75 1926 y(in)k(order)h(to)g(decide)g(where)h(to)e(receiv)o(e)i(a)e (message.)23 b(Therefore,)17 b(prob)q(e)f(cannot,)g(in)f(general,)h (return)g(correct)75 1976 y(length)10 b(information.)k(Still,)9 b(it)h(is)f(often)i(the)f(case)h(that)f(prob)q(e)h(is)f(used)h(to)f (decide)h(ho)o(w)e(m)o(uc)o(h)g(storage)i(to)e(allo)q(cate)75 2025 y(in)15 b(order)i(to)e(receiv)o(e)i(a)f(message.)23 b(Enco)q(ding)16 b(suc)o(h)g(information)d(in)i(the)h(message)g(tag)f (is)h(deemed)f(to)h(b)q(e)g(to)q(o)75 2075 y(a)o(wkw)o(ard,)g(and)h(it) f(is)g(deemed)h(imp)q(ortan)o(t)e(for)h Ff(MPI)p 923 2075 V 15 w(PROBE)g Fj(to)g(return)i(some)e(useful)h(size)g (information.)24 b(The)75 2125 y(curren)o(t)13 b(de\014nition)e(of)f Ff(MPI)p 512 2125 V 15 w(PROBE)g Fj(is)i(a)e(compromise)f(b)q(et)o(w)o (een)k(these)g(t)o(w)o(o)e(goals.)16 b(F)m(or)11 b(the)h(most)e(common) e(case)75 2175 y(of)k(messages)h(where)h(all)d(en)o(tries)j(ha)o(v)o(e) e(the)h(same)f(t)o(yp)q(e,)h Ff(MPI)p 1043 2175 V 15 w(PROBE)f Fj(returns)i(the)f(correct)i(length)d(information;)75 2225 y(the)i(more)f(esoteric)i(case)f(is)g(handled)f(b)o(y)g(the)i Ff(MPI)p 883 2225 V 15 w(GET)p 964 2225 V 15 w(LEN)e Fj(that)g(is)h(describ)q(ed)h(b)q(elo)o(w.)j(The)c(curren)o(t)h (solution)75 2274 y(sa)o(v)o(es)h(us)f(the)h(need)g(for)f(one)g (additional)f(w)o(ord)h(p)q(er)h(message)f(that)g(w)o(ould)f(otherwise) j(b)q(e)e(needed)i(to)e(transfer)75 2324 y(the)f(message)g(length)g (\(in)f(elemen)o(ts\))h(with)g(the)g(message.)166 2499 y Ft(MPI)p 275 2499 16 2 v 18 w(GET)p 404 2499 V 20 w(LEN\()k(coun)o (t,)f(status,)h(bu\013er)p 1030 2499 V 17 w(descriptor\))166 2591 y Fs(Computes)j(the)f(n)o(um)o(b)q(er)i(of)e(elemen)o(ts)i(that)e (w)o(ere)h(to)f(b)q(e)h(receiv)o(ed,)i(if)f(the)f(message)f(that)g(is) 75 2647 y(asso)q(ciated)13 b(with)g(the)g(return)g(status)f(handle)j(w) o(ould)e(b)q(e)h(receiv)o(ed)g(in)f(the)h(bu\013er)e(asso)q(ciated)i (with)f(the)75 2704 y(bu\013er)i(descriptor)h(handle.)p eop %%Page: 41 46 41 45 bop 75 -100 a Fq(2.12.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1138 b Fs(41)75 45 y Ft(OUT)18 b(coun)o(t)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(elemen)o(ts)g(that)g(w)o(ere)g(to)f(b)q(e)i(receiv)o(ed)h(\(in)o (teger\))75 141 y Ft(IN)g(status)23 b Fs(handle)17 b(to)d(return)h (status)g(descriptor)75 237 y Ft(IN)i(bu\013er)p 291 237 16 2 v 18 w(descriptor)23 b Fs(handle)16 b(to)f(bu\013er)g (descriptor)166 409 y Fk(Discussion:)166 460 y Fj(In)e(order)i(to)e (supp)q(ort)h(this)g(function,)f(an)g(additional)f(\014eld)h(is)g (needed)i(in)e(the)i(return)f(status)h(ob)r(ject,)e(i.e.,)75 510 y(n)o(um)o(b)q(er)g(of)g(b)o(ytes)i(in)e(the)i(incoming)d(message)h (matc)o(hed)g(b)o(y)h(prob)q(e.)166 685 y Ft(MPI)p 275 685 V 18 w(CANCEL\()j(handle,)h(\015ag\))75 829 y(IN)f(handle)24 b Fs(handle)16 b(to)f(comm)o(unication)h(ob)s(ject)75 925 y Ft(OUT)i(\015ag)24 b Fs(\(logical\))166 1035 y(A)19 b(call)i(to)e Fl(MPI)p 442 1035 15 2 v 17 w(CANCEL)f Fs(cancels)j(a)e(p)q(ending)i(comm)o(unication)f(op)q(eration)g(\(send) g(or)f(receiv)o(e\).)75 1091 y(The)d(call)h(returns)e Fl(flag)24 b(=)f(true)15 b Fs(if)i(the)e(cancel)i(op)q(eration)f (succeeded,)h Fl(flag)23 b(=)h(false)15 b Fs(otherwise.)75 1148 y(It)h(m)o(ust)f(b)q(e)i(the)f(case)g(that)f(either)i(the)f (cancel)h(op)q(eration)f(succeeds)h(or)e(that)g(the)h(p)q(ending)i (comm)o(u-)75 1204 y(nication)e(op)q(eration)g(completes)g(\(but)f(not) f(b)q(oth\).)166 1261 y(If)20 b(a)g(send)g(w)o(as)f(cancelled)j (successfully)g(then)e(the)g(message)g(sen)o(t)f(will)j(not)d(b)q(e)i (receiv)o(ed,)h(the)75 1317 y(receiv)o(e)g(bu\013er)f(of)g(an)o(y)g(p)q (osted)g(receiv)o(e)h(for)f(that)f(message)h(will)i(not)e(b)q(e)h (altered,)g(and)g(an)o(y)f(suc)o(h)75 1374 y(receiv)o(e)g(has)e(to)g(b) q(e)h(satis\014ed)g(b)o(y)g(another)f(send.)34 b(If)20 b(a)f(receiv)o(e)h(w)o(as)f(cancelled)j(successfully)l(,)h(then)75 1430 y(the)16 b(receiv)o(e)g(bu\013er)g(p)q(osted)g(will)h(not)e(b)q(e) i(altered,)e(and)h(an)o(y)f(send)i(matc)o(hing)e(this)h(receiv)o(e)h (has)e(to)g(b)q(e)75 1487 y(satis\014ed)h(b)o(y)f(another)g(receiv)o (e.)166 1544 y(A)g(successful)h(cancel)g(op)q(eration)e(do)q(es)h(not)g (complete)g(the)g(cancelled)i(comm)o(unication;)e(the)g(op-)75 1600 y(eration)d(still)h(needs)g(to)e(b)q(e)i(completed)g(with)f(an)g Fl(MPI)p 992 1600 V 17 w(WAIT)f Fs(or)g Fl(MPI)p 1240 1600 V 17 w(TEST)g Fs(call,)j(if)e(it)g(is)h(an)f(non)o(blo)q(c)o(king) 75 1657 y(comm)o(unication.)1539 b Fg(TO)11 b(DISCUSS)166 1790 y Fk(Discussion:)166 1840 y Fj(W)m(e)j(ma)o(y)f(w)o(an)o(t)i(a)f (non)o(blo)q(c)o(king)g(cancel,)h(that)g(returns)h(asap,)f(since,)g(an) o(yho)o(w,)f(one)h(has)g(to)g(w)o(ait)f(for)g(the)75 1890 y Ff(MPI)p 144 1890 14 2 v 15 w(TEST)c Fj(or)h Ff(MPI)p 371 1890 V 15 w(WAIT)f Fj(to)h(return)h(b)q(efore)g(the)f(bu\013er)h (is)f(reused.)19 b(If)10 b(w)o(e)i(do)e(so,)h(then)h(the)g Ff(MPI)p 1584 1890 V 15 w(CANCEL)d Fj(call)h(will)75 1940 y(not)j(return)h(information)d(on)i(the)g(success)j(or)d(failure)g (of)f(the)i(cancel;)f(this)h(will)d(b)q(e)j(returned)h(b)o(y)e(the)h Ff(MPI)p 1776 1940 V 15 w(WAIT)75 1990 y Fj(or)g Ff(MPI)p 195 1990 V 15 w(TEST)f Fj(op)q(eration.)166 2040 y(The)e Ff(MPI)p 317 2040 V 15 w(TEST)g Fj(executed)i(after)e(an)g(op)q (eration)g(has)g(b)q(een)h(cancelled)g(needs)g(to)f(return)h(an)f (indication)f(that)75 2090 y(the)k(op)q(eration)g(failed.)j(On)d(the)g (other)g(hand,)g(w)o(e)g(do)f(not)h(w)o(an)o(t)f(the)h(program)e(to)i (blo)o(w)f(up)g(in)h(suc)o(h)g(situation.)75 2140 y(Th)o(us,)h(this)f (indication)g(has)g(to)h(b)q(e)g(handled)g(as)f(a)h(nonfatal)e(error,)i (ev)o(en)g(in)g(the)g(execution)g(mo)q(de)f(where)h(an)o(y)75 2189 y(unsuccessful)h(comm)o(uni)o(cation)11 b(is)i(fatal.)k(Do)c(w)o (e)h(w)o(an)o(t)f(this)h(information)d(to)i(b)q(e)h(returned)i(in)d (the)h(error)h(co)q(de,)75 2239 y(or)f(to)g(b)q(e)g(returned)i(as)e (part)g(of)f(the)h(status?)166 2290 y(Can)g(a)f(thread)i(cancel)f(a)g (blo)q(c)o(king)f(comm)o(unicati)o(on)e(executed)16 b(b)o(y)e(another)g (thread?)166 2347 y(Do)f(w)o(e)h(w)o(an)o(t)g Ff(MPI)p 464 2347 V 15 w(CANCEL)e Fj(to)i(free)h(the)f(comm)o(unication)d(ob)r (ject?)166 2562 y Fk(Implemen)o(tati)o(on)h(note:)166 2613 y Fj(The)i(cancel)h(op)q(eration)f(ma)o(y)d(b)q(e)k(exp)q(ensiv)o (e)g(to)f(implem)o(en)o(t)e(on)h(some)g(systems.)p eop %%Page: 42 47 42 46 bop 75 -100 a Fs(42)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fm(2.13)70 b(send-receiv)n(e)-370 92 y Fg(TO)11 b(DISCUSS)75 226 y Fk(Discussion:)166 283 y Fj(This)j(section)g(has)g(not)g(y)o(et)g(b)q (een)h(discussed)h(b)o(y)e(MPIF.)166 424 y Fs(The)20 b Ft(send-receiv)o(e)e Fs(op)q(erations)i(com)o(bines)g(in)g(one)g (call)h(the)e(sending)i(of)e(a)g(message)g(to)g(one)75 481 y(destination)h(and)f(the)g(receiving)h(of)e(another)h(message,)g (form)f(another)g(destination,)i(p)q(ossibly)h(the)75 537 y(same.)f(The)c Ft(exc)o(hange)f Fs(op)q(erations)h(are)f(the)h (same)f(as)g(send-receiv)o(e,)i(except)f(that)e(the)i(send)g(bu\013er) 75 593 y(and)f(the)h(receiv)o(e)g(bu\013er)f(are)g(iden)o(tical.)22 b(A)15 b(send-receiv)o(e)i(op)q(eration)e(is)h(v)o(ery)f(useful)h(for)f (executing)h(a)75 650 y(shift)f(op)q(eration)g(across)g(a)f(c)o(hain)i (of)e(pro)q(cesses.)20 b(If)c(blo)q(c)o(king)g(sends)f(and)g(receiv)o (es)h(are)f(used)g(for)g(suc)o(h)75 706 y(shift,)d(then)g(one)f(needs)h (to)f(order)g(correctly)g(the)g(sends)h(and)g(receiv)o(es)g(\(e.g.)17 b(ev)o(en)12 b(sends,)g(next)f(receiv)o(es,)75 763 y(o)q(dd)18 b(receiv)o(es)g(\014rst,)f(next)g(sends\))h(so)f(as)g(to)f(prev)o(en)o (t)i(cyclic)h(dep)q(endencies)h(that)d(lead)h(to)e(deadlo)q(c)o(k.)75 819 y(When)k(a)f(send-receiv)o(e)h(ro)f(exc)o(hange)h(op)q(eration)f (is)h(used,)g(the)g(comm)o(unication)g(subsystem)f(tak)o(es)75 876 y(care)c(of)g(these)g(issues.)166 934 y(A)10 b(handle)i(for)d(a)h (send-receiv)o(e)i(\(exc)o(hange\))d(op)q(eration)i(is)f(created)h(b)o (y)f(a)g(call)h(to)e Fl(MPI)p 1609 934 15 2 v 17 w(INIT)p 1722 934 V 17 w(SENDRECV)75 990 y Fs(\()p Fl(MPI)p 168 990 V 16 w(INIT)p 280 990 V 17 w(EXCHANGE)p Fs(\).)16 b(The)j(op)q(erations)f(can)g(b)q(e)h(started)e(and)i(completed)g(lik)o (e)g(regular)f(comm)o(u-)75 1047 y(nication)e(op)q(erations,)f(using)h (the)f Fl(MPI)p 755 1047 V 16 w(START)p Fs(,)f Fl(MPI)p 990 1047 V 17 w(WAIT)p Fs(,)g Fl(MPI)p 1202 1047 V 17 w(WAITALL)p Fs(,)g(etc.)19 b(In)d(addition,)g(op)q(era-)75 1103 y(tions)d(are)g(pro)o(vided)h(for)e(blo)q(c)o(king)j(and)e(non)o (blo)q(c)o(king)h(send-receiv)o(e)h(and)e(exc)o(hange)g(op)q(erations,) h(with)75 1160 y(general)i(and)f(con)o(tiguous)g(bu\013ers.)166 1217 y(A)i(message)f(sen)o(t)h(b)o(y)g(a)f(send-receiv)o(e)j(or)d(exc)o (hange)h(op)q(eration)g(can)g(b)q(e)h(receiv)o(ed)g(b)o(y)e(a)h (regular)75 1274 y(receiv)o(e)f(op)q(eration,)f(and)h(vice)g(v)o(ersa.) 166 1408 y Fk(Discussion:)166 1459 y Fj(The)f(correctness)j(c)o(hapter) f(need)f(a)e(section)i(on)f(send-receiv)o(es)j({)c(the)i(correctness)i (issue)e(is)f(not)g(en)o(tirely)75 1509 y(trivial,)d(b)q(ecause)j(of)e (p)q(ossible)h(cyclic)f(dep)q(endencies.)21 b(Ev)o(en)14 b(in)f(an)g(implemen)o(tatio)o(n)e(where)k(messages)e(are)h(not)75 1559 y(bu\013ered,)j(MPi)f(has)g(to)f(guaran)o(tee)h(that)g(blo)q(c)o (king)f(calls)g(to)h(send-receiv)o(e)h(that)f(collectiv)o(ely)f (execute)j(a)d(shift)75 1609 y(on)f(a)f(cycle)i(will)d(not)i(deadlo)q (c)o(k.)k(Some)12 b(system)i(bu\013ering)g(is,)f(in)h(fact,)f (necessary)j(when)e(exc)o(hanges)h(are)g(used.)166 1785 y Ft(MPI)p 275 1785 16 2 v 18 w(INIT)p 410 1785 V 19 w(SENDRECV\(handle,)e(send)p 1025 1785 V 19 w(bu\013er)p 1179 1785 V 18 w(handle,)g(dest,)f(recv)p 1593 1785 V 19 w(bu\013er)p 1747 1785 V 17 w(handle,)75 1841 y(source,)k(tag,)i (con)o(text,)f(p)q(ersistence\))166 1935 y Fs(Creates)d(a)h(comm)o (unication)h(ob)s(ject)f(for)f(a)h(send-receiv)o(e)i(op)q(eration.)75 2048 y Ft(OUT)h(handle)24 b Fs(message)11 b(handle.)20 b(The)12 b(handle)h(should)f(not)f(b)q(e)i(asso)q(ciated)f(with)f(an)o (y)h(ob)s(ject)f(b)q(efore)189 2105 y(the)k(call.)75 2205 y Ft(IN)i(send)p 259 2205 V 19 w(bu\013er)p 413 2205 V 18 w(handle)23 b Fs(handle)17 b(to)d(send)i(bu\013er)f (descriptor)75 2304 y Ft(IN)i(dest)23 b Fs(rank)15 b(in)h(con)o(text)e (of)h(destination)h(\(in)o(teger\))75 2404 y Ft(IN)h(recv)p 253 2404 V 18 w(bu\013er)p 406 2404 V 18 w(handle)24 b Fs(handle)16 b(to)f(receiv)o(e)h(bu\013er)f(descriptor)75 2504 y Ft(IN)i(source)22 b Fs(rank)15 b(of)g(source)g(in)h(con)o(text)f (\(in)o(teger\))75 2604 y Ft(IN)i(tag)24 b Fs(tag)14 b(used)i(b)q(oth)f(for)g(send)h(and)f(for)f(receiv)o(e)j(op)q(eration)e (\(in)o(teger\))75 2704 y Ft(IN)i(con)o(text)23 b Fs(con)o(text)15 b(b)q(oth)g(for)g(send)g(and)h(receiv)o(e)g(op)q(eration)f(\(handle\))p eop %%Page: 43 48 43 47 bop 75 -100 a Fq(2.13.)29 b(SEND-RECEIVE)1282 b Fs(43)75 45 y Ft(IN)17 b(p)q(ersistence)23 b Fs(comm)o(unication)16 b(ob)s(ject)f(p)q(ersistence)166 190 y Ft(MPI)p 275 190 16 2 v 18 w(INIT)p 410 190 V 19 w(EX)o(CHANGE\(handle,)20 b(bu\013er)p 1077 190 V 17 w(handle,)h(dest,)f(source,)f(tag,)h(con)o (text,)75 247 y(p)q(ersistence\))166 339 y Fs(Creates)14 b(a)h(comm)o(unication)h(ob)s(ject)f(for)f(an)i(exc)o(hange)f(op)q (eration.)75 449 y Ft(OUT)j(handle)24 b Fs(message)11 b(handle.)20 b(The)12 b(handle)h(should)f(not)f(b)q(e)i(asso)q(ciated)f (with)f(an)o(y)h(ob)s(ject)f(b)q(efore)189 505 y(the)k(call.)75 602 y Ft(IN)i(bu\013er)p 291 602 V 18 w(handle)24 b Fs(handle)16 b(to)f(send)h(and)f(receiv)o(e)h(bu\013er)f(descriptor)75 699 y Ft(IN)i(dest)23 b Fs(rank)15 b(in)h(con)o(text)e(of)h (destination)h(\(in)o(teger\))75 795 y Ft(IN)h(source)22 b Fs(rank)15 b(of)g(source)g(in)h(con)o(text)f(\(in)o(teger\))75 892 y Ft(IN)i(tag)24 b Fs(tag)14 b(used)i(b)q(oth)f(for)g(send)h(and)f (for)f(receiv)o(e)j(op)q(eration)e(\(in)o(teger\))75 988 y Ft(IN)i(con)o(text)23 b Fs(con)o(text)15 b(b)q(oth)g(for)g(send)g (and)h(receiv)o(e)g(op)q(eration)f(\(handle\))75 1085 y Ft(IN)i(p)q(ersistence)23 b Fs(comm)o(unication)16 b(ob)s(ject)f(p)q(ersistence)166 1271 y Fk(Discussion:)166 1321 y Fj(Do)c(w)o(e)g(also)f(w)o(an)o(t)h(send-receiv)o(e)i(and)e(exc) o(hange)h(op)q(erations)g(with)f(the)g Ff(ready)g Fj(and)g Ff(synchronous)d Fj(mo)q(de?)166 1372 y(Do)13 b(w)o(e)h(w)o(an)o(t)f Ff(tag)p Fj(?)k(\(Collectiv)o(e)c(comm)o(unication)e(don't)i(use)h (tag;)f(but)h(tag)f(is)g(needed)i(if)e(w)o(e)h(w)o(an)o(t)f(to)g(b)q(e) 75 1421 y(able)h(to)f(mix)f(send-receiv)o(e)k(and)e(exc)o(hange)g(with) g(regular)g(sends)h(and)f(receiv)o(es.\))166 1472 y(Do)g(w)o(e)i(w)o (an)o(t)e(to)h(allo)o(w)f(wildcard)g(source)i(and/or)f(tag?)21 b(Since)16 b(I)f(assumed)f(that)i(the)f(message)g(sen)o(t)h(has)75 1522 y(the)e(same)f(tag)h(as)g(the)g(message)g(receiv)o(ed,)h(then)f (widcard)g(tag)g(do)q(es)g(not)g(mak)o(e)f(m)o(uc)o(h)f(sense.)166 1697 y Ft(MPI)p 275 1697 V 18 w(SENDRECV\(handle,)i(send)p 890 1697 V 18 w(bu\013er)p 1043 1697 V 18 w(handle,)f(dest,)g(recv)p 1458 1697 V 18 w(bu\013er)p 1611 1697 V 18 w(handle,)g(source,)75 1753 y(tag,)18 b(con)o(text\))166 1846 y Fs(Blo)q(c)o(king)i (send-receiv)o(e)g(op)q(eration.)31 b(The)19 b(op)q(eration)f(returns)h (when)g(b)q(oth)g(send)g(and)g(receiv)o(e)75 1902 y(ha)o(v)o(e)c (completed.)166 1960 y Fl(MPI)p 241 1960 15 2 v 17 w(SENDRECV\(handle,) 21 b(send)p 759 1960 V 17 w(buffer)p 920 1960 V 16 w(handle,)i(dest,)g (recv)p 1366 1960 V 17 w(buffer)p 1527 1960 V 16 w(handle,)g(source,)75 2016 y(tag,)g(context,)g(status\))14 b Fs(is)75 2126 y Fl(MPI_INIT_SENDRECV\(handle,)20 b(send_buffer_handle,)i(dest,)147 2183 y(recv_buffer_handle,)f(source,)i(tag,)g(context,)g (MPI_EPHEMERAL\))75 2239 y(MPI_START\(handle\))75 2296 y(MPI_WAIT\(handle,)f(status\))166 2441 y Ft(MPI)p 275 2441 16 2 v 18 w(EX)o(CHANGE\(handle,)13 b(bu\013er)p 934 2441 V 18 w(handle,)h(dest,)e(source,)h(tag,)g(con)o(text,)g (status\))166 2590 y Fs(Blo)q(c)o(king)j(exc)o(hange)f(op)q(eration.)20 b(The)15 b(op)q(eration)g(returns)f(when)i(b)q(oth)e(send)i(and)f (receiv)o(e)g(ha)o(v)o(e)75 2646 y(completed.)166 2704 y Fl(MPI)p 241 2704 15 2 v 17 w(EXCHANGE\(handle,)21 b(buffer)p 807 2704 V 17 w(handle,)i(dest,)g(source,)g(tag,)g(context,) g(status\))9 b Fs(is)p eop %%Page: 44 49 44 48 bop 75 -100 a Fs(44)613 b Fq(CHAPTER)16 b(2.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fl (MPI_INIT_EXCHANGE\(handle,)k(buffer_handle,)i(dest,)147 102 y(source,)g(tag,)i(context,)f(MPI_EPHEMERAL\))75 158 y(MPI_START\(handle\))75 214 y(MPI_WAIT\(handle,)f(status\))166 360 y Ft(MPI)p 275 360 16 2 v 18 w(ISENDRECV\(handle,)14 b(send)p 910 360 V 18 w(bu\013er)p 1063 360 V 18 w(handle,)f(dest,)f (recv)p 1477 360 V 19 w(bu\013er)p 1631 360 V 17 w(handle,)i(source,)75 416 y(tag,)k(con)o(text\))166 509 y Fs(Non)o(blo)q(c)o(king)e (send-receiv)o(e)h(op)q(eration.)166 566 y Fl(MPI)p 241 566 15 2 v 17 w(ISENDRECV\(handle,)k(send)p 783 566 V 17 w(buffer)p 944 566 V 16 w(handle,)i(dest,)g(recv)p 1390 566 V 17 w(buffer)p 1551 566 V 16 w(handle,)g(source,)75 622 y(tag,)g(context\))14 b Fs(is)75 732 y Fl (MPI_INIT_SENDRECV\(handle,)20 b(send_buffer_handle,)i(dest,)147 789 y(recv_buffer_handle,)f(source,)i(tag,)g(context,)g (MPI_EPHEMERAL\))75 845 y(MPI_START\(handle\))166 990 y Ft(MPI)p 275 990 16 2 v 18 w(IEX)o(CHANGE\(handle,)17 b(bu\013er)p 958 990 V 18 w(handle,)h(dest,)f(source,)f(tag,)i(con)o (text\))166 1083 y Fs(Non)o(blo)q(c)o(king)e(exc)o(hange)g(op)q (eration.)166 1140 y Fl(MPI)p 241 1140 15 2 v 17 w(IEXCHANGE\(handle,) 21 b(buffer)p 831 1140 V 17 w(handle,)h(dest,)i(source,)f(tag,)g (context\))14 b Fs(is)75 1250 y Fl(MPI_INIT_EXCHANGE\(handle,)20 b(buffer_handle,)i(dest,)147 1307 y(source,)g(tag,)i(context,)f (MPI_EPHEMERAL\))75 1363 y(MPI_START\(handle\))166 1508 y Ft(MPI)p 275 1508 16 2 v 18 w(SENDRECV)o(C\(handle,)29 b(send)p 942 1508 V 19 w(start,)f(send)p 1215 1508 V 19 w(coun)o(t,)h(send)p 1507 1508 V 18 w(datat)o(yp)q(e,)h(dest,)75 1565 y(recv)p 175 1565 V 18 w(start,)17 b(recv)p 430 1565 V 18 w(coun)o(t,)h(recv)p 704 1565 V 18 w(datat)o(yp)q(e,)g (source,)e(tag,)i(con)o(text,)f(status\))166 1657 y Fs(Blo)q(c)o(king)f (send-receiv)o(e)h(with)f(con)o(tiguous)f(bu\013ers.)166 1714 y Fl(MPI)p 241 1714 15 2 v 17 w(SENDRECVC\(handle,)21 b(send)p 783 1714 V 17 w(start,)i(send)p 1063 1714 V 17 w(count,)g(send)p 1343 1714 V 16 w(datatype,)g(dest,)g(recv)p 1837 1714 V 17 w(start,)75 1771 y(recv)p 174 1771 V 17 w(count,)g(recv)p 454 1771 V 16 w(datatype,)g(source,)g(tag,)g (context,)g(status\))14 b Fs(is)75 1881 y Fl (MPI_CREATE_BUFFER\(send_buf)o(fer_han)o(dle,)21 b(MPI_EPHEMERAL\))75 1938 y(MPI_CREATE_BUFFER\(recv_buf)o(fer_han)o(dle,)g(MPI_EPHEMERAL\)) 75 1994 y(MPI_APPEND_CONTIGUOUS\(send)o(_buffer)o(_handle,)f (send_start,)j(send_count,)194 2050 y(send_datatype\))75 2107 y(MPI_APPEND_CONTIGUOUS\(recv)o(_buffer)o(_handle,)d(recv_start,)j (recv_count,)194 2163 y(recv_datatype\))75 2220 y (MPI_COMMIT\(send_buffer_han)o(dle\))75 2276 y (MPI_COMMIT\(recv_buffer_han)o(dle\))75 2333 y (MPI_INIT_SENDRECV\(handle,)d(send_buffer_handle,)i(dest,)147 2389 y(recv_buffer_handle,)f(source,)i(tag,)g(context,)g (MPI_EPHEMERAL\))75 2446 y(MPI_START\(handle\))75 2502 y(MPI_WAIT\(handle,)f(status\))166 2647 y Ft(MPI)p 275 2647 16 2 v 18 w(EX)o(CHANGEC\(handle,)17 b(start,)h(coun)o(t,)g(datat) o(yp)q(e,)g(dest,)g(source,)f(tag,)h(con-)75 2704 y(text\))p eop %%Page: 45 50 45 49 bop 75 -100 a Fq(2.13.)34 b(SEND-RECEIVE)1277 b Fs(45)166 45 y(Blo)q(c)o(king)16 b(exc)o(hange)g(with)f(con)o(tiguous)h (bu\013er)166 102 y Fl(MPI)p 241 102 15 2 v 17 w(EXCHANGEC\(handle,)21 b(start,)i(count,)g(datatype,)g(dest,)g(source,)g(tag,)g(context\))75 158 y Fs(is)75 258 y Fl(MPI_CREATE_BUFFER\(buffer_h)o(andle,)d (MPI_EPHEMERAL\))75 314 y(MPI_APPEND_CONTIGUOUS\(buff)o(er_hand)o(le,)h (start,)i(count,)g(datatype\))75 371 y(MPI_COMMIT\(buffer_handle\))75 427 y(MPI_INIT_EXCHANGE\(handle,)d(buffer_handle,)i(dest,)147 483 y(source,)g(tag,)i(context,)f(MPI_EPHEMERAL\))75 540 y(MPI_START\(handle\))75 596 y(MPI_WAIT\(handle,)f(status\))166 731 y Ft(MPI)p 275 731 16 2 v 18 w(ISENDRECV)o(C\(handle,)i(send)p 957 731 V 19 w(start,)f(send)p 1225 731 V 19 w(coun)o(t,)g(send)p 1511 731 V 19 w(datat)o(yp)q(e,)i(dest,)75 788 y(recv)p 175 788 V 18 w(start,)17 b(recv)p 430 788 V 18 w(coun)o(t,)h(recv)p 704 788 V 18 w(datat)o(yp)q(e,)g(source,)e(tag,)i(con)o(text\))166 880 y Fs(Non)o(blo)q(c)o(king)e(send-receiv)o(e)h(with)f(con)o(tiguous) f(bu\013ers.)166 936 y Fl(MPI)p 241 936 15 2 v 17 w (ISENDRECVC\(handle,)21 b(send)p 807 936 V 17 w(start,)i(send)p 1087 936 V 16 w(count,)g(send)p 1366 936 V 17 w(datatype,)g(dest,)g (recv)p 1861 936 V 17 w(start,)75 993 y(recv)p 174 993 V 17 w(count,)g(recv)p 454 993 V 16 w(datatype,)g(source,)g(tag,)g (context\))14 b Fs(is)75 1092 y Fl(MPI_CREATE_BUFFER\(send_buf)o (fer_han)o(dle,)21 b(MPI_EPHEMERAL\))75 1149 y (MPI_CREATE_BUFFER\(recv_buf)o(fer_han)o(dle,)g(MPI_EPHEMERAL\))75 1205 y(MPI_APPEND_CONTIGUOUS\(send)o(_buffer)o(_handle,)f(send_start,)j (send_count,)194 1262 y(send_datatype\))75 1318 y (MPI_APPEND_CONTIGUOUS\(recv)o(_buffer)o(_handle,)d(recv_start,)j (recv_count,)194 1375 y(recv_datatype\))75 1431 y (MPI_COMMIT\(send_buffer_han)o(dle\))75 1487 y (MPI_COMMIT\(recv_buffer_han)o(dle\))75 1544 y (MPI_INIT_SENDRECV\(handle,)d(send_buffer_handle,)i(dest,)147 1600 y(recv_buffer_handle,)f(source,)i(tag,)g(context,)g (MPI_EPHEMERAL\))75 1657 y(MPI_START\(handle\))166 1792 y Ft(MPI)p 275 1792 16 2 v 18 w(IEX)o(CHANGEC\(handle,)14 b(start,)h(coun)o(t,)g(datat)o(yp)q(e,)h(dest,)f(source,)f(tag,)h(con-) 75 1848 y(text\))166 1940 y Fs(Non)o(blo)q(c)o(king)h(exc)o(hange)g (with)f(con)o(tiguous)h(bu\013er)166 1997 y Fl(MPI)p 241 1997 15 2 v 17 w(IEXCHANGEC\(handle,)21 b(start,)i(count,)g (datatype,)g(dest,)g(source,)g(tag,)g(context\))75 2053 y Fs(is)75 2153 y Fl(MPI_CREATE_BUFFER\(buffer_h)o(andle,)d (MPI_EPHEMERAL\))75 2209 y(MPI_APPEND_CONTIGUOUS\(buff)o(er_hand)o(le,) h(start,)i(count,)g(datatype\))75 2266 y(MPI_INIT_EXCHANGE\(handle,)d (buffer_handle,)i(dest,)147 2322 y(source,)g(tag,)i(context,)f (MPI_EPHEMERAL\))75 2379 y(MPI_START\(handle\))166 2554 y Fk(Discussion:)166 2604 y Fj(Can)15 b(w)o(e)g(cancel)h(a)f (send-receiv)o(e)j(or)d(exc)o(hange?)23 b(If)15 b(y)o(es,)g(should)g (it)g(b)q(e)h(the)g(case)g(that)g(either)g(w)o(e)f(cancel)75 2654 y(b)q(oth)i(send)h(and)f(receiv)o(e)h(or)f(neither?)28 b(This)16 b(seems)i(natural,)e(but)h(ma)o(y)e(cause)j(problems)e(\(The) i(receiv)o(e)g(has)75 2704 y(already)c(executed,)h(but)f(the)g(send)h (is)f(h)o(ung.)k(Ho)o(w)13 b(do)h(w)o(e)g(cancel?\))p eop %%Page: 46 51 46 50 bop 75 -100 a Fs(46)608 b Fq(CHAPTER)15 b(2.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y Fj(This)e(is)f(the) i(pream)o(ble)e(to)h(a)f(m)o(uc)o(h)g(more)g(general)h(question:)k(can) c(w)o(e)g(cancel)h(collectiv)o(e)f(op)q(erations?)75 271 y Fm(2.14)70 b(Null)21 b(pro)r(cesses)75 455 y Fk(Discussion:)34 b Fj(This)14 b(section)g(has)g(not)g(b)q(een)h(review)o(ed)g(b)o(y)f (MPIF.)166 594 y Fs(In)21 b(man)o(y)f(instances,)h(it)g(is)g(con)o(v)o (enien)o(t)g(to)e(sp)q(ecify)j(a)e(\\dumm)o(y")f(source)i(or)e (destination)j(for)75 651 y(comm)o(unication.)27 b(This)18 b(simpli\014es)i(the)d(co)q(de)h(that)f(is)h(needed)g(for)f(dealing)i (with)e(b)q(oundaries,)i(e.g.,)75 707 y(in)d(the)f(case)h(of)e(a)h (noncircular)i(shift)e(done)h(with)f(calls)i(to)d(send-receiv)o(e.)166 764 y(The)i(sp)q(ecial)h(v)m(alue)g Fl(MPI)p 601 764 15 2 v 17 w(PROCNULL)e Fs(can)g(b)q(e)i(used)f(instead)g(of)g(a)f(rank) g(wherev)o(er)h(a)f(source)h(or)f(a)75 820 y(destination)j(parameter)e (is)i(required)g(in)g(a)e(call.)27 b(A)17 b(comm)o(unication)h(with)f (pro)q(cess)g Fl(MPI)p 1670 820 V 17 w(PROCNULL)75 877 y Fs(has)10 b(no)g(e\013ect:)17 b(a)10 b(send)h(to)e Fl(MPI)p 601 877 V 17 w(PROCNULL)g Fs(succeeds)i(and)g(returns)f(asap.) 17 b(A)11 b(receiv)o(e)g(from)e Fl(MPI)p 1714 877 V 17 w(PROCNULL)75 933 y Fs(succeeds)16 b(and)g(returns)f(asap)g(with)g(no)g (mo)q(di\014cations)i(to)d(the)h(receiv)o(e)h(bu\013er.)166 1065 y Fk(Discussion:)166 1115 y Fj(Need)c(to)f(decide)i(what)e(is)g (returned)i(in)e(the)h(status)g(ob)r(ject:)17 b(I)12 b(suggest)g Ff(source)20 b(=)i(MPI)p 1557 1115 14 2 v 15 w(PROCNULL)p Fj(,)9 b Ff(tag=0)75 1165 y Fj(and)14 b Ff(count=0)p Fj(.)p eop %%Page: 47 52 47 51 bop 75 358 a Fn(Chapter)34 b(3)75 567 y Fu(Collecti)q(v)m(e)42 b(Comm)m(unication)892 791 y Fs(Al)16 b(Geist)876 847 y(Marc)e(Snir)75 998 y Fm(3.1)70 b(In)n(tro)r(duction)75 1102 y Fs(This)22 b(section)f(is)h(a)e(draft)h(of)f(the)h(curren)o(t)g (prop)q(osal)g(for)g(collectiv)o(e)i(comm)o(unication.)38 b(Collectiv)o(e)75 1158 y(comm)o(unication)12 b(is)f(de\014ned)i(to)d (b)q(e)i(comm)o(unication)g(that)e(in)o(v)o(olv)o(es)h(a)g(group)g(of)g (pro)q(cesses.)18 b(Examples)75 1215 y(are)c(broadcast)g(and)g(global)i (sum.)j(A)c(collectiv)o(e)h(op)q(eration)f(is)f(executed)i(b)o(y)e(ha)o (ving)h(all)g(pro)q(cesses)g(in)75 1271 y(the)j(group)f(call)h(the)g (comm)o(unication)g(routine,)h(with)e(matc)o(hing)h(parameters.)26 b(Routines)18 b(can)g(\(but)75 1328 y(are)f(not)g(required)h(to\))e (return)i(as)f(so)q(on)g(as)g(their)g(participation)i(in)f(the)f (collectiv)o(e)i(comm)o(unication)75 1384 y(is)h(complete.)34 b(The)19 b(completion)i(of)e(a)g(call)i(indicates)g(that)e(the)g (caller)i(is)f(no)o(w)f(free)g(to)g(access)h(the)75 1441 y(lo)q(cations)i(in)g(the)f(comm)o(unication)g(bu\013er,)h(or)f(an)o(y) g(other)f(lo)q(cation)i(that)e(can)h(b)q(e)h(referenced)g(b)o(y)75 1497 y(the)c(collectiv)o(e)j(op)q(eration.)29 b(Ho)o(w)o(ev)o(er,)18 b(it)h(do)q(es)f(not)g(indicate)i(that)e(other)g(pro)q(cesses)h(in)g (the)f(group)75 1554 y(ha)o(v)o(e)c(started)g(the)h(op)q(eration)g (\(unless)h(otherwise)f(indicated)h(in)g(the)f(description)h(of)f(the)f (op)q(eration\).)75 1610 y(Ho)o(w)o(ev)o(er,)i(the)g(successful)i (completion)g(of)e(a)h(collectiv)o(e)h(comm)o(unication)f(call)h(ma)o (y)e(dep)q(end)i(on)f(the)75 1667 y(execution)f(of)f(a)g(matc)o(hing)g (call)h(at)f(all)h(pro)q(cesses)g(in)g(the)f(group.)166 1724 y(The)g(syn)o(tax)g(and)g(seman)o(tics)g(of)g(the)g(collectiv)o(e) i(op)q(erations)f(is)f(de\014ned)i(so)e(as)f(to)h(b)q(e)h(consisten)o (t)75 1781 y(with)g(the)f(syn)o(tax)f(and)i(seman)o(tics)f(of)g(the)g (p)q(oin)o(t)h(to)e(p)q(oin)o(t)i(op)q(erations.)166 1839 y(The)f(reader)f(is)h(referred)f(to)g(the)g(p)q(oin)o(t-to-p)q (oin)o(t)h(comm)o(unication)g(section)g(of)f(the)h(curren)o(t)f(MPI)75 1895 y(draft)g(for)f(information)i(concerning)g(comm)o(unication)g (bu\013ers)g(and)f(their)h(manipulations.)21 b(The)15 b(con-)75 1952 y(text)d(section)h(describ)q(es)g(the)g(formation,)e (manipulation,)j(and)f(query)f(functions)h(\(suc)o(h)g(as)e(group)h (size\))75 2008 y(that)i(are)h(a)o(v)m(ailable)i(for)e(groups)g(and)g (group)g(ob)s(jects.)166 2066 y(The)23 b(collectiv)o(e)i(comm)o (unication)f(routines)f(are)g(built)h(ab)q(o)o(v)o(e)f(the)g(p)q(oin)o (t-to-p)q(oin)o(t)h(routines.)75 2122 y(While)14 b(v)o(endors)d(ma)o(y) h(optimize)h(certain)g(collectiv)o(e)h(routines)e(for)g(their)g(arc)o (hitectures,)h(a)f(complete)h(li-)75 2179 y(brary)h(of)g(the)h (collectiv)o(e)h(comm)o(unication)g(routines)f(can)f(b)q(e)i(written)e (en)o(tirely)i(using)f(p)q(oin)o(t-to-p)q(oin)o(t)75 2235 y(comm)o(unication)21 b(functions.)37 b(W)l(e)20 b(are)h(using)g(naiv)o(e)g(implemen)o(tations)h(of)e(the)g(collectiv)o (e)j(calls)e(in)75 2292 y(terms)f(of)f(p)q(oin)o(t)i(to)e(p)q(oin)o(t)i (op)q(erations)f(in)h(order)f(to)f(pro)o(vide)i(an)f(op)q(erational)h (de\014nition)h(of)d(their)75 2348 y(seman)o(tics.)166 2406 y(The)c(follo)o(wing)h(comm)o(unication)g(functions)g(are)f(prop)q (osed.)143 2505 y Fo(\017)23 b Fs(Broadcast)14 b(from)g(one)i(mem)o(b)q (er)f(to)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 2604 y Fo(\017)23 b Fs(Barrier)15 b(across)f(all)j(group)d(mem)o(b)q (ers)143 2704 y Fo(\017)23 b Fs(Gather)14 b(data)h(from)f(all)i(group)f (mem)o(b)q(ers)g(to)g(one)g(mem)o(b)q(er.)952 2828 y(47)p eop %%Page: 48 53 48 52 bop 75 -100 a Fs(48)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)143 45 y Fo(\017)23 b Fs(Scatter)14 b(data)h(from)f(one)i(mem)o(b)q(er)f(to)g(all)h(mem)o (b)q(ers)f(of)g(a)g(group.)143 137 y Fo(\017)23 b Fs(Global)15 b(op)q(erations)g(suc)o(h)h(as)e(sum,)h(max,)f(min,)i(etc.,)e(w)o(ere)h (the)g(result)g(is)h(kno)o(wn)e(b)o(y)h(all)h(group)189 193 y(mem)o(b)q(ers)e(and)h(a)g(v)m(ariation)g(where)g(the)g(result)g (is)g(kno)o(wn)f(b)o(y)h(only)g(one)g(mem)o(b)q(er.)20 b(The)15 b(abilit)o(y)189 250 y(to)f(ha)o(v)o(e)h(user)g(de\014ned)i (global)f(op)q(erations.)143 342 y Fo(\017)23 b Fs(Sim)o(ultaneous)14 b(shift)g(of)f(data)g(around)g(the)g(group,)h(the)f(simplest)i(example) f(b)q(eing)h(all)f(mem)o(b)q(ers)189 398 y(sending)i(their)g(data)e(to) h(\(rank+1\))f(with)i(wrap)e(around.)143 490 y Fo(\017)23 b Fs(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g (called)h(parallel)h(pre\014x\).)143 582 y Fo(\017)23 b Fs(Broadcast)14 b(from)g(all)j(mem)o(b)q(ers)e(to)f(all)j(mem)o(b)q (ers)e(of)g(a)g(group.)143 674 y Fo(\017)23 b Fs(Scatter)18 b(data)f(from)h(all)h(mem)o(b)q(ers)g(to)f(all)h(mem)o(b)q(ers)f(of)g (a)g(group)h(\(also)f(called)i(complete)f(ex-)189 730 y(c)o(hange)c(or)g(index\).)166 819 y(T)l(o)g(simplify)i(the)f (collectiv)o(e)h(comm)o(unication)g(in)o(terface)e(it)h(is)g(designed)h (with)f(t)o(w)o(o)e(la)o(y)o(ers.)21 b(The)75 876 y(lo)o(w)e(lev)o(el)i (routines)e(ha)o(v)o(e)g(all)i(the)e(generalit)o(y)h(of,)f(and)h(mak)o (e)e(use)i(of,)g(the)f(comm)o(unication)h(bu\013er)75 932 y(routines)15 b(of)g(the)f(p)q(oin)o(t-to-p)q(oin)o(t)i(section)f (whic)o(h)h(allo)o(ws)f(arbitrarily)g(complex)h(messages)e(to)g(b)q(e)i (con-)75 989 y(structed.)k(The)14 b(second)h(lev)o(el)g(routines)g(are) f(similar)i(to)d(the)i(upp)q(er)g(lev)o(el)g(p)q(oin)o(t-to-p)q(oin)o (t)g(routines)g(in)75 1045 y(that)f(they)i(send)g(only)f(a)g(con)o (tiguous)g(bu\013er.)75 1187 y Fm(3.2)70 b(Group)24 b(F)-6 b(unctions)75 1289 y Fs(A)17 b(full)h(description)g(of)e(the)h(group)f (formation)g(and)h(manipulation)h(functions)f(can)g(b)q(e)g(found)g(in) h(the)75 1345 y(con)o(text)f(c)o(hapter)h(of)f(the)g(MPI)h(do)q(cumen)o (t.)28 b(Here)18 b(w)o(e)f(describ)q(e)i(only)g(those)e(group)g (functions)i(that)75 1402 y(are)c(used)h(in)g(the)f(seman)o(tic)g (description)i(of)e(the)g(collectiv)o(e)i(comm)o(unication)f(routines.) 166 1458 y(An)g(initial)j(group)d(con)o(taining)h(all)g(pro)q(cesses)g (is)g(supplied)h(b)o(y)e(default)h(in)g(MPI.)f(MPI)g(pro)o(vides)75 1515 y(a)f(pro)q(cedure)g(that)f(returns)h(the)g(handle)h(to)e(this)i (initial)h(group.)i(The)c(pro)q(cesses)g(in)h(the)f(inital)h(group)75 1571 y(eac)o(h)22 b(ha)o(v)o(e)f(a)h(unique)h(rank)e(in)i(the)f(group)f (represen)o(ted)h(b)o(y)g(in)o(tegers)g(\(0,)g(1,)h(2,)f(...,)g([n)o (um)o(b)q(er-of-)75 1628 y(pro)q(cesses)16 b(-)f(1].)166 1720 y Ft(MPI)p 275 1720 16 2 v 18 w(COMM)p 470 1720 V 18 w(INIT\(group\))d Fs(Return)g(the)g(comm)o(unicator)g(of)f(the)h (inital)h(group)e(con)o(taining)75 1811 y(all)16 b(pro)q(cesses.)75 1900 y Ft(OUT)i(group)23 b Fs(group)15 b(comm)o(unicator.)166 2025 y Ft(MPI)p 275 2025 V 18 w(RANK\(group,)i(rank\))d Fs(Return)i(the)f(rank)g(of)g(the)g(calling)i(pro)q(cess)e(within)h (the)f(sp)q(eci-)75 2117 y(\014ed)h(group.)75 2206 y Ft(IN)h(group)23 b Fs(group)15 b(comm)o(unicator)75 2298 y Ft(OUT)j(rank)23 b Fs(in)o(teger)166 2422 y Ft(MPI)p 275 2422 V 18 w(GSIZE\(group,)18 b(size\))e Fs(Return)g(the)f(n)o(um)o (b)q(er)h(of)f(pro)q(cesses)h(that)e(b)q(elong)j(to)e(the)g(sp)q(ec-)75 2514 y(i\014ed)h(group.)75 2603 y Ft(IN)h(group)23 b Fs(group)15 b(comm)o(unicator)75 2695 y Ft(OUT)j(size)23 b Fs(in)o(teger)p eop %%Page: 49 54 49 53 bop 75 -100 a Fq(3.3.)34 b(COMMUNICA)l(TION)16 b(FUNCTIONS)942 b Fs(49)75 45 y Fm(3.3)70 b(Comm)n(unication)20 b(F)-6 b(unctions)75 147 y Fs(The)11 b(prop)q(osed)g(comm)o(unication)h (functions)g(are)e(divided)j(in)o(to)e(t)o(w)o(o)f(la)o(y)o(ers.)18 b(The)11 b(lo)o(w)o(est)f(lev)o(el)j(uses)e(the)75 203 y(same)j(bu\013er)g(descriptor)h(ob)s(jects)e(a)o(v)m(ailable)j(in)f(p) q(oin)o(t-to-p)q(oin)o(t)g(to)f(create)g(noncon)o(tiguous,)g(m)o (ultiple)75 259 y(data)k(t)o(yp)q(e)h(messages.)31 b(The)19 b(second)h(lev)o(el)g(is)g(similar)g(to)e(the)h(blo)q(c)o(k)h (send/receiv)o(e)g(p)q(oin)o(t-to-p)q(oin)o(t)75 316 y(op)q(erations)g(in)g(that)f(it)h(supp)q(orts)g(only)g(con)o(tiguous)g (bu\013ers)g(of)f(data.)32 b(F)l(or)19 b(eac)o(h)h(comm)o(unication)75 372 y(op)q(eration,)15 b(w)o(e)g(list)h(these)f(t)o(w)o(o)f(lev)o(el)j (of)e(calls)h(together.)75 515 y Fm(3.4)70 b(Sync)n(hronization)75 617 y Ft(Barrier)17 b(sync)o(hronization)75 738 y(MPI)p 184 738 16 2 v 18 w(BARRIER\()h(group)g(\))166 830 y Fs(MPI)p 258 830 14 2 v 16 w(BARRIER)g(blo)q(c)o(ks)f(the)f(calling)i (pro)q(cess)e(un)o(til)h(all)g(group)e(mem)o(b)q(ers)i(ha)o(v)o(e)e (called)j(it;)e(the)75 886 y(call)g(returns)f(at)g(an)o(y)g(pro)q(cess) g(only)h(after)e(all)j(group)e(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered) g(the)g(call.)75 978 y Ft(IN)i(group)23 b Fs(group)15 b(comm)o(unicator)166 1070 y(The)20 b(follo)o(wing)g(sho)o(ws)f(one)h (p)q(ossible)h(implemen)o(tation)g(of)e(the)h(barrier)f(function.)34 b(Similarly)l(,)75 1126 y(man)o(y)16 b(of)g(the)h(other)f(collectiv)o (e)j(comm)o(unication)e(functions)g(are)g(illustrated)h(b)o(y)e(p)q (ossible)j(p)q(oin)o(t-to-)75 1183 y(p)q(oin)o(t)h(implemen)o(tations.) 34 b(These)20 b(are)g(men)o(t)f(for)g(illustration)i(and)f(are)f(not)g (suggested)h(to)f(b)q(e)h(the)75 1239 y(most)14 b(e\016cien)o(t)i(w)o (a)o(ys)e(to)h(implemen)o(t)h(the)g(giv)o(en)f(functions.)21 b Fl(MPI)p 1229 1239 15 2 v 17 w(BARRIER\()i(group)g(\))15 b Fs(is)75 1331 y Fl(MPI_CREATE_BUFFER\(buffer_h)o(andle,)20 b(MPI_BUFFER,)j(MPI_PERSISTENT\);)75 1388 y(MPI_GSIZE\()f(group,)i (&size)f(\);)75 1444 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 1500 y(if)h(\(rank==0\))75 1557 y({)147 1613 y(for)f(\(i=1;)g(i)h(<)g (size;)f(i++\))218 1670 y(MPI_RECV\(buffer_handle,)e(i,)i(tag,)h (group\);)147 1726 y(for)f(\(i=1;)g(i)h(<)g(size;)f(i++\))218 1783 y(MPI_SEND\(buffer_handle,)e(i,)i(tag,)h(group\);)75 1839 y(})75 1896 y(else)75 1952 y({)147 2009 y (MPI_SEND\(buffer_handle,)c(0,)k(tag,)f(group\);)147 2065 y(MPI_RECV\(buffer_handle,)d(0,)k(tag,)f(group\);)75 2121 y(})75 2178 y(MPI_FREE\(buffer_handle\);)166 2305 y Ft(MPI)p 275 2305 16 2 v 18 w(SYNCH\()17 b(group,)g(map)g(\))166 2397 y Fs(MPI)p 258 2397 14 2 v 16 w(SYNCH)i(sync)o(hronizes)g(only)g (the)g(subgroup)f(of)g(pro)q(cesses)h(sp)q(eci\014ed)h(b)o(y)e(map.)30 b(Map)17 b(is)75 2453 y(an)d(in)o(teger)f(arra)o(y)g(con)o(taining)h (the)g(ranks)f(of)h(the)f(pro)q(cesses)h(in)h(group)e(that)g(are)h(to)f (b)q(e)h(sync)o(hronized.)75 2510 y(This)i(function)g(is)g(needed)g(to) f(b)q(o)q(otstrap)f(the)h(safe)g(formation)g(of)f(subgroups.)75 2602 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)75 2695 y Ft(IN)i(map)23 b Fs(an)15 b(arra)o(y)f(of)h(in)o(tegers.)p eop %%Page: 50 55 50 54 bop 75 -100 a Fs(50)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fm(3.5)70 b(Data)23 b(mo)n(v)n(e)f(functions)75 149 y Ft(Broadcast)75 272 y(MPI)p 184 272 16 2 v 18 w(BCAST\()c(bu\013er)p 555 272 V 17 w(handle,)g(group,)f(ro)q(ot)h(\))166 365 y Fl(MPI)p 241 365 15 2 v 17 w(BCAST)12 b Fs(broadcasts)f(a)h(message)h (from)e(the)i(pro)q(cess)g(with)g(rank)f Fl(root)g Fs(to)g(all)h(other) g(pro)q(cesses)75 421 y(of)18 b(the)g(group.)29 b(It)18 b(is)h(called)h(b)o(y)e(all)h(mem)o(b)q(ers)g(of)e(group)h(using)h(the) g(same)f(argumen)o(ts)f(for)h Fl(group,)75 478 y(and)23 b(root)p Fs(.)d(On)15 b(return)h(the)f(con)o(ten)o(ts)f(of)h(the)g (bu\013er)g(of)g(the)g(pro)q(cess)h(with)f(rank)g Fl(root)g Fs(is)g(con)o(tained)75 534 y(in)h(the)f(bu\013er)g(of)g(the)h(calling) g(pro)q(cess.)75 632 y Ft(INOUT)i(bu\013er)p 407 632 16 2 v 18 w(handle)24 b Fs(Handle)16 b(for)e(bu\013er)i(where)f(from)g (message)f(is)i(sen)o(t)f(or)g(receiv)o(ed.)75 730 y Ft(IN)i(group)23 b Fs(group)15 b(comm)o(unicator)75 828 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(broadcast)g(ro)q(ot)f(\(in)o (teger\))166 961 y Ft(MPI)p 275 961 V 18 w(BCASTC\()j(buf,)g(len,)h(t)o (yp)q(e,)f(group,)f(ro)q(ot)i(\))166 1054 y Fl(MPI)p 241 1054 15 2 v 17 w(BCASTC)i Fs(b)q(eha)o(v)o(es)h(lik)o(e)i (broadcast,)e(restricted)h(to)e(a)h(blo)q(c)o(k)h(bu\013er.)37 b(It)22 b(is)f(called)i(b)o(y)e(all)75 1111 y(pro)q(cesses)16 b(with)f(the)g(same)g(argumen)o(ts)g(for)f Fl(len,)24 b(group)14 b Fs(and)h Fl(root)p Fs(.)75 1208 y Ft(INOUT)j(bu\013er)k Fs(Starting)15 b(address)g(of)g(bu\013er)g(\(matc)o(hes)f(t)o(yp)q(e\)) 75 1306 y Ft(IN)j(len)23 b Fs(Num)o(b)q(er)16 b(of)f(en)o(tries)g(in)h (bu\013er)g(\(in)o(teger\))75 1404 y Ft(IN)h(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 1502 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)75 1600 y Ft(in)j(ro)q(ot)23 b Fs(rank)15 b(of)g(broadcast)f(ro)q(ot)h(\(in)o(teger\))166 1698 y Fl(MPI)p 241 1698 V 17 w(BCAST\()23 b(buffer)p 569 1698 V 16 w(handle,)g(group,)g(root)g(\))15 b Fs(is)75 1797 y Fl(MPI_GSIZE\()22 b(group,)i(&size)f(\);)75 1853 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 1910 y(MPI_IRECV\(handle,)f (buffer_handle,)f(root,)j(tag,)f(group\);)75 1966 y(if)h (\(rank==root\))147 2023 y(for)f(\(i=0;)g(i)h(<)g(size;)f(i++\))218 2079 y(MPI_SEND\(buffer_handle,)e(i,)i(tag,)h(group\);)75 2136 y(MPI_WAIT\(handle,)e(return_handle\))75 2262 y Ft(Gather)75 2385 y(MPI)p 184 2385 16 2 v 18 w(GA)l(THER\()c(inhandle,) g(list)p 762 2385 V 20 w(of)p 824 2385 V 19 w(outhandles,)g(group,)f (ro)q(ot)h(\))166 2478 y Fs(Eac)o(h)f(pro)q(cess)g(\(including)i(the)e (ro)q(ot)f(pro)q(cess\))h(sends)h(the)f(con)o(ten)o(t)f(of)h(its)g (input)h(bu\013er)f(to)f(the)75 2534 y(ro)q(ot)d(pro)q(cess.)19 b(The)14 b(ro)q(ot)f(pro)q(cess)h(places)g(all)h(the)f(incoming)h (messages)e(in)h(the)g(lo)q(cation)g(sp)q(eci\014ed)i(b)o(y)75 2591 y(the)g(output)f(bu\013er)h(handle)h(corresp)q(onding)g(to)e(the)h (sender's)g(rank.)21 b(F)l(or)15 b(example,)h(the)g(ro)q(ot)f(places)75 2647 y(the)f(data)g(from)f(pro)q(cess)h(with)h(rank)f(3)f(in)i(the)g (lo)q(cation)f(sp)q(eci\014ed)j(b)o(y)d(the)g(third)h(bu\013er)f (descriptor)g(in)75 2704 y(the)f(list)g(of)f(outhandles.)20 b(The)12 b(list)p 673 2704 14 2 v 18 w(of)p 728 2704 V 16 w(outhandles)h(argumen)o(t)f(is)h(ignored)g(for)f(all)h(non-ro)q (ot)f(pro)q(cesses.)p eop %%Page: 51 56 51 55 bop 75 -100 a Fq(3.5.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b Fs(51)75 45 y(The)19 b(routine)g(is)g(called)i(b)o(y) d(all)i(mem)o(b)q(ers)f(of)f(group)h(using)g(the)g(same)f(argumen)o(ts) g(for)g Fl(group)p Fs(,)h(and)75 102 y Fl(root)p Fs(.)g(The)d(input)g (bu\013er)f(of)g(eac)o(h)g(pro)q(cess)h(ma)o(y)e(ha)o(v)o(e)h(a)g (di\013eren)o(t)g(length.)166 244 y Fk(Discussion:)55 b Fj(Jim)17 b(Co)o(wnie)i(and)g(Al)g(Geist)g(ha)o(v)o(e)g(prop)q(osed)h (that)f(MPI)g(allo)o(w)f(a)g(handle)h(to)g(p)q(oin)o(t)75 300 y(to)e(a)g(list)f(of)h(handles.)27 b(This)17 b(feature)h(w)o(ould)e (clean)h(up)g(the)h(collectiv)o(e)f(comm)o(unication)d(in)o(terface)j (and)g(add)75 357 y(additional)e(functionalit)o(y)h(to)h(MPI.)g (Because)i(the)f(list)e(of)h(handles)g(concept)i(\014rst)f(app)q(ears)g (in)f(the)g(p)q(oin)o(t)g(to)75 413 y(p)q(oin)o(t)12 b(c)o(hapter)i(under)f(MPI)p 529 413 13 2 v 16 w(W)-5 b(AIT)m(ANY,)12 b(this)h(prop)q(osal)f(as)h(written)g(up)g(b)o(y)f(Jim) f(should)h(b)q(e)i(discussed)g(there.)75 659 y Ft(IN)j(inhandle)24 b Fs(handle)17 b(to)d(input)j(bu\013er)e(descriptor)75 766 y Ft(IN)i(list)p 225 766 16 2 v 20 w(of)p 287 766 V 19 w(outhandles)24 b Fs(list)16 b(of)f(bu\013er)g(descriptor)g (handles)i(\(ro)q(ot\))75 873 y Ft(IN)g(group)23 b Fs(group)15 b(comm)o(unicator)75 980 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(receiving)i(pro)q(cess)e(\(in)o(teger\))166 1122 y Ft(MPI)p 275 1122 V 18 w(GA)l(THER)o(C\()i(in)o(buf,)g(outbuf,)h (inlen,)g(t)o(yp)q(e,)f(group,)f(ro)q(ot\))166 1218 y Fl(MPI)p 241 1218 15 2 v 17 w(GATHERC)g Fs(b)q(eha)o(v)o(es)i(lik)o(e)h Fl(MPI)p 772 1218 V 17 w(GATHER)e Fs(restricted)h(to)f(blo)q(c)o(k)h (bu\013ers,)g(and)g(with)g(the)g(addi-)75 1274 y(tional)f(restriction)h (that)e(all)i(input)g(bu\013ers)e(should)i(ha)o(v)o(e)f(the)g(same)f (length.)26 b(All)18 b(pro)q(cesses)f(should)75 1331 y(pro)o(vided)f(the)f(same)g(v)m(alues)i(for)d Fl(inlen,)23 b(group)p Fs(,)14 b(and)i Fl(root)e Fs(.)75 1437 y Ft(IN)j(in)o(buf)23 b Fs(\014rst)15 b(v)m(ariable)h(of)f(input)h(bu\013er)g(\(matc)o(hes)e (t)o(yp)q(e\))75 1544 y Ft(OUT)k(outbuf)24 b Fs(\014rst)14 b(v)m(ariable)j(of)e(output)g(bu\013er)g({)g(signi\014can)o(t)h(only)g (at)e(ro)q(ot)h(\(matc)o(hes)f(t)o(yp)q(e\))75 1651 y Ft(IN)j(inlen)24 b Fs(Num)o(b)q(er)16 b(of)e(\(w)o(ord\))g(v)m (ariables)j(in)f(input)g(bu\013er)f(\(in)o(teger\))75 1758 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er) 75 1865 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)75 1972 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(receiving)i(pro)q(cess)e (\(in)o(teger\))166 2079 y Fl(MPI)p 241 2079 V 17 w(GATHERC\()22 b(inbuf,)h(outbuf,)g(inlen,)g(type,)h(group,)f(root\))38 b Fs(is)75 2189 y Fl(MPI_GSIZE\()22 b(&size,)i(group\);)75 2246 y(MPI_RANK\()f(&rank,)g(group\);)75 2302 y(MPI_ISENDC\(handle,)e (inbuf,)i(inlen,)g(root,)h(tag,)f(group\);)75 2359 y(if)h (\(rank==root\))147 2415 y(for)f(\(i=0;)g(i)h(<)g(size;)f(i++\))147 2471 y({)218 2528 y(MPI_RECVC\(outbuf,)f(inlen,)h(i,)g(tag,)h(group\);) 218 2584 y(outbuf)f(+=)h(inlen;)147 2641 y(})75 2697 y(MPI_WAIT\(handle,)e(return_handle\);)p eop %%Page: 52 57 52 56 bop 75 -100 a Fs(52)708 b Fq(CHAPTER)15 b(3.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(Scatter)75 167 y(MPI)p 184 167 16 2 v 18 w(SCA)l(TTER\()i(list)p 557 167 V 20 w(of)p 619 167 V 19 w(inhandles,)g(outhandle,)g(group,)f (ro)q(ot)h(\))166 259 y Fs(The)d(ro)q(ot)f(pro)q(cess)i(sends)f(the)g (con)o(ten)o(t)g(of)f(its)i Fl(i)p Fs(-th)f(input)h(bu\013er)f(to)f (the)h(pro)q(cess)g(with)h(rank)f Fl(i)p Fs(;)75 315 y(eac)o(h)g(pro)q(cess)h(\(including)i(the)e(ro)q(ot)e(pro)q(cess\))h (stores)g(the)h(incoming)g(message)f(in)i(its)e(output)g(bu\013er.)75 371 y(The)f(routine)g(is)g(called)h(b)o(y)f(all)h(mem)o(b)q(ers)e(of)h (the)f(group)h(using)g(the)g(same)f(argumen)o(ts)g(for)g Fl(group)p Fs(,)g(and)75 428 y Fl(root)p Fs(.)75 522 y Ft(IN)k(list)p 225 522 V 20 w(of)p 287 522 V 19 w(inhandles)24 b Fs(list)16 b(of)f(bu\013er)g(descriptor)g(handles)i(\(only)e (signi\014can)o(t)h(on)f(the)h(ro)q(ot\))75 616 y Ft(IN)h(outhandle)24 b Fs(bu\013er)16 b(descriptor)f(handle)75 711 y Ft(IN)i(group)23 b Fs(group)15 b(comm)o(unicator)75 805 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(sending)i(pro)q(cess)e(\(in)o(teger\))166 899 y Fl(MPI)p 241 899 15 2 v 17 w(SCATTER\()22 b(list)p 568 899 V 17 w(of)p 633 899 V 17 w(inhandles,)g(outhandle,)h(group,)g (root)g(\))15 b Fs(is)75 994 y Fl(MPI_GSIZE\()22 b(group,)i(&size)f (\);)75 1050 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 1107 y(MPI_IRECV\(handle,)f(outhandle,)g(root,)h(tag,)h(group\);)75 1163 y(if)g(\(rank=root\))147 1219 y(for)f(\(i=0;)g(i)h(<)g(size;)f (i++\))218 1276 y(MPI_SEND\(list_of_inhandles[)o(i],)e(i,)j(tag,)f (group\);)75 1332 y(MPI_WAIT\(handle,)f(return_handle\);)166 1462 y Ft(MPI)p 275 1462 16 2 v 18 w(SCA)l(TTER)o(C\()17 b(in)o(buf,)g(outbuf,)h(len,)f(t)o(yp)q(e,)g(group,)g(ro)q(ot\))166 1554 y Fl(MPI)p 241 1554 15 2 v 17 w(SCATTERC)e Fs(b)q(eha)o(v)o(es)i (lik)o(e)h Fl(MPI)p 793 1554 V 17 w(SCATTER)e Fs(restricted)h(to)f(blo) q(c)o(k)i(bu\013ers,)f(and)g(with)g(the)g(ad-)75 1611 y(ditional)j(restriction)g(that)e(all)h(output)g(bu\013ers)g(ha)o(v)o (e)f(the)h(same)f(length.)32 b(The)19 b(input)g(bu\013er)g(blo)q(c)o(k) 75 1667 y(of)e(the)h(ro)q(ot)e(pro)q(cess)i(is)g(partitioned)g(in)o(to) g Fl(n)f Fs(consecutiv)o(e)i(blo)q(c)o(ks,)f(eac)o(h)g(consisting)g(of) f Fl(len)g Fs(w)o(ords.)75 1723 y(The)d Fl(i)p Fs(-th)h(blo)q(c)o(k)f (is)h(sen)o(t)f(to)g(the)g Fl(i)p Fs(-th)g(pro)q(cess)g(in)i(the)e (group)g(and)g(stored)g(in)h(its)f(output)g(bu\013er.)20 b(The)75 1780 y(routine)e(is)h(called)g(b)o(y)f(all)h(mem)o(b)q(ers)f (of)g(the)g(group)f(using)i(the)f(same)f(argumen)o(ts)g(for)h Fl(group,)23 b(len)p Fs(,)75 1836 y(and)15 b Fl(root)p Fs(.)75 1931 y Ft(IN)i(in)o(buf)23 b Fs(\014rst)15 b(en)o(try)g(in)h (input)g(bu\013er)f({)g(signi\014can)o(t)h(only)g(at)e(ro)q(ot)h (\(matc)o(hes)f(t)o(yp)q(e\).)75 2025 y Ft(OUT)k(outbuf)24 b Fs(\014rst)14 b(en)o(try)h(in)h(output)f(bu\013er)g(\(matc)o(hes)g(t) o(yp)q(e\).)75 2119 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(to)g(b)q(e)h(stored)e(in)i(output)f(bu\013er)h (\(in)o(teger\))75 2213 y Ft(IN)h(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 2308 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)75 2402 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(sending)i(pro)q(cess)e(\(in)o(teger\))166 2496 y Fl(MPI)p 241 2496 V 17 w(SCATTERC\()22 b(inbuf,)h(outbuf,)g (outlen,)g(type,)g(group,)g(root\))39 b Fs(is)75 2591 y Fl(MPI_GSIZE\()22 b(&size,)i(group\);)75 2647 y(MPI_RANK\()f(&rank,)g (group\);)75 2704 y(MPI_IRECVC\()f(handle,)h(outbuf,)g(outlen,)g(type,) g(root,)g(tag,)h(group\);)p eop %%Page: 53 58 53 57 bop 75 -100 a Fq(3.5.)29 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1082 b Fs(53)75 45 y Fl(if)24 b(\(rank=root\))147 102 y(for)f(\(i=0;)g(i)h(<)g(size;)f(i++\))147 158 y({)218 214 y(MPI_SENDC\(inbuf,)f(outlen,)h(type,)g(i,)h(tag,)f(group\);)218 271 y(inbuf)g(+=)h(outlen;)147 327 y(})75 384 y(MPI_WAIT\(handle\);)75 503 y Ft(All-to-all)d(scatter)75 625 y(MPI)p 184 625 16 2 v 18 w(ALLSCA)l(TTER\()d(list)p 659 625 V 20 w(of)p 721 625 V 19 w(inhandles,)g(list)p 1056 625 V 20 w(of)p 1118 625 V 19 w(outhandles,)g(group)f(\))166 717 y Fs(Eac)o(h)h(pro)q (cess)g(in)h(the)f(group)f(sends)i(its)f Fl(i)p Fs(-th)g(bu\013er)g(in) h(its)f(input)h(bu\013er)f(list)g(to)g(the)g(pro)q(cess)75 773 y(with)i(rank)g Fl(i)g Fs(\(itself)g(included\);)k(eac)o(h)c(pro)q (cess)g(places)h(the)f(incoming)h(messages)e(in)i(the)f(lo)q(cation)75 829 y(sp)q(eci\014ed)h(b)o(y)d(output)h(bu\013er)f(handle)i(corresp)q (onding)f(to)f(the)h(rank)f(of)g(the)h(sender.)31 b(F)l(or)17 b(example,)75 886 y(eac)o(h)d(pro)q(cess)h(places)g(the)f(data)f(from)h (pro)q(cess)g(with)h(rank)f(3)g(in)h(the)f(lo)q(cation)h(sp)q (eci\014ed)h(b)o(y)e(the)g(third)75 942 y(bu\013er)j(descriptor)g(in)h (the)f(list)h(of)e(outbufs.)25 b(The)17 b(routine)g(is)h(called)g(b)o (y)f(all)h(mem)o(b)q(ers)f(of)f(the)h(group)75 999 y(using)f(the)f (same)g(argumen)o(ts)f(for)h Fl(group)p Fs(.)75 1090 y Ft(IN)i(list)p 225 1090 V 20 w(of)p 287 1090 V 19 w(inhandles)24 b Fs(list)16 b(of)f(bu\013er)g(descriptor)g(handles)75 1183 y Ft(IN)i(list)p 225 1183 V 20 w(of)p 287 1183 V 19 w(outhandles)24 b Fs(list)16 b(of)f(bu\013er)g(descriptor)g(handles) 75 1276 y Ft(IN)i(group)23 b Fs(group)15 b(comm)o(unicator)166 1402 y Ft(MPI)p 275 1402 V 18 w(ALLSCA)l(TTER)o(C\()j(in)o(buf,)f (outbuf,)g(len,)h(t)o(yp)q(e,)f(group\))166 1494 y Fl(MPI)p 241 1494 15 2 v 17 w(ALLSCATTERC)g Fs(b)q(eha)o(v)o(es)i(lik)o(e)g Fl(MPI)p 870 1494 V 17 w(ALLSCATTER)e Fs(restricted)i(to)f(blo)q(c)o(k) i(bu\013ers,)f(and)g(with)75 1551 y(the)f(additional)i(restriction)f (that)e(all)i(blo)q(c)o(ks)g(sen)o(t)f(from)f(one)i(pro)q(cess)f(to)f (another)h(ha)o(v)o(e)g(the)g(same)75 1607 y(length.)i(The)14 b(input)g(bu\013er)f(blo)q(c)o(k)i(of)e(eac)o(h)g(pro)q(cess)h(is)g (partitioned)g(in)o(to)f Fl(n)g Fs(consecutiv)o(e)i(blo)q(c)o(ks,)f (eac)o(h)75 1664 y(consisting)20 b(of)g Fl(len)f Fs(w)o(ords.)32 b(The)20 b Fl(i)p Fs(-th)f(blo)q(c)o(k)h(is)g(sen)o(t)g(to)f(the)g Fl(i)p Fs(-th)h(pro)q(cess)f(in)i(the)f(group.)32 b(Eac)o(h)75 1720 y(pro)q(cess)17 b(concatenates)g(the)g(incoming)h(messages,)e(in)i (the)f(order)g(of)f(the)h(senders')g(ranks,)g(and)g(store)75 1776 y(them)c(in)i(its)f(output)f(bu\013er.)19 b(The)14 b(routine)g(is)g(called)h(b)o(y)e(all)i(mem)o(b)q(ers)f(of)f(the)g (group)g(using)i(the)e(same)75 1833 y(argumen)o(ts)h(for)h Fl(group)p Fs(,)f(and)h Fl(len)p Fs(.)75 1924 y Ft(IN)i(in)o(buf)23 b Fs(\014rst)15 b(en)o(try)g(in)h(input)g(bu\013er)f(\(matc)o(hes)g(t)o (yp)q(e\).)75 2017 y Ft(OUT)j(outbuf)24 b Fs(\014rst)14 b(en)o(try)h(in)h(output)f(bu\013er)g(\(matc)o(hes)g(t)o(yp)q(e\).)75 2110 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(sen)o (t)g(from)g(eac)o(h)g(pro)q(cess)g(to)g(eac)o(h)g(other)g(\(in)o (teger\).)75 2203 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 2295 y Ft(IN)j(group)23 b Fs(comm)o(unicator)166 2387 y Fl(MPI)p 241 2387 V 17 w(ALLSCATTERC\()f(inbuf,)h(outbuf,)g(len,)g(type,)g(group\))15 b Fs(is)75 2478 y Fl(MPI_GSIZE\()22 b(group,)i(&size)f(\);)75 2534 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 2591 y(for)g(\(i=0;)h(i)f (<)h(rank;)f(i++\))147 2647 y({)170 2704 y (MPI_IRECVC\(recv_handles[i],)e(outbuf,)i(len,)g(type,)g(tag,)g (group\);)p eop %%Page: 54 59 54 58 bop 75 -100 a Fs(54)708 b Fq(CHAPTER)15 b(3.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 45 y Fl(outbuf)23 b(+=)h(len;)147 102 y(})75 158 y(for)f(\(i=0;)h(i)f(<)h(size;)f(i++\)) 147 214 y({)170 271 y(MPI_ISENDC\(send_handles[i],)e(inbuf,)i(len,)g (type,)g(i,)h(tag,)f(group\);)170 327 y(inbuf)h(+=)f(len;)147 384 y(})75 440 y(MPI_WAITALL\(send_handles,)d(size,)k (return_handles\);)75 497 y(MPI_WAITALL\(recv_handles,)c(size,)k (return_handles\);)75 616 y Ft(All-to-all)d(broadcast)75 738 y(MPI)p 184 738 16 2 v 18 w(ALLCAST\()d(inhandle,)g(list)p 780 738 V 20 w(of)p 842 738 V 19 w(outhandles,)g(group)f(\))166 829 y Fs(Eac)o(h)c(pro)q(cess)h(in)g(the)g(group)f(broadcasts)g(its)h (input)g(bu\013er)g(to)f(all)h(pro)q(cesses)g(\(including)i(itself)t (\);)75 886 y(eac)o(h)e(pro)q(cess)f(places)i(the)f(incoming)g (messages)f(in)i(the)e(lo)q(cation)i(sp)q(eci\014ed)g(b)o(y)f(output)f (bu\013er)h(handle)75 942 y(corresp)q(onding)19 b(to)d(the)i(rank)f(of) h(the)f(sender.)28 b(F)l(or)17 b(example,)h(eac)o(h)g(pro)q(cess)g (places)g(the)g(data)f(from)75 999 y(pro)q(cess)j(with)f(rank)g(3)g(in) h(the)g(lo)q(cation)g(sp)q(eci\014ed)h(b)o(y)e(the)h(third)g(bu\013er)f (descriptor)h(in)g(the)f(list)h(of)75 1055 y(outbufs.)25 b(The)17 b(routine)h(is)f(called)i(b)o(y)e(all)h(mem)o(b)q(ers)f(of)f (the)h(group)g(using)h(the)f(same)f(argumen)o(ts)h(for)75 1112 y Fl(group)p Fs(.)75 1203 y Ft(IN)g(inhandle)24 b Fs(bu\013er)16 b(descriptor)f(handle)i(for)d(input)j(bu\013er)75 1296 y Ft(OUT)h(list)p 280 1296 V 20 w(of)p 342 1296 V 19 w(outhandles)24 b Fs(list)16 b(of)f(bu\013er)g(descriptor)g (handles)75 1389 y Ft(IN)i(group)23 b Fs(handle)166 1515 y Ft(MPI)p 275 1515 V 18 w(ALLCASTC\()17 b(in)o(buf,)h(outbuf,)f(len,)h (t)o(yp)q(e,)e(group\))166 1607 y Fl(MPI)p 241 1607 15 2 v 17 w(ALLCASTC)f Fs(b)q(eha)o(v)o(es)i(lik)o(e)h Fl(MPI)p 793 1607 V 17 w(ALLCAST)e Fs(restricted)h(to)f(blo)q(c)o(k)i (bu\013ers,)f(and)g(with)g(the)g(ad-)75 1664 y(ditional)h(restriction)f (that)e(all)j(blo)q(c)o(ks)f(sen)o(t)f(from)f(one)i(pro)q(cess)g(to)e (another)h(ha)o(v)o(e)g(the)h(same)f(length.)75 1720 y(Eac)o(h)h(pro)q(cess)g(concatenates)f(the)h(incoming)h(messages,)e (in)i(the)f(order)f(of)h(the)f(senders')h(ranks,)g(and)75 1776 y(store)c(them)h(in)g(its)g(output)f(bu\013er.)20 b(The)14 b(routine)g(is)g(called)h(b)o(y)f(all)g(mem)o(b)q(ers)g(of)f (the)h(group)f(using)i(the)75 1833 y(same)g(argumen)o(ts)f(for)h Fl(group)p Fs(,)f(and)h Fl(len)p Fs(.)75 1924 y Ft(IN)i(in)o(buf)23 b Fs(\014rst)15 b(en)o(try)g(in)h(input)g(bu\013er)f(\(matc)o(hes)g(t)o (yp)q(e\).)75 2017 y Ft(OUT)j(outbuf)24 b Fs(\014rst)14 b(en)o(try)h(in)h(output)f(bu\013er)g(\(matc)o(hes)g(t)o(yp)q(e\).)75 2110 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(sen)o (t)g(from)g(eac)o(h)g(pro)q(cess)g(to)g(eac)o(h)g(other)g(\(including)j (itself)t(\).)75 2203 y Ft(IN)f(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 2295 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)166 2387 y Fl(MPI)p 241 2387 V 17 w(ALLCASTC\()22 b(inbuf,)h(outbuf,)g(len,)h(type,)f(group\)) 14 b Fs(is)75 2478 y Fl(MPI_GSIZE\()22 b(group,)i(&size)f(\);)75 2534 y(MPI_RANK\()g(group,)g(&rank)g(\);)75 2591 y(for)g(\(i=0;)h(i)f (<)h(rank;)f(i++\))147 2647 y({)170 2704 y (MPI_IRECVC\(recv_handles[i],)e(outbuf,)i(len,)g(type,)g(tag,)g (group\);)p eop %%Page: 55 60 55 59 bop 75 -100 a Fq(3.6.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b Fs(55)170 45 y Fl(outbuf)23 b(+=)h(len;)147 102 y(})75 158 y(for)f(\(i=0;)h(i)f(<)h(size;)f(i++\))147 214 y({)170 271 y(MPI_ISENDC\(send_handle[i],)e(inbuf,)i(len,)g(type,)g (i,)h(tag,)f(group\);)147 327 y(})75 384 y(MPI_WAITALL\(send_handle\);) 75 440 y(MPI_WAITALL\(recv_handle\);)75 589 y Fm(3.6)70 b(Global)22 b(Compute)g(Op)r(erations)75 692 y Ft(Reduce)75 815 y(MPI)p 184 815 16 2 v 18 w(REDUCE\()d(inhandle,)f(outhandle,)h (group,)d(ro)q(ot,)i(op\))166 908 y Fs(Com)o(bines)f(the)h(v)m(alues)g (pro)o(vided)g(in)g(the)f(input)h(bu\013er)g(of)e(eac)o(h)i(pro)q(cess) f(in)h(the)f(group,)g(using)75 964 y(the)f(op)q(eration)g Fl(op)p Fs(,)g(and)g(returns)f(the)h(com)o(bined)h(v)m(alue)h(in)e(the) g(output)g(bu\013er)g(of)f(the)h(pro)q(cess)h(with)75 1020 y(rank)f Fl(root)p Fs(.)21 b(Eac)o(h)16 b(pro)q(cess)g(can)g(pro)o (vide)g(one)g(v)m(alue,)h(or)f(a)f(sequence)i(of)f(v)m(alues,)h(in)f (whic)o(h)h(case)f(the)75 1077 y(com)o(bine)f(op)q(eration)f(is)h (executed)g(p)q(oin)o(t)o(wise)g(on)f(eac)o(h)g(en)o(try)g(of)g(the)g (sequence.)21 b(F)l(or)13 b(example,)i(if)g(the)75 1133 y(op)q(eration)i(is)g Fl(max)g Fs(and)g(input)g(bu\013ers)g(con)o (tains)g(t)o(w)o(o)f(\015oating)g(p)q(oin)o(t)i(n)o(um)o(b)q(ers,)f (then)g(outbuf\(1\))f(=)75 1190 y(global)e(max\(in)o(buf\(1\)\))e(and)i (outbuf\(2\))e(=)h(global)h(max\(in)o(buf\(2\)\).)19 b(All)14 b(input)g(bu\013ers)g(should)g(de\014ne)75 1246 y(sequences)g(of)f(equal)g(length)h(of)e(en)o(tries)i(of)e(t)o(yp)q(es) h(that)f(matc)o(h)h(the)g(t)o(yp)q(e)g(of)f(the)h(op)q(erands)g(of)g Fl(op)p Fs(.)19 b(The)75 1303 y(output)f(bu\013er)g(should)i(de\014ne)f (a)f(sequence)h(of)f(the)g(same)g(length)h(of)f(en)o(tries)h(of)f(t)o (yp)q(es)g(that)g(matc)o(h)75 1359 y(the)g(t)o(yp)q(e)g(of)g(the)g (result)h(of)e Fl(op)p Fs(.)29 b(\(Note)17 b(that,)h(here)g(as)g(for)g (all)h(other)e(comm)o(unication)i(op)q(erations,)75 1416 y(the)d(t)o(yp)q(e)f(of)g(en)o(tries)h(inserted)g(in)h(a)e(message)g (dep)q(end)i(on)e(the)h(information)g(pro)o(vided)g(b)o(y)f(the)h (input)75 1472 y(bu\013er)d(descriptor,)g(and)h(not)e(on)h(the)g (declarations)h(of)f(these)g(v)m(ariables)h(in)g(the)f(calling)i (program.)j(The)75 1529 y(t)o(yp)q(es)13 b(of)g(the)g(v)m(ariables)i (in)f(the)f(calling)i(program)d(need)i(not)f(matc)o(h)g(the)g(t)o(yp)q (es)g(de\014ned)i(b)o(y)e(the)g(bu\013er)75 1585 y(descriptor,)22 b(but)f(in)g(suc)o(h)g(case)g(the)g(outcome)f(of)h(a)f(reduce)h(op)q (eration)g(ma)o(y)f(b)q(e)i(implemen)o(tation)75 1641 y(dep)q(enden)o(t.\))166 1699 y(The)16 b(op)q(eration)g(de\014ned)i(b)o (y)e Fl(op)f Fs(is)i(asso)q(ciativ)o(e)f(and)g(comm)o(utativ)o(e,)f (and)i(the)f(implemen)o(tation)75 1755 y(can)i(tak)o(e)f(adv)m(an)o (tage)g(of)g(asso)q(ciativit)o(y)h(and)g(comm)o(utativit)o(y)f(in)h (order)f(to)g(c)o(hange)h(order)f(of)h(ev)m(alu-)75 1812 y(ation.)29 b(The)18 b(routine)h(is)g(called)h(b)o(y)e(all)h(group)f (mem)o(b)q(ers)h(using)g(the)f(same)g(argumen)o(ts)f(for)h Fl(group,)75 1868 y(root)d Fs(and)g Fl(op)p Fs(.)75 1966 y Ft(IN)i(inhandle)24 b Fs(handle)17 b(to)d(input)j(bu\013er)75 2063 y Ft(IN)g(outhandle)24 b Fs(handle)17 b(to)d(output)h(bu\013er)h ({)e(signi\014can)o(t)j(only)e(at)g(ro)q(ot)75 2160 y Ft(IN)i(group)23 b Fs(group)15 b(comm)o(unicator)75 2258 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(ro)q(ot)g(pro)q(cess)g(\(in)o (teger\))75 2355 y Ft(IN)i(op)23 b Fs(op)q(eration)166 2452 y(The)18 b(bu\013er)g(descriptor)g(con)o(tains)g(data)g(t)o(yp)q (e)g(information)g(so)f(that)h(the)g(correct)f(form)g(of)h(the)75 2509 y(op)q(eration)d(can)h(b)q(e)g(p)q(erformed.)k(W)l(e)15 b(list)h(b)q(elo)o(w)g(the)f(op)q(erations)g(whic)o(h)h(are)f(supp)q (orted.)75 2606 y Ft(MPI)p 184 2606 V 18 w(MAX)21 b Fs(maxim)o(um)75 2704 y Ft(MPI)p 184 2704 V 18 w(MIN)h Fs(minim)o(um)p eop %%Page: 56 61 56 60 bop 75 -100 a Fs(56)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(MPI)p 184 45 16 2 v 18 w(SUM)23 b Fs(sum)75 138 y Ft(MPI)p 184 138 V 18 w(PR)o(OD)g Fs(pro)q(duct)75 232 y Ft(MPI)p 184 232 V 18 w(AND)f Fs(and)16 b(\(logical)g(or)e(bit-wise)j(in)o (teger\))75 325 y Ft(MPI)p 184 325 V 18 w(OR)23 b Fs(or)15 b(\(logical)h(or)f(bit-wise)h(in)o(teger\))75 419 y Ft(MPI)p 184 419 V 18 w(X)o(OR)22 b Fs(xor)15 b(\(logical)h(or)f(bit-wise)h(in)o (teger\))75 512 y Ft(MPI)p 184 512 V 18 w(MAXLOC)22 b Fs(maxim)o(um)15 b(v)m(alue)h(and)g(rank)f(of)f(pro)q(cess)i(with)f (maxim)o(um)h(v)m(alue)75 605 y Ft(MPI)p 184 605 V 18 w(MINLOC)23 b Fs(minim)o(um)16 b(v)m(alue)g(and)g(rank)f(of)f(pro)q (cess)i(with)f(minim)o(um)i(v)m(alue)166 733 y Ft(MPI)p 275 733 V 18 w(REDUCEC\()h(in)o(buf,)f(outbuf,)h(len,)f(t)o(yp)q(e,)g (group,)g(ro)q(ot,)h(op\))166 825 y Fs(Is)d(same)g(as)g Fl(MPI)p 460 825 15 2 v 17 w(REDUCE)p Fs(,)f(restricted)h(to)g(a)g(blo) q(c)o(k)h(bu\013er.)75 918 y Ft(IN)h(in)o(buf)23 b Fs(\014rst)15 b(lo)q(cation)h(in)g(input)g(bu\013er)75 1011 y Ft(OUT)i(outbuf)24 b Fs(\014rst)14 b(lo)q(cation)i(in)g(output)f(bu\013er)h({)e (signi\014can)o(t)j(only)e(at)g(ro)q(ot)75 1105 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(and)e(output)g (bu\013er)g(\(in)o(teger\))75 1198 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 1291 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)75 1385 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(ro)q(ot)g(pro)q(cess)g(\(in)o(teger\))75 1478 y Ft(IN)i(op)23 b Fs(op)q(eration)166 1606 y Ft(MPI)p 275 1606 16 2 v 18 w(USER)p 435 1606 V 20 w(REDUCE\()j(inhandle,)h (outhandle,)h(group,)e(ro)q(ot,)h(function,)h(unit-)75 1663 y(size\))166 1755 y Fs(Same)18 b(as)f(the)i(reduce)f(op)q(eration) h(function)f(ab)q(o)o(v)o(e)g(except)g(that)g(a)f(user)h(supplied)j (function)d(is)75 1811 y(used.)39 b Fl(function)20 b Fs(is)i(an)f(asso)q(ciativ)o(e)h(and)g(comm)o(utativ)o(e)e(function)j (with)e(three)h(argumen)o(ts.)37 b Fl(f\()75 1868 y(invec,)23 b(inoutvec,)g(*len\))75 1960 y Ft(IN)17 b(inhandle)24 b Fs(handle)17 b(to)d(input)j(bu\013er)75 2054 y Ft(IN)g(outhandle)24 b Fs(handle)17 b(to)d(output)h(bu\013er)h({)e(signi\014can)o(t)j(only)e (at)g(ro)q(ot)75 2147 y Ft(IN)i(group)23 b Fs(group)15 b(comm)o(unicator)75 2240 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(ro)q(ot)g(pro)q(cess)g(\(in)o(teger\))75 2334 y Ft(IN)i(function)24 b Fs(user)16 b(de\014ned)g(function)75 2427 y Ft(IN)h(unitsize)24 b Fs(minim)o(um)16 b(size)h(of)d(data)h (that)f(function)i(requires.)166 2555 y Ft(MPI)p 275 2555 V 18 w(USER)p 435 2555 V 20 w(REDUCEC\()26 b(in)o(buf,)h(outbuf,)g (len,)g(t)o(yp)q(e,)g(group,)g(ro)q(ot,)g(function,)75 2612 y(unitsize\))166 2704 y Fs(Is)15 b(same)g(as)g Fl(MPI)p 460 2704 15 2 v 477 2704 V 34 w(USER)p 590 2704 V 17 w(REDUCE)p Fs(,)f(restricted)h(to)g(a)g(blo)q(c)o(k)h(bu\013er.)p eop %%Page: 57 62 57 61 bop 75 -100 a Fq(3.6.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b Fs(57)75 45 y Ft(IN)17 b(in)o(buf)23 b Fs(\014rst)15 b(lo)q(cation)h(in)g(input)g(bu\013er)75 142 y Ft(OUT)i(outbuf)24 b Fs(\014rst)14 b(lo)q(cation)i(in)g(output)f(bu\013er)h({)e (signi\014can)o(t)j(only)e(at)g(ro)q(ot)75 239 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(and)e(output)g (bu\013er)g(\(in)o(teger\))75 336 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 434 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)75 531 y Ft(IN)i(ro)q(ot)23 b Fs(rank)15 b(of)g(ro)q(ot)g(pro)q(cess)g(\(in)o(teger\))75 628 y Ft(IN)i(function)24 b Fs(user)16 b(pro)o(vided)g(function)75 725 y Ft(IN)h(unitsize)24 b Fs(minim)o(um)16 b(size)h(of)d(data)h(that) f(function)i(requires.)166 822 y(MPI)g(also)f(includes)j(v)m(arian)o (ts)e(of)f(eac)o(h)h(of)f(the)h(reduce)g(op)q(erations)g(where)g(the)g (result)g(is)g(kno)o(wn)75 879 y(to)f(all)h(pro)q(cesses)f(in)h(the)g (group)f(on)g(return.)166 971 y Ft(MPI)p 275 971 16 2 v 18 w(ALLREDUCE\()k(inhandle,)g(outhandle,)f(group,)f(op\))166 1064 y Fs(Same)j(as)h(the)f(reduce)h(op)q(eration)g(function)h(ab)q(o)o (v)o(e)e(except)h(that)e(the)i(result)g(app)q(ears)g(in)g(the)75 1120 y(outbuf)15 b(of)g(all)h(the)f(group)g(mem)o(b)q(ers.)166 1213 y Ft(MPI)p 275 1213 V 18 w(ALLREDUCEC\()k(in)o(buf,)e(outbuf,)g (len,)h(t)o(yp)q(e,)f(group,)g(op\))166 1306 y Fs(Same)i(as)f(the)h (reducec)h(op)q(eration)g(function)f(ab)q(o)o(v)o(e)g(except)g(that)g (the)g(result)g(app)q(ears)g(in)h(the)75 1362 y(outbuf)15 b(of)g(all)h(the)f(group)g(mem)o(b)q(ers.)166 1455 y Ft(MPI)p 275 1455 V 18 w(USER)p 435 1455 V 20 w(ALLREDUCE\()k (inhandle,)f(outhandle,)h(group,)d(function\))166 1548 y Fs(Same)d(as)g(the)g(user)p 499 1548 14 2 v 16 w(reduce)h(op)q (eration)f(function)h(ab)q(o)o(v)o(e)f(except)g(that)f(the)h(result)h (app)q(ears)f(in)h(the)75 1604 y(outbuf)h(of)g(all)h(the)f(group)g(mem) o(b)q(ers.)166 1697 y Ft(MPI)p 275 1697 16 2 v 18 w(USER)p 435 1697 V 20 w(ALLREDUCEC\()j(in)o(buf,)f(outbuf,)h(len,)f(t)o(yp)q (e,)g(group,)g(function\))166 1790 y Fs(Same)h(as)f(the)h(user)p 513 1790 14 2 v 16 w(reducec)h(op)q(eration)f(function)g(ab)q(o)o(v)o (e)f(except)i(that)e(the)g(result)h(app)q(ears)g(in)75 1846 y(the)d(outbuf)g(of)g(all)h(the)g(group)f(mem)o(b)q(ers.)75 1971 y Ft(Scan)75 2094 y(MPI)p 184 2094 16 2 v 18 w(SCAN\()i(inhandle,) h(outhandle,)h(group,)e(op)g(\))166 2186 y Fs(MPI)p 258 2186 14 2 v 16 w(SCAN)c(is)g(used)g(to)f(p)q(erform)h(a)f(parallel)i (pre\014x)f(with)g(resp)q(ect)h(to)d(an)i(asso)q(ciativ)o(e)g (reduction)75 2243 y(op)q(eration)h(on)h(data)e(distributed)j(across)e (the)g(group.)19 b(The)c(op)q(eration)f(returns)g(in)i(the)e(output)g (bu\013er)75 2299 y(of)g(the)g(pro)q(cess)g(with)h(rank)e Fl(i)h Fs(the)g(reduction)h(of)f(the)g(v)m(alues)h(in)g(the)f(input)i (bu\013ers)e(of)f(pro)q(cesses)i(with)75 2356 y(ranks)h Fl(0,...,i)p Fs(.)21 b(The)c(t)o(yp)q(e)f(of)g(op)q(erations)g(supp)q (orted)h(and)f(their)h(seman)o(tic,)f(and)g(the)g(constrain)o(ts)75 2412 y(on)f(input)h(and)g(output)f(bu\013ers)g(are)g(as)g(for)f Fl(MPI)p 921 2412 15 2 v 17 w(REDUCE)p Fs(.)75 2509 y Ft(IN)j(inhandle)24 b Fs(handle)17 b(to)d(input)j(bu\013er)75 2606 y Ft(IN)g(outhandle)24 b Fs(handle)17 b(to)d(output)h(bu\013er)75 2704 y Ft(IN)i(group)23 b Fs(group)15 b(comm)o(unicator)p eop %%Page: 58 63 58 62 bop 75 -100 a Fs(58)703 b Fq(CHAPTER)15 b(3.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(IN)h(op)23 b Fs(op)q(eration)166 174 y Ft(MPI)p 275 174 16 2 v 18 w(SCANC\()17 b(in)o(buf,)g(outbuf,)g(len,)h(t)o(yp)q(e,)f(group,)f(op)i (\))166 266 y Fs(Same)d(as)g Fl(MPI)p 418 266 15 2 v 17 w(SCAN)p Fs(,)f(restricted)h(to)g(blo)q(c)o(k)h(bu\013ers.)75 360 y Ft(IN)h(in)o(buf)23 b Fs(\014rst)15 b(input)h(bu\013er)f(elemen)o (t)h(\(matc)o(hes)f(t)o(yp)q(e\))75 454 y Ft(OUT)j(outbuf)24 b Fs(\014rst)14 b(output)h(bu\013er)h(elemen)o(t)f(\(matc)o(hes)g(t)o (yp)q(e\))75 548 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(and)e(output)g(bu\013er)g(\(in)o (teger\))75 641 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 735 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)75 829 y Ft(IN)i(op)23 b Fs(op)q(eration)166 958 y Ft(MPI)p 275 958 16 2 v 18 w(USER)p 435 958 V 20 w(SCAN\()16 b(inhandle,)j(outhandle,)f(group,)f (function)i(\))166 1050 y Fs(Same)12 b(as)h(the)f(scan)h(op)q(eration)f (function)i(ab)q(o)o(v)o(e)e(except)h(that)f(a)g(user)h(supplied)h (function)g(is)f(used.)75 1107 y Fl(function)f Fs(is)h(an)f(asso)q (ciativ)o(e)h(and)g(comm)o(utativ)o(e)f(function)i(with)f(an)f(input)i (v)o(ector,)e(an)h(inout)g(v)o(ector,)75 1163 y(and)i(a)g(length)h (argumen)o(t.)j(The)d(t)o(yp)q(es)f(of)g(the)g(t)o(w)o(o)f(v)o(ectors)g (and)i(of)e(the)i(returned)f(v)m(alues)i(all)f(agree.)75 1257 y Ft(IN)h(inhandle)24 b Fs(handle)17 b(to)d(input)j(bu\013er)75 1351 y Ft(IN)g(outhandle)24 b Fs(handle)17 b(to)d(output)h(bu\013er)75 1445 y Ft(IN)i(group)23 b Fs(group)15 b(comm)o(unicator)75 1538 y Ft(IN)i(function)24 b Fs(user)16 b(pro)o(vided)g(function)166 1668 y Ft(MPI)p 275 1668 V 18 w(USER)p 435 1668 V 20 w(SCANC\()g(in)o(buf,)h(outbuf,)h(len,)f(t)o(yp)q(e,)g(group,)g (function\))166 1760 y Fs(Is)e(same)g(as)g Fl(MPI)p 460 1760 15 2 v 17 w(USER)p 573 1760 V 17 w(SCAN)p Fs(,)f(restricted)h(to)g (a)g(blo)q(c)o(k)h(bu\013er.)75 1853 y Ft(IN)h(in)o(buf)23 b Fs(\014rst)15 b(lo)q(cation)h(in)g(input)g(bu\013er)75 1947 y Ft(OUT)i(outbuf)24 b Fs(\014rst)14 b(lo)q(cation)i(in)g(output)f (bu\013er)75 2041 y Ft(IN)i(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(input)h(and)e(output)g(bu\013er)g(\(in)o (teger\))75 2135 y Ft(IN)i(t)o(yp)q(e)23 b Fs(data)14 b(t)o(yp)q(e)i(of)e(bu\013er)75 2229 y Ft(IN)j(group)23 b Fs(group)15 b(comm)o(unicator)75 2322 y Ft(IN)i(function)24 b Fs(user)16 b(pro)o(vided)g(function)p eop %%Page: 59 64 59 63 bop 75 367 a Fn(Chapter)34 b(4)75 586 y Fu(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)850 819 y Fs(Rolf)16 b(Hemp)q(el)75 1024 y Fm(4.1)70 b(In)n(tro)r(duction)75 1147 y Fs(A)14 b(pro)q(cess)f(group)g(in)i(MPI)e(is)h(a)f(collection)j(of)d(n)h(pro)q (cesses.)19 b(Eac)o(h)14 b(pro)q(cess)f(in)i(the)e(group)g(is)h (assigned)75 1203 y(a)g(rank)g(b)q(et)o(w)o(een)h(0)g(and)f(n-1.)20 b(In)15 b(the)g(ma)s(jorit)o(y)e(of)h(parallel)i(applications)g(this)f (linear)h(ordering)f(do)q(es)75 1260 y(not)e(re\015ect)g(the)g(comm)o (unication)h(structure)e(of)h(the)g(pro)q(cesses)g(whic)o(h)h(is)f (usually)i(determined)f(b)o(y)f(the)75 1316 y(underlying)20 b(problem)e(geometry)f(and)h(the)f(n)o(umerical)i(algorithm)f(used.)28 b(Often)18 b(the)f(pro)q(cesses)h(are)75 1373 y(arranged)d(in)h(top)q (ological)g(patterns)e(as)h(simple)i(as)e(a)g(t)o(w)o(o{)e(or)i (three{dimensional)i(grid.)166 1440 y(F)l(or)12 b(the)h(user)g(who)g (programs)f(an)h(application)h(on)f(suc)o(h)g(a)g(pro)q(cess)g (structure)g(the)g(most)f(natural)75 1496 y(w)o(a)o(y)k(of)g(referring) i(to)e(individual)k(pro)q(cesses)d(is)h(to)e(sp)q(ecify)i(their)g(co)q (ordinates)f(in)h(the)f(pro)q(cess)g(grid.)75 1553 y(Usually)22 b(a)e(pro)q(cess)g(do)q(es)h(not)f(ha)o(v)o(e)g(to)g(kno)o(w)g(the)g (PID)h(n)o(um)o(b)q(ers)f(of)g Fh(al)r(l)g Fs(pro)q(cesses)h(in)g(the)g (grid,)75 1609 y(b)q(ecause)h(comm)o(unication)g(only)g(tak)o(es)f (place)h(with)g(pro)q(cesses)f(in)h(a)f(certain)h(neigh)o(b)q(orho)q(o) q(d,)i(and)75 1666 y(p)q(erhaps)18 b(with)g(a)f(few)h(distan)o(t)f(pro) q(cesses.)27 b(Non{scalable)19 b(features)e(lik)o(e)i(the)e(explicit)j (listing)f(of)e(all)75 1722 y(participating)c(pro)q(cesses)g(in)g(the)g (application)h(could,)g(therefore,)e(b)q(e)h(a)o(v)o(oided,)g(if)f (there)h(w)o(ere)f(a)g(means)75 1779 y(to)j(sp)q(ecify)h(the)f (information)h(whic)o(h)g(is)f(really)i(required.)166 1846 y(Additionally)l(,)g(the)e(user{de\014ned)h(pro)q(cess)f (structure)g(can)g(b)q(e)h(exploited)g(b)o(y)f(the)g(system)f(in)i(the) 75 1902 y(assignmen)o(t)d(of)g(pro)q(cesses)h(to)f(hardw)o(are)f(pro)q (cessors,)h(if)h(this)g(helps)h(to)d(impro)o(v)o(e)i(the)f(comm)o (unication)75 1959 y(p)q(erformance)g(on)h(a)f(giv)o(en)h(mac)o(hine.) 20 b(Ho)o(w)12 b(this)i(mapping)g(is)g(done)g(in)g(detail)g(is)g (outside)g(the)g(scop)q(e)f(of)75 2015 y(this)k(standardization)h(prop) q(osal,)f(since)h(it)f(is)h(to)q(o)e(m)o(uc)o(h)h(dep)q(enden)o(t)i(on) d(the)h(particular)h(hardw)o(are.)75 2072 y(The)d(description)i(of)d(a) h(\\virtual)g(pro)q(cess)g(top)q(ology",)f(ho)o(w)o(ev)o(er,)g(only)h (dep)q(ends)i(on)e(the)g(application,)75 2128 y(and)k(is,)g(therefore,) g(mac)o(hine{indep)q(endent.)32 b(The)19 b(functions)g(whic)o(h)h(are)e (prop)q(osed)h(in)g(this)g(pap)q(er)75 2185 y(only)d(deal)g(with)f (this)h(side)g(of)f(the)g(mapping)h(problem.)166 2252 y(There)e(are)g(w)o(ell{kno)o(wn)h(tec)o(hniques)g(for)f(mapping)h (grid/torus)e(structures)h(to)g(hardw)o(are)f(top)q(o-)75 2308 y(logies)18 b(suc)o(h)f(as)g(h)o(yp)q(ercub)q(es)h(or)f(grids.)26 b(But)17 b(ev)o(en)g(for)g(more)f(complicated)j(graph)d(structures)h (go)q(o)q(d)75 2365 y(heuristics)i(often)e(yield)i(near{optimal)e (results)h([14)o(].)26 b(On)18 b(the)f(other)g(hand,)h(if)g(there)f(is) h(no)f(w)o(a)o(y)g(for)75 2421 y(the)c(user)g(to)f(sp)q(ecify)j(the)e (logical)h(pro)q(cess)f(arrangemen)o(t)f(as)g(a)h(\\virtual)g(top)q (ology",)f(this)i(information)75 2478 y(is)k(most)f(di\016cult)j(to)d (reco)o(v)o(er)g(later.)28 b(A)18 b(random)g(mapping)g(is)g(most)g(lik) o(ely)h(the)f(consequence.)29 b(On)75 2534 y(some)15 b(mac)o(hines)h(this)g(will)h(lead)f(to)f(unnecessary)h(con)o(ten)o (tion)g(in)g(the)f(in)o(terconnect)i(net)o(w)o(ork.)i(Some)75 2591 y(details)g(ab)q(out)e(predicted)i(and)f(measured)g(p)q (erformance)f(impro)o(v)o(emen)o(ts)h(whic)o(h)g(result)g(from)f(go)q (o)q(d)75 2647 y(pro)q(cess{to{pro)q(cessor)e(mapping)h(on)f(mo)q(dern) h(w)o(ormhole{routing)f(arc)o(hitectures)h(can)g(b)q(e)g(found)g(in)75 2704 y([5)o(,)f(4].)952 2828 y(59)p eop %%Page: 60 65 60 64 bop 75 -100 a Fs(60)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fm(4.2)70 b(Virtual)21 b(T)-6 b(op)r(ologies)75 147 y Fs(In)17 b(the)g(most)e(general)i(case)g (the)f(connection)i(structure)e(of)g(a)g(set)h(of)f(pro)q(cesses)h(can) f(b)q(e)h(represen)o(ted)75 203 y(b)o(y)c(a)g(graph.)19 b(The)14 b(no)q(des)f(stand)h(for)e(the)i(pro)q(cesses,)f(and)h(the)f (edges)h(connect)f(pro)q(cesses)h(whic)o(h)g(com-)75 260 y(m)o(unicate)h(with)g(eac)o(h)g(other.)20 b(MPI)14 b(pro)o(vides)i(message{passing)e(b)q(et)o(w)o(een)h(an)o(y)g(pair)g (of)f(pro)q(cesses)h(in)75 316 y(a)i(group.)27 b(There)18 b(is)g(no)g(requiremen)o(t)g(for)f(op)q(ening)i(a)e(c)o(hannel)i (explicitly)l(.)30 b(Therefore,)18 b(a)f(\\missing)75 373 y(link")g(in)f(the)f(user{de\014ned)i(pro)q(cess)f(graph)f(do)q(es) h(not)f(prev)o(en)o(t)g(the)g(corresp)q(onding)i(pro)q(cesses)f(from)75 429 y(exc)o(hanging)h(messages.)k(It)16 b(only)g(means)g(that)f(this)i (connection)g(is)f(neglected)h(in)g(the)f(virtual)h(top)q(ol-)75 486 y(ogy)l(.)24 b(As)16 b(a)g(consequence,)i(an)f(automatic)f(mapping) h(to)q(ol)g(will)h(not)e(try)g(to)g(assign)h(the)g(pro)q(cesses)g(to)75 542 y(neigh)o(b)q(oring)g(pro)q(cessors.)166 599 y(The)g(graph)g (de\014nition)i(function)f(MPI)p 868 599 14 2 v 16 w(GRAPH)f(presen)o (ted)h(in)g(this)f(pap)q(er)h(do)q(es)f(not)g(include)75 655 y(an)o(y)e(w)o(eigh)o(ting)g(of)f(the)h(comm)o(unication)h(v)o (olume)f(along)g(the)g(di\013eren)o(t)g(edges.)20 b(Pro)q(cesses)15 b(are)f(either)75 711 y(neigh)o(b)q(ors)22 b(or)e(not.)37 b(Exp)q(erience)23 b(with)f(similar)g(tec)o(hniques)g(in)g(P)l(ARMA)o (CS)f([11)o(,)g(12])f(sho)o(w)h(that)75 768 y(this)h(information)g(is)g (usually)h(su\016cien)o(t)g(for)e(a)g(go)q(o)q(d)h(mapping.)40 b(Additionally)l(,)25 b(a)d(more)f(precise)75 824 y(sp)q(eci\014cation) 16 b(w)o(ould)f(b)q(e)g(more)e(di\016cult)j(for)e(the)g(user)g(to)g (set)g(up,)g(and)h(it)f(w)o(ould)h(mak)o(e)e(the)i(in)o(terface)75 881 y(functions)h(more)f(complicated.)166 937 y(In)i(principle)i(a)c (function)i(for)f(the)g(de\014nition)i(of)e(suc)o(h)g(a)g(graph)g (structure)g(w)o(ould)g(b)q(e)h(su\016cien)o(t)75 994 y(for)c(all)i(applications.)21 b(Ho)o(w)o(ev)o(er,)13 b(in)h(man)o(y)g(applications)h(the)f(graph)g(structure)f(is)h(v)o(ery) g(regular,)g(and)75 1050 y(the)e(detailed)i(set{up)f(of)e(the)i(graph)f (w)o(ould)h(b)q(e)g(incon)o(v)o(enien)o(t)g(for)f(the)g(user)h(and)f (resource{consuming)75 1107 y(during)g(run)g(time.)19 b(A)12 b(large)f(fraction)g(of)g(all)i(parallel)g(applications)g(use)f (pro)q(cess)g(top)q(ologies)f(lik)o(e)i(rings,)75 1163 y(or)g(t)o(w)o(o{)f(or)h(higher{dimensional)k(grids)c(or)h(tori.)19 b(These)14 b(structures)f(are)g(completely)i(de\014ned)g(b)o(y)f(the)75 1220 y(n)o(um)o(b)q(er)g(of)g(dimensions)i(and)e(the)g(n)o(um)o(b)q (ers)h(of)f(pro)q(cesses)g(in)h(eac)o(h)f(co)q(ordinate)h(direction.)21 b(Also,)14 b(the)75 1276 y(mapping)h(of)e(grids)h(and)h(tori)e(is)i (generally)g(an)f(easier)g(problem)h(then)f(that)f(of)h(general)g (graphs.)19 b(Th)o(us,)75 1333 y(it)14 b(is)h(desirable)h(to)d(do)h (the)h(mapping)g(as)e(grids)i(and)f(tori)g(rather)g(than)g(obscuring)h (the)f(information)g(in)75 1389 y(a)h(graph.)166 1446 y(Therefore,)f(in)h(addition)g(to)e(the)i(general)f(graph)g(cartesian)g (structures)g(of)g(arbitrary)f(dimension)75 1502 y(are)j(supp)q(orted.) 22 b(The)16 b(top)q(ology)g(de\014nition)i(function)e(MPI)p 1140 1502 V 17 w(CAR)l(T)g(can)g(b)q(e)h(used)f(to)f(describ)q(e)j(b)q (oth)75 1559 y(grids)11 b(and)f(tori)g(b)o(y)g(sp)q(ecifying)j(p)q(er)d (co)q(ordinate)h(direction)h(whether)e(the)g(pro)q(cess)h(structure)f (is)h(p)q(erio)q(dic)75 1615 y(or)i(not.)20 b(Similar)15 b(functions)g(are)e(con)o(tained)i(in)g(EXPRESS)g([15)o(])e(and)h(P)l (ARMA)o(CS.)g(Note)g(that)f(an)h(n{)75 1671 y(dimensional)h(h)o(yp)q (ercub)q(e)g(is)f(an)f(n{dimensional)i(torus)e(with)g(2)g(pro)q(cesses) h(p)q(er)f(co)q(ordinate)h(direction.)75 1728 y(Th)o(us,)h(sp)q(ecial)i (supp)q(ort)e(for)g(h)o(yp)q(ercub)q(e)h(structures)f(is)h(not)f (necessary)l(.)166 1784 y(Based)g(on)f(the)h(p)q(ositions)g(in)g(the)g (virtual)g(top)q(ology)f(a)g(pro)q(cess)h(can)g(lo)q(ok)g(up)g(the)f (PID)h(n)o(um)o(b)q(ers)75 1841 y(of)21 b(other)g(pro)q(cesses)i(b)o(y) e(calling)i(MPI)p 773 1841 V 17 w(INQPID.)f(The)g(pro)q(cesses)g(are)f (sp)q(eci\014ed)j(b)o(y)e(their)g(logical)75 1897 y(no)q(de)16 b(n)o(um)o(b)q(ers)f(\(graph\))g(or)g(rectangular)g(pro)q(cess)g(co)q (ordinates)h(\(grid/torus\).)j(Th)o(us,)c(it)g(is)h(p)q(ossible)75 1954 y(to)j(inquire)i(for)e(the)h(PID)g(n)o(um)o(b)q(ers)g(of)f(only)h (those)g(\(few\))f(pro)q(cesses)h(with)g(whic)o(h)g(messages)f(ha)o(v)o (e)75 2010 y(to)h(b)q(e)h(exc)o(hanged.)37 b(The)21 b(in)o(v)o(erse)g (function)g(MPI)p 987 2010 V 16 w(INQLOC)h(translates)e(a)h(pid)g(n)o (um)o(b)q(er)g(in)o(to)g(the)75 2067 y(corresp)q(onding)16 b(cartesian)f(co)q(ordinates.)166 2206 y Fk(Discussion:)g Fj(In)e(the)h(PT2PT)g(discussion)g(it)f(is)g(not)g(y)o(et)h(decided)g (ho)o(w)f(pro)q(cesses)j(are)d(addressed.)20 b(This)75 2262 y(could,)10 b(for)g(example,)f(b)q(e)i(done)f(b)o(y)g(unique)g (iden)o(ti\014ers,)h(or)f(b)o(y)g(their)g(ranks)h(in)e(a)h(group.)17 b(The)10 b(top)q(ology)f(handling)75 2319 y(will)j(b)q(e)j(adapted)f (to)g(whatev)o(er)g(decision)h(will)d(b)q(e)i(tak)o(en)g(in)g(PT2PT.)75 2545 y Fm(4.3)70 b(Em)n(b)r(edding)22 b(in)g(MPI)75 2647 y Fs(The)17 b(supp)q(ort)g(for)f(virtual)i(top)q(ologies)f(has)f(to)h (b)q(e)g(fully)h(in)o(tegrated)f(in)g(the)g(framew)o(ork)f(of)g(MPI.)g (In)75 2704 y(particular,)21 b(the)f(virtual)g(top)q(ology)f(functions) i(do)e(not)h(in)o(terfere)g(with)g(the)f(functions)i(for)e(pro)q(cess)p eop %%Page: 61 66 61 65 bop 75 -100 a Fq(4.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b Fs(61)75 45 y(group)21 b(creation)g(and)g(with)g(those)f(for)h(collectiv)o(e)h (comm)o(unication.)38 b(The)21 b(curren)o(t)g(design)g(of)g(the)75 102 y(pro)q(cess)15 b(top)q(ology)g(functions)h(is)g(based)g(on)f(the)g (follo)o(wing)h(assumptions:)143 197 y Fo(\017)23 b Fs(MPI)15 b(is)h(free)g(to)f(c)o(ho)q(ose)h(an)o(y)f(pro)q(cess)h(in)h(a)e(group) g(for)g(an)o(y)h(lo)q(cation)g(in)h(the)f(top)q(ology)l(.)21 b(There)189 254 y(are)16 b(no)g(sp)q(ecial)i(pro)q(cesses)f(whic)o(h)g (ha)o(v)o(e)f(to)g(b)q(e)h(aligned)g(with)g(top)q(ology)f (substructures.)23 b(This)189 310 y(assumption)d(\014ts)g(b)q(est)h(to) e(a)h(\\lo)q(cal)h(SPMD)f(mo)q(del")h(in)g(whic)o(h)g(all)g(pro)q (cesses)g(in)g(the)f(same)189 366 y(group)15 b(execute)g(the)h(same)f (program.)143 462 y Fo(\017)23 b Fs(One)g(imp)q(ortan)o(t)f(feature)g (of)g(MPI)g(are)g(the)h(pro)q(cess)f(groups.)42 b(The)22 b(treatmen)o(t)f(of)h(virtual)189 519 y(top)q(ologies)f(has)g(to)g(tak) o(e)g(this)g(concept)h(in)o(to)f(accoun)o(t.)38 b(The)22 b(functions)g(presen)o(ted)f(in)i(this)189 575 y(pap)q(er)16 b(are)g(not)g(only)h(compatible)h(with)e(the)h(pro)q(cess)f(groups,)g (they)g(ev)o(en)h(tak)o(e)f(adv)m(an)o(tage)f(of)189 631 y(them)e(for)g(the)g(de\014nition)i(of)e(mapping)h(o)o(v)o(erla)o (ys)f(and)g(dynamic)i(remapping)f(during)g(execution)189 688 y(time.)42 b(Virtual)23 b(pro)q(cess)g(top)q(ologies)g(help)h(in)f (the)g(sub)q(division)i(of)d(pro)q(cess)h(groups)f(whic)o(h)189 744 y(in)f(man)o(y)e(applications)j(has)e(to)f(b)q(e)i(done)g(along)f (pro)q(cess)g(co)q(ordinate)h(lines.)36 b(F)l(urthermore,)189 801 y(the)14 b(de\014nition)i(of)e(collectiv)o(e)i(gather{t)o(yp)q(e)d (comm)o(unication)i(calls)g(b)q(ene\014ts)g(from)f(the)g(natural)189 857 y(ordering)h(of)g(pro)q(cesses)h(in)g(a)e(logical)j(top)q(ology)l (.)75 953 y(The)g(virtual)g(top)q(ology)g(handling)h(follo)o(ws)f(the)g (general)g(MPI)g(rule)g(of)g(functional)h(la)o(y)o(ering.)25 b(A)o(t)16 b(the)75 1009 y(b)q(ottom)f(it)g(is)h(based)g(on)f(the)h (general)g(pro)q(cess)g(group)f(creation)g(function)i(MPI)p 1486 1009 14 2 v 16 w(CREA)l(TE)p 1693 1009 V 17 w(GR)o(OUP)75 1066 y(plus)i(a)f(few)g(top)q(ology)g(sp)q(eci\014c)i(mapping)e (functions.)30 b(In)18 b(principle)j(this)e(is)f(su\016cien)o(t)h(for)e (de\014ning)75 1122 y(top)q(ologies)f(and)f(use)h(them)f(in)h (applications.)166 1179 y(A)f(n)o(um)o(b)q(er)g(of)f(higher{lev)o(el)j (functions)e(pro)o(vide)g(an)g(in)o(terface)g(whic)o(h)h(is)f(similar)h (to)e(what)g(users)75 1236 y(of)19 b(pac)o(k)m(ages)g(lik)o(e)h (Express)f(and)g(P)l(ARMA)o(CS)h(are)f(familiar)h(with.)32 b(This)19 b(lev)o(el)i(can)e(b)q(e)h(written)f(in)75 1292 y(a)j(p)q(ortable)h(w)o(a)o(y)e(in)i(terms)f(of)g(the)g(lo)o (w{lev)o(el)h(functions.)42 b(Implemen)o(tations)24 b(of)e(the)g (high{lev)o(el)75 1348 y(functions)e(whic)o(h)h(b)o(ypass)e(the)g(lo)o (w)h(lev)o(el)h(ma)o(y)d(b)q(e)j(adv)m(an)o(tageous)d(on)i(some)f (systems)g(in)h(terms)f(of)75 1405 y(memory)c(and)g(cpu)h(time)f (usage,)g(but)g(this)h(is)g(not)f(a)f(requiremen)o(t.)75 1551 y Fm(4.4)70 b(Ov)n(erview)21 b(of)i(the)f(prop)r(osed)j(MPI)e (functions)75 1653 y Fs(This)d(c)o(hapter)f(assumes)f(the)i(a)o(v)m (ailabilit)o(y)h(of)d(the)i(follo)o(wing)f(MPI)g(features)g(whic)o(h)h (are)f(de\014ned)h(in)75 1709 y(other)15 b(c)o(hapters:)131 1805 y(1.)22 b(An)11 b(existing)g(pro)q(cess)g(group)f(\(e.g.,)g(the)h (ALL)h(group\))e(can)g(b)q(e)i(partitioned)f(and/or)f(ren)o(um)o(b)q (ered)189 1861 y(using)189 1973 y Ft(MPI)p 298 1973 16 2 v 18 w(CREA)l(TE)p 533 1973 V 19 w(GR)o(OUP\(gid)p 835 1973 V 20 w(old,)17 b(gid)p 1025 1973 V 20 w(new,)f(group)p 1299 1973 V 19 w(k)o(ey)l(,)f(order)p 1546 1973 V 18 w(k)o(ey\))g Fs(with)189 2104 y Ft(IN)i(gid)p 339 2104 V 19 w(old)24 b Fs(group)15 b(id)h(of)e(paren)o(t)h(group,)189 2178 y Ft(OUT)j(gid)p 394 2178 V 19 w(new)23 b Fs(group)15 b(id)h(of)f(newly)h(created)f(subgroup)g(of)g(whic)o(h)h(the)f(calling) i(pro)q(cess)f(is)f(a)289 2234 y(mem)o(b)q(er,)189 2308 y Ft(IN)i(group)p 402 2308 V 18 w(k)o(ey)22 b Fs(in)o(teger)15 b(v)m(alue)i(used)e(for)g(the)g(partitioning,)189 2382 y Ft(IN)i(order)p 393 2382 V 18 w(k)o(ey)k Fs(in)o(teger)e(v)m(alue)i (used)e(for)g(determining)h(the)f(ranks)g(in)h(the)f(new)g(group)g (\(see)289 2439 y(b)q(elo)o(w\),)189 2534 y(This)j(function)g(is)f (called)i(b)o(y)e(all)i(pro)q(cesses)e(in)i(group)d Fh(gid)p 1265 2534 14 2 v 17 w(old)i Fs(lo)q(osely)g(sync)o(hronously)l(.)39 b(It)189 2591 y(creates)18 b(as)g(man)o(y)g(di\013eren)o(t)g(subgroups) h(as)f(there)h(are)f(di\013eren)o(t)g(v)m(alues)i(of)e Fh(gr)n(oup)p 1676 2591 V 17 w(key)p Fs(.)30 b(The)189 2647 y(ranking)16 b(in)h(the)g(newly)g(created)f(groups)g(is)h (determined)h(b)o(y)e(the)h Fh(or)n(der)p 1464 2647 V 17 w(key)f Fs(v)m(alues,)h(with)g(the)189 2704 y(pro)q(cess)e(ranks)g (in)h(the)f(paren)o(t)g(group)g(for)g(tie)g(breaking.)p eop %%Page: 62 67 62 66 bop 75 -100 a Fs(62)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)131 45 y Fs(2.)22 b(If)13 b(there)g(is)h(a)e(standardized)i(in)o(ternal)g(information)f(cac)o (heing)h(facilit)o(y)l(,)g(it)g(can)f(b)q(e)g(used)h(b)o(y)f(MPI)189 102 y(routines)g(to)f(store)g(and)h(retriev)o(e)g(information)f (related)i(to)e(a)g(pro)q(cess)h(group.)19 b(This)13 b(facilit)o(y)h(w)o(as)189 158 y(prop)q(osed)e(in)h(the)g(Con)o(text)e (Sub)q(committee.)20 b(High{lev)o(el)14 b(top)q(ology)e(functions)h (store)e(structure)189 214 y(information)g(for)f(a)g(pro)q(cess)h (group,)g(whic)o(h)h(is)f(then)g(later)g(used)g(b)o(y)g(top)q(ology)f (inquiry)j(functions.)189 289 y(In)19 b(case)h(the)f(cac)o(heing)h(mec) o(hanism)g(do)q(es)f(not)g(b)q(ecome)h(part)f(of)g(MPI,)f(the)i(top)q (ology)f(infor-)189 345 y(mation)c(can)g(in)o(ternally)i(b)q(e)f (stored)e(with)i(other)f(group)f(information)i(in)g(an)f(implemen)o (tation{)189 402 y(dep)q(enden)o(t)h(w)o(a)o(y)l(,)e(whic)o(h)i(eac)o (h)g(v)o(endor)f(can)g(do)g(as)g(is)h(most)e(con)o(v)o(enien)o(t.)166 492 y(There)j(has)g(b)q(een)h(a)e(general)i(decision)g(in)g(MPI)f(to)f (start)f(all)j(n)o(um)o(b)q(ering)g(with)f(0)g(instead)g(of)g(1.)75 548 y(Th)o(us,)f(co)q(ordinates)h(in)h(a)e(cartesian)g(structure)h(are) f(0,1,)f(...,)h(dim-1.)24 b(In)17 b(the)g(same)f(w)o(a)o(y)l(,)g(no)q (des)h(in)g(a)75 605 y(general)f(graph)f(are)f(n)o(um)o(b)q(ered)i (zero)g(up.)75 726 y Fi(4.4.1)55 b(Lo)n(w{lev)n(el)18 b(top)r(ology)f(functions)75 812 y Fs(On)k(the)g(lo)o(w)o(est)e(lev)o (el)j(groups)e(are)g(created)h(b)o(y)f(the)h(general)g(function)g(MPI)p 1477 812 14 2 v 16 w(CREA)l(TE)p 1684 812 V 17 w(GR)o(OUP)l(.)75 868 y(Using)13 b(the)g(input)g(argumen)o(t)f Fh(newr)n(ank)g Fs(the)g(caller)i(can)e(c)o(hange)h(the)g(pro)q(cess)f(ranks)h(in)g (the)f(new)h(group)75 925 y(as)h(compared)g(to)g(the)g(paren)o(t)g (group.)19 b(This)c(feature)f(can)h(b)q(e)g(used)g(to)e(e\016cien)o (tly)j(map)e(the)h(pro)q(cesses)75 981 y(in)i(the)e(group)h(to)f(the)h (p)q(ositions)h(in)f(the)g(top)q(ology)l(.)21 b(F)l(or)15 b(eac)o(h)h(class)g(of)g(top)q(ologies,)g(therefore,)f(there)75 1038 y(is)f(a)f(function)h(whic)o(h)h(computes)e(the)h(rank)f(of)g(the) h(calling)h(pro)q(cess)f(in)g(the)g(new)f(group,)h(based)f(on)h(the)75 1094 y(size)i(and)g(structure)g(of)f(the)g(top)q(ology)l(.)21 b(Note)16 b(that)e(it)i(is)g(a)g(v)m(alid)h(implemen)o(tation)g(if)f (the)f(new)h(ranks)75 1151 y(are)f(iden)o(tical)j(to)c(the)i(old)g (ranks.)k(So,)c(if)f(for)g(a)h(giv)o(en)g(mac)o(hine)g(the)f(placemen)o (t)i(of)e(pro)q(cesses)h(mak)o(es)75 1207 y(no)f(di\013erence,)h(the)f (implemen)o(tation)i(of)e(the)g(rank)g(computation)g(functions)h(is)g (trivial.)166 1263 y(F)l(or)e(cartesian)i(structures)f(the)g(function)h (is)166 1355 y Ft(MPI)p 275 1355 16 2 v 18 w(MAP)p 419 1355 V 18 w(CAR)l(T\(gid)p 675 1355 V 19 w(old,)i(ndims,)e(dims,)h(p)q (erio)q(d,)h(newrank\))d Fs(with)75 1481 y Ft(IN)i(gid)p 225 1481 V 20 w(old)23 b Fs(group)15 b(id)h(of)f(paren)o(t)g(group,)75 1573 y Ft(IN)i(ndims)23 b Fs(n)o(um)o(b)q(er)15 b(of)g(dimensions)i(of) e(cartesian)g(structure,)75 1666 y Ft(IN)i(dims\(*\))23 b Fs(n)o(um)o(b)q(er)16 b(of)f(pro)q(cesses)g(in)h(eac)o(h)g(co)q (ordinate)f(direction,)75 1758 y Ft(IN)i(p)q(erio)q(d\(*\))25 b Fs(p)q(erio)q(dicit)o(y)17 b(sp)q(eci\014cation)g(in)f(eac)o(h)g(co)q (ordinate)f(direction.)75 1850 y Ft(OUT)j(newrank)k Fs(reordered)16 b(rank)f(of)f(the)i(calling)h(pro)q(cess.)166 2075 y(The)e(corresp)q (onding)i(function)f(for)e(general)i(graph)f(structures)g(is)166 2167 y Ft(MPI)p 275 2167 V 18 w(MAP)p 419 2167 V 18 w(GRAPH\(gid)p 723 2167 V 19 w(old,)j(no)q(des,)f(index,)g(nglist,)h(newrank\))d Fs(with)75 2293 y Ft(IN)i(gid)p 225 2293 V 20 w(old)23 b Fs(group)15 b(id)h(of)f(paren)o(t)g(group,)75 2385 y Ft(IN)i(no)q(des)23 b Fs(n)o(um)o(b)q(er)16 b(of)e(graph)h(no)q(des,) 75 2478 y Ft(IN)i(index\(*\),)h(nglist\(*\))25 b Fs(de\014nition)13 b(of)e(graph)g(structure:)17 b(F)l(or)11 b(eac)o(h)g(graph)g(no)q(de)h (its)g(set)f(of)g(neigh-)189 2534 y(b)q(ors)i(is)h(en)o(tered,)g(i.)f (e.,)g(the)h(set)f(of)g(pro)q(cesses)h(with)f(whic)o(h)i(the)e(no)q(de) h(pro)q(cess)g(is)g(going)f(to)g(com-)189 2591 y(m)o(unicate)k(most)f (frequen)o(tly)l(.)25 b(Since)19 b(there)d(is)i(no)e(w)o(eigh)o(ting)h (of)g(no)q(de)g(connections,)h(it)f(migh)o(t)189 2647 y(b)q(e)d(sensible)j(to)c(sp)q(ecify)i(only)g(those)f(neigh)o(b)q(ors)h (with)f(whic)o(h)h(there)f(is)h(a)f(considerable)i(amoun)o(t)189 2704 y(of)e(comm)o(unication.)p eop %%Page: 63 68 63 67 bop 75 -100 a Fq(4.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b Fs(63)189 45 y(F)l(or)20 b(eac)o(h)h(no)q(de)h Fr(i)35 b Fs(\(0)22 b Fo(\024)h Fr(i)e Fo(\024)i Fr(nodes)14 b Fo(\000)h Fs(1\))20 b(the)h(list)h(of)e(the)h(neigh)o(b)q(or)h(no)q(des)g(is)f (stored)g(at)189 102 y Fr(ng)r(l)q(ist)p Fs(\()p Fr(j)s Fs(\))p Fr(;)30 b(index)p Fs(\()p Fr(i)p Fs(\))13 b Fo(\024)i Fr(j)i Fo(\024)e Fr(index)p Fs(\()p Fr(i)10 b Fs(+)h(1\))f Fo(\000)h Fs(1.)23 b(F)l(or)16 b(the)g(last)g(pro)q(cess)h Fr(index)p Fs(\()p Fr(i)10 b Fs(+)h(1\))16 b(p)q(oin)o(ts)189 158 y(to)g(the)g(\014rst)g(free)h(p)q(osition)h(in)f(v)o(ector)f (nglist.)25 b(Note)16 b(that)g(the)g(matrix)h(is)g(symmetric,)f(i.)h (e.,)f(if)189 214 y(no)q(de)f Fh(i)f Fs(is)h(a)f(neigh)o(b)q(or)h(of)f (no)q(de)h Fh(j)p Fs(,)f(than)g(no)q(de)h Fh(j)f Fs(is)h(also)f(a)g (neigh)o(b)q(or)h(of)f(no)q(de)h Fh(i)p Fs(.)k(This)c(prop)q(ert)o(y) 189 271 y(can)g(b)q(e)h(used)g(for)e(an)h(in)o(ternal)h(c)o(hec)o(king) g(of)f(the)h(input)g(argumen)o(ts.)75 365 y Ft(OUT)i(newrank)k Fs(reordered)16 b(rank)f(of)f(the)i(calling)h(pro)q(cess.)75 555 y(The)11 b(de\014nitions)h(of)e(the)h(argumen)o(ts)f Fl(nodes)p Fs(,)g Fl(index)p Fs(,)h(and)g Fl(nglist)e Fs(are)i(illustrated)h(with)f(the)f(follo)o(wing)75 611 y(simple)17 b(example:)166 668 y(Assume)e(there)h(are)f(four)f(pro)q (cesses)i(0,)e(1,)h(2,)f(3)h(with)h(the)f(follo)o(wing)h(adjacency)g (matrix:)100 721 y(pro)q(cess)50 b(neigh)o(b)q(ors)100 777 y(0)170 b(1,)15 b(3)100 833 y(1)170 b(0)100 890 y(2)g(3)100 946 y(3)g(0,)15 b(2)166 999 y(Then,)g(the)h(input)g(argumen)o(ts)e (are:)100 1052 y(no)q(des)i(=)50 b(4)100 1108 y(index)16 b(=)56 b(0,)14 b(2,)h(3,)f(4,)h(6)100 1165 y(nglist)h(=)54 b(1,)14 b(3,)h(0,)f(3,)h(0,)f(2)166 1217 y(If)i(the)g(top)q(ology)g (con)o(tains)g(few)o(er)f(pro)q(cesses)i(than)e(there)h(are)g(pro)q (cesses)g(in)h(the)f(paren)o(t)g(group,)75 1274 y(the)g(newrank)g(v)m (alues)h(for)e(the)g(remaining)i(pro)q(cesses)f(are)g(set)f(to)h(MPI)p 1329 1274 14 2 v 16 w(NORANK.)h(It)e(is)i(an)e(error)g(if)75 1330 y(the)g(top)q(ology)g(de\014nes)h(more)f(pro)q(cesses)h(than)f (are)g(con)o(tained)h(in)g(the)f(paren)o(t)g(group.)75 1453 y Fi(4.4.2)55 b(High{lev)n(el)17 b(top)r(ology)g(functions)75 1539 y Fs(In)i(principle,)i(all)e(that)e(is)i(needed)g(to)f(create)g (an)g(MPI)g(pro)q(cess)g(group)g(with)h(top)q(ological)g(ordering)75 1595 y(are)13 b(the)h(t)o(w)o(o)f(mapping)h(functions)g(and)g(the)g (general)g(group)g(creation)f(routine.)20 b(Ho)o(w)o(ev)o(er,)13 b(this)h(is)g(not)75 1651 y(what)19 b(the)g(applications)i(programmer)d (exp)q(ects)i(to)f(see)h(on)f(his)h(lev)o(el)h(of)e(abstraction.)31 b(Therefore,)75 1708 y(another)13 b(la)o(y)o(er)h(of)f(functions)h(is)g (de\014ned)i(on)d(top)g(of)h(the)f(lo)o(w{lev)o(el)i(functions.)20 b(This)14 b(higher)h(lev)o(el)g(can)75 1764 y(b)q(e)g(p)q(ortably)f (implemen)o(ted)i(in)f(terms)f(of)g(the)g(lev)o(el)i(b)q(elo)o(w.)k(Of) 14 b(course)h(MPI)f(implemen)o(tors)h(are)f(free)75 1821 y(to)h(b)o(ypass)g(the)g(lo)o(w)o(er)g(lev)o(el)h(for)f(the)g(sak)o(e)g (of)g(b)q(etter)g(p)q(erformance)g(or)g(resource)g(usage.)166 1877 y(F)l(or)e(cartesian)h(and)g(graph)f(structures)g(the)h(functions) h(MPI)p 1226 1877 V 16 w(CAR)l(T)f(and)g(MPI)p 1561 1877 V 16 w(GRAPH)g(create)75 1934 y(a)g(pro)q(cess)g(group)g(with)h(top)q (ological)g(structure.)k(They)14 b(com)o(bine)h(the)f(call)i(to)d(the)h (mapping)h(function)75 1990 y(with)f(the)g(call)h(to)e(the)g(group)h (creation)g(routine.)20 b(The)14 b(user)f(can)h(decide)i(whether)e(the) f(ranking)h(in)h(the)75 2047 y(new)i(group)g(ma)o(y)f(b)q(e)i (reordered)f(\(for)f(a)g(more)h(e\016cien)o(t)g(mapping\))h(or)e(not.) 25 b(In)17 b(the)g(latter)g(case)g(the)75 2103 y(rank)e(of)g(the)g (calling)i(pro)q(cess)e(is)h(copied)g(from)f(the)g(paren)o(t)g(group.) 166 2160 y(The)h(option)g(of)f(not)g(reordering)h(the)g(pro)q(cess)f (ranks)h(can)f(b)q(e)i(useful)f(in)h(the)e(case)h(of)f(an)h(o)o(v)o (erla)o(y)75 2216 y(of)f(di\013eren)o(t)h(top)q(ologies.)21 b(The)16 b(user)f(can)h(then)g(fully)h(con)o(trol)e(whic)o(h)h(lo)q (cation)g(in)h(the)e(\014rst)g(top)q(ology)75 2273 y(coincides)i(with)e (whic)o(h)h(lo)q(cation)g(in)f(the)g(second)h(one.)k(The)15 b(price)h(whic)o(h)f(one)h(ob)o(viously)f(has)g(to)f(pa)o(y)75 2329 y(in)i(this)g(case)f(is)h(that)e(for)h(only)g(one)h(top)q(ology)f (the)g(pro)q(cess)g(arrangemen)o(t)f(can)i(b)q(e)g(optimized.)166 2386 y(As)11 b(sp)q(eci\014ed)j(for)d(the)h(lo)o(w{lev)o(el)g (functions)h(it)f(is)g(an)f(error)g(if)h(one)g(attempts)e(to)h (de\014ne)i(a)e(top)q(ology)75 2442 y(with)j(more)g(lo)q(cations)g (than)g(there)g(are)g(pro)q(cesses)g(in)h(the)f(paren)o(t)f(pro)q(cess) h(group.)20 b(If)14 b(it)g(do)q(es)g(not)g(use)75 2499 y(all)j(a)o(v)m(ailable)h(pro)q(cesses,)f(the)f(un)o(used)h(pro)q(cess) f(will)i(return)e(MPI)p 1249 2499 V 17 w(NORANK)h(as)f(their)h(rank)f (in)h(the)75 2555 y(new)e(group.)166 2612 y(MPI)p 258 2612 V 16 w(CAR)l(T)h(is)g(de\014ned)g(as)166 2704 y Ft(MPI)p 275 2704 16 2 v 18 w(CAR)l(T\(gid)p 531 2704 V 19 w(old,)i(gid)p 721 2704 V 19 w(cart,)g(ndims,)f(dims,)f(p)q(erio)q (d,)i(reorder,)e(newrank\))f Fs(with)p eop %%Page: 64 69 64 68 bop 75 -100 a Fs(64)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Ft(IN)i(gid)p 225 45 16 2 v 20 w(old)23 b Fs(group)15 b(id)h(of)f(paren)o(t)g(group,)75 145 y Ft(OUT)j(gid)p 280 145 V 20 w(cart)23 b Fs(group)17 b(id)h(of)f(cartesian)g(group)g(\(if)g(calling)i(pro)q(cess)e(is)h (part)e(of)h(it,)h(else)g(id)g(of)e(rest)189 202 y(group\),)75 302 y Ft(IN)h(ndims,)g(dims\(*\),)g(p)q(erio)q(d\(*\))25 b Fs(de\014nition)17 b(of)d(cartesian)i(structure,)e(\(for)g(details)i (see)f(the)h(def-)189 358 y(inition)h(of)d(MPI)p 476 358 14 2 v 17 w(MAP)p 600 358 V 16 w(CAR)l(T\))75 459 y Ft(IN)j(reorder)22 b Fs(ranking)15 b(ma)o(y)g(b)q(e)g(reordered)h(or) f(not,)75 559 y Ft(OUT)j(newrank)k Fs(rank)15 b(of)g(the)g(calling)i (pro)q(cess)f(in)g(the)f(cartesian)g(group)g(\(or)g(MPI)p 1581 559 V 16 w(NORANK\).)166 659 y(Apart)g(from)f(some)h(lo)q(cal)h (computations,)f(MPI)p 1015 659 V 16 w(CAR)l(T)h(is)g(based)f(on)g(the) h(t)o(w)o(o)d(function)j(calls)166 753 y Ft(MPI)p 275 753 16 2 v 18 w(MAP)p 419 753 V 18 w(CAR)l(T)h(\(gid)p 692 753 V 19 w(old,)h(ndims,)f(dims,)f(p)q(erio)q(d,)i(newrank\))166 846 y Fs(and)166 940 y Ft(MPI)p 275 940 V 18 w(CREA)l(TE)p 510 940 V 19 w(GR)o(OUP)f(\(gid)p 829 940 V 20 w(old,)h(gid)p 1020 940 V 19 w(cart,)g(0,)f(newrank\))e Fs(.)166 1092 y(F)l(or)f(graph)h(structures)g(the)h(corresp)q(onding)g(routine)f(is) 166 1185 y Ft(MPI)p 275 1185 V 18 w(GRAPH\(gid)p 579 1185 V 19 w(old,)24 b(gid)p 775 1185 V 20 w(graph,)f(no)q(des,)g (index,)g(nglist,)i(reorder,)c(newrank\))75 1242 y Fs(with)75 1377 y Ft(IN)c(gid)p 225 1377 V 20 w(old)23 b Fs(group)15 b(id)h(of)f(paren)o(t)g(group,)75 1477 y Ft(OUT)j(gid)p 280 1477 V 20 w(graph)23 b Fs(group)18 b(id)h(of)g(graph)f(group)g (\(if)h(calling)i(pro)q(cess)d(is)i(part)e(of)g(it,)h(else)h(id)f(of)f (rest)189 1534 y(group\),)75 1634 y Ft(IN)f(no)q(des,)g(index\(*\),)h (nglist\(*\))25 b Fs(de\014nition)12 b(of)e(graph)g(structure,)h(\(for) e(details)i(see)g(the)f(de\014nition)189 1691 y(of)k(MPI)p 332 1691 14 2 v 17 w(MAP)p 456 1691 V 16 w(GRAPH\))75 1791 y Ft(IN)j(reorder)22 b Fs(ranking)15 b(ma)o(y)g(b)q(e)g(reordered) h(or)f(not.)75 1891 y Ft(OUT)j(newrank)k Fs(rank)10 b(of)g(the)g (calling)i(pro)q(cess)e(in)h(the)g(graph)f(top)q(ology)f(group)h(\(or)g (MPI)p 1649 1891 V 16 w(NORANK\).)166 2050 y(The)16 b(follo)o(wing)h (top)q(ology)f(information)h(is)g(lik)o(ely)h(to)e(b)q(e)g(stored)g (with)h(the)f(pro)q(cess)h(group)f(b)o(y)g(a)75 2106 y(high{lev)o(el)h(top)q(ology)e(de\014nition)i(function:)131 2207 y(1.)22 b(T)o(yp)q(e)15 b(of)g(top)q(ology)g(\(cartesian/graph\)) 131 2307 y(2.)22 b(F)l(or)14 b(a)h(cartesian)g(top)q(ology:)243 2407 y Fo(\017)23 b Fs(ndims)16 b(\(n)o(um)o(b)q(er)f(of)g (dimensions\),)243 2483 y Fo(\017)23 b Fs(dims\(*\))15 b(\(n)o(um)o(b)q(ers)g(of)f(pro)q(cesses)i(p)q(er)g(co)q(ordinate)f (direction\),)243 2560 y Fo(\017)23 b Fs(p)q(erio)q(d\(*\))15 b(\(p)q(erio)q(dicit)o(y)i(information\),)243 2636 y Fo(\017)23 b Fs(o)o(wn)p 372 2636 V 16 w(p)q(osition\(*\))c(\(o)o(wn)f (p)q(osition)i(in)g(grid,)h(could)f(also)f(b)q(e)h(computed)f(from)g (rank)g(and)289 2692 y(dims\(*\)\))p eop %%Page: 65 70 65 69 bop 75 -100 a Fq(4.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b Fs(65)75 45 y(F)l(or)16 b(a)g(graph)g(structure)g(no)g(information)h(has)f(to)g (b)q(e)h(stored,)f(b)q(ecause)h(the)g(graph)f(no)q(de)h(n)o(um)o(b)q (er)f(is)75 102 y(the)h(rank)f(in)i(the)f(graph)f(group,)h(and)g(the)g (size)g(of)g(the)g(graph)f(is)i(the)e(n)o(um)o(b)q(er)i(of)e(pro)q (cesses)h(in)h(that)75 158 y(group.)166 214 y(The)d(actual)g(set)g(of)g (top)q(ology)f(data)h(stored)f(with)i(a)f(pro)q(cess)g(group)g(is)g (implemen)o(tation)h(dep)q(en-)75 271 y(den)o(t.)k(It)13 b(can)h(con)o(tain)g(more)g(information)g(than)g(giv)o(en)g(ab)q(o)o(v) o(e,)f(whereas)h(in)h(other)e(cases)h(some)g(items)75 327 y(migh)o(t)h(b)q(e)h(redundan)o(t.)75 449 y Fi(4.4.3)55 b(T)-5 b(op)r(ology)18 b(inquiry)f(functions)75 534 y Fs(If)j(a)g(top)q(ology)g(has)f(b)q(een)j(de\014ned)f(with)f(one)g(of)g (the)g(ab)q(o)o(v)o(e)g(high{lev)o(el)i(functions,)f(the)f(top)q(ology) 75 591 y(information)c(can)h(b)q(e)f(lo)q(ok)o(ed)h(up)g(using)f (inquiry)i(functions.)23 b(They)17 b(all)g(are)f(lo)q(cal)h(calls,)g (and)f(do)g(not)75 647 y(require)g(an)o(y)f(comm)o(unication.)166 704 y(The)e(function)h(MPI)p 525 704 14 2 v 17 w(INQMAP)f(returns)g (the)g(t)o(yp)q(e)h(of)e(mapping)i(whic)o(h)g(is)g(assigned)f(to)g(a)g (group.)166 796 y Ft(MPI)p 275 796 16 2 v 18 w(INQMAP\(gid,)k(map\).) 166 888 y Fs(The)e(output)g(v)m(alue)i Fl(map)d Fs(is)i(one)g(of)e(the) i(follo)o(wing:)143 979 y Fo(\017)23 b Fs(MPI)p 281 979 14 2 v 16 w(CAR)l(TTOP:)15 b(cartesian)h(top)q(ology)l(,)143 1072 y Fo(\017)23 b Fs(MPI)p 281 1072 V 16 w(GRAPHTOP:)15 b(graph)g(top)q(ology)l(,)143 1164 y Fo(\017)23 b Fs(MPI)p 281 1164 V 16 w(NOTOP:)15 b(no)g(top)q(ology)l(.)166 1256 y(F)l(or)j(a)g(pro)q(cess)g(group)h(with)f(cartesian)h(ordering)g (whic)o(h)g(has)f(b)q(een)i(created)e(b)o(y)h(MPI)p 1720 1256 V 16 w(CAR)l(T,)75 1312 y(functions)d(MPI)p 363 1312 V 16 w(INQDIM)g(and)g(MPI)p 751 1312 V 16 w(INQCAR)l(T)g(lo)q(ok)g (up)f(top)q(ology)g(information:)166 1404 y Ft(MPI)p 275 1404 16 2 v 18 w(INQDIM\(gid,)i(dim\))f Fs(with)75 1531 y Ft(IN)h(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with)h (cartesian)f(structure,)75 1623 y Ft(OUT)j(dim)23 b Fs(dimension)17 b(of)e(the)g(cartesian)g(structure.)166 1750 y Ft(MPI)p 275 1750 V 18 w(INQCAR)l(T\(gid,)i(maxdim,)g(size,)g(p)q(erio)q(d,)h(p) q(os\))e Fs(with)75 1877 y Ft(IN)h(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with)h(cartesian)f(structure,)75 1970 y Ft(IN)i(maxdim)23 b Fs(length)15 b(of)g(v)o(ectors)g(size\(*\),) f(p)q(erio)q(d\(*\),)i(and)f(p)q(os\(*\))f(in)j(the)e(calling)i (program,)75 2062 y Ft(OUT)h(size\(*\))24 b Fs(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(for)f(eac)o(h)g(co)q(ordinate)g(direction,)75 2155 y Ft(OUT)j(p)q(erio)q(d\(*\))25 b Fs(p)q(erio)q(dicit)o(y)17 b(y)o(es/no)e(for)g(eac)o(h)g(co)q(ordinate)g(direction,)75 2248 y Ft(OUT)j(p)q(os\(*\))24 b Fs(p)q(osition)16 b(of)f(calling)i (pro)q(cess)e(in)h(cartesian)g(structure.)166 2339 y(The)k(inquiry)i (for)e(cartesian)g(top)q(ology)g(information)h(is)f(th)o(us)g(split)i (up)e(in)o(to)h(t)o(w)o(o)e(calls.)36 b(The)75 2396 y(reason)21 b(is)h(that)f(otherwise)h(the)f(user)h(migh)o(t)f(not)g(kno)o(w)g(a)h (sa)o(v)o(e)e(v)m(alue)j(for)e(the)h(dimensioning)h(of)75 2452 y(v)o(ectors)14 b Fl(size,)24 b(period)14 b Fs(and)h Fl(pos)g Fs(b)q(efore)h(calling)g(MPI)p 1078 2452 14 2 v 17 w(INQCAR)l(T)g(\(input)g(argumen)o(t)f Fl(maxdim)p Fs(\).)166 2508 y(F)l(or)e(a)g(pro)q(cess)h(group)g(with)g(cartesian)f (structure,)h(the)f(function)i(MPI)p 1411 2508 V 16 w(INQPID)g (translates)e(the)75 2565 y(logical)j(pro)q(cess)g(co)q(ordinates)f(to) g(pro)q(cess)g(ids)h(as)f(they)g(are)g(used)h(b)o(y)f(the)h(p)q(oin)o (t{to{p)q(oin)o(t)f(routines:)166 2657 y Ft(MPI)p 275 2657 16 2 v 18 w(INQPID\(gid,)j(lo)q(cation,)h(pid\))d Fs(with)p eop %%Page: 66 71 66 70 bop 75 -100 a Fs(66)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Ft(IN)i(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with)h(cartesian)f(structure,)75 142 y Ft(IN)i(lo)q(cation)26 b Fs(co)q(ordinate)15 b(v)o(ector)g(of)g (pro)q(cess,)75 239 y Ft(OUT)j(pid)24 b Fs(pro)q(cess)15 b(id.)166 350 y(In)h(a)f(sense,)g(the)g(in)o(v)o(erse)h(op)q(eration)f (to)g(MPI)p 970 350 14 2 v 16 w(INQPID)h(is)g(pro)o(vided)g(b)o(y)f (the)g(function:)166 442 y Ft(MPI)p 275 442 16 2 v 18 w(INQLOC\(gid,)j(rank,)f(lo)q(cation,)j(maxdim\))15 b Fs(with)75 588 y Ft(IN)i(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with) h(cartesian)f(structure,)75 685 y Ft(IN)i(rank)22 b Fs(rank)15 b(of)g(a)g(pro)q(cess)g(within)i(group)e Fl(gid)p Fs(,)75 782 y Ft(OUT)j(lo)q(cation\(*)q(\))26 b Fs(co)q(ordinate)15 b(v)o(ector)g(of)f(pro)q(cess,)75 879 y Ft(IN)j(maxdim)23 b Fs(length)15 b(of)g(v)o(ector)g Fl(location)f Fs(in)i(the)f(calling)i (program)d(\(only)i(for)e(error)h(c)o(hec)o(king\).)75 990 y(MPI)p 167 990 14 2 v 16 w(INQLOC)i(translates)e(a)g(rank)f(in)j (the)e(group)g(in)o(to)g(a)g(top)q(ology)g(lo)q(cation.)166 1130 y Fk(Discussion:)22 b Fj(Dep)q(ending)17 b(on)g(the)h(decisions)g (tak)o(en)f(in)g(the)h(PT2PT)f(sub)q(committee,)f(MPI)p 1708 1130 13 2 v 16 w(INQPID)75 1186 y(migh)o(t)c(b)q(e)i(c)o(hanged)h (to)e(return)i(the)g(rank)f(in)f(the)i(group)e(rather)i(than)f(a)g (pid.)75 1395 y Fi(4.4.4)55 b(P)n(artitioning)18 b(of)h(cartesian)f (structures)75 1482 y Fs(If)h(a)g(group)f(has)h(b)q(een)h(created)f (with)g(MPI)p 856 1482 14 2 v 16 w(CAR)l(T,)g(the)g(function)h(MPI)p 1386 1482 V 16 w(P)l(AR)l(TC)g(can)f(b)q(e)g(used)h(to)75 1539 y(partition)12 b(the)g(group)g(in)o(to)g(subgroups)g(whic)o(h)h (form)e(h)o(yp)q(erplanes)j(of)d(the)h(original)h(pro)q(cess)g (structure.)75 1595 y(As)k(an)f(example,)i(assume)e(that)g(MPI)p 751 1595 V 17 w(CAR)l(T)h(has)f(de\014ned)i(a)e(t)o(w)o(o{dimensional)i (grid)f(structure)f(on)75 1652 y(a)g(pro)q(cess)g(group.)22 b(With)16 b(a)g(call)h(to)e(MPI)p 811 1652 V 16 w(P)l(AR)l(TC)i(this)f (group)g(can)g(then)g(b)q(e)h(split)g(up)f(in)o(to)g(disjoin)o(t)75 1708 y(subgroups)f(whic)o(h)h(represen)o(t)g(ro)o(ws)e(or)g(columns)i (of)f(the)g(original)i(group.)i(The)d(function)g(syn)o(tax)e(is:)166 1801 y Ft(MPI)p 275 1801 16 2 v 18 w(P)l(AR)l(TC\(gid,)j(lostdim,)g (gid)p 864 1801 V 20 w(new\))e Fs(with)75 1947 y Ft(IN)i(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with)h(cartesian)f(structure,)75 2044 y Ft(IN)i(lostdim)24 b Fs(direction)16 b(p)q(erp)q(endicul)q(ar)h (to)e(resulting)h(h)o(yp)q(erplanes.)75 2141 y Ft(OUT)i(gid)p 280 2141 V 20 w(new)k Fs(group)e(iden)o(ti\014er)j(of)d(the)g(h)o(yp)q (erplane)i(group)e(in)i(whic)o(h)f(the)f(calling)j(pro)q(cess)d(is)189 2197 y(con)o(tained.)166 2308 y(MPI)p 258 2308 14 2 v 16 w(P)l(AR)l(TC)15 b(uses)f(the)g(function)h(MPI)p 883 2308 V 17 w(CREA)l(TE)p 1091 2308 V 16 w(GR)o(OUP)g(to)e(partition)i (the)f(paren)o(t)g(group,)75 2364 y(using)22 b(the)f(pro)q(cess)g(co)q (ordinate)h(in)f(direction)i Fl(lostdim)d Fs(as)g(the)i(k)o(ey)e(v)m (alue.)39 b(After)21 b(a)f(group)h(has)75 2421 y(b)q(een)14 b(partitioned)f(b)o(y)g(MPI)p 565 2421 V 16 w(P)l(AR)l(TC,)g(the)f (inquiry)i(functions)g(MPI)p 1275 2421 V 16 w(INQCAR)l(T)g(and)f(MPI)p 1694 2421 V 16 w(INQPID)75 2477 y(consisten)o(tly)j(refer)f(to)g(the)g (grid)g(sections)h(if)g(applied)h(to)d(a)h(subgroup.)166 2534 y(If)d(the)g(pro)q(cess)g(top)q(ology)g(is)g(a)g(cartesian)f (structure,)h(a)g(collectiv)o(e)i(shift)e(op)q(eration)g(most)f(lik)o (ely)i(is)75 2591 y(used)i(along)f(a)g(co)q(ordinate)h(direction.)21 b(In)15 b(MPI)f(a)g(shift)h(op)q(eration)f(is)h(handled)h(b)o(y)e(the)h (more)e(general)75 2647 y(send{receiv)o(e)h(functions.)20 b(Those)12 b(tak)o(e)g(as)g(input)i(the)f(iden)o(ti\014er)h(of)e(a)g (source)h(pro)q(cess)g(for)f(the)g(receiv)o(e,)75 2704 y(and)j(the)h(iden)o(ti\014er)g(of)f(a)g(destination)h(pro)q(cess)g (for)e(the)i(send.)p eop %%Page: 67 72 67 71 bop 75 -100 a Fq(4.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(PR)o(OPOSED)h(MPI)f(FUNCTIONS)536 b Fs(67)166 45 y(If)11 b(the)g(function)h(MPI)p 547 45 14 2 v 16 w(SHIFT)g(is)f(called)i(for)d(a)h(cartesian)g(pro)q(cess)g(group,)g(it) h(pro)o(vides)f(the)g(calling)75 102 y(pro)q(cess)17 b(with)f(the)h(ab)q(o)o(v)o(e)f(iden)o(ti\014ers,)i(whic)o(h)f(then)g (can)g(b)q(e)g(passed)f(to)g(the)h(send{receiv)o(e)g(function.)75 158 y(The)e(user)h(sp)q(eci\014es)h(the)e(co)q(ordinate)h(direction)g (for)f(the)g(shift)h(and)f(the)g(step)g(length:)166 256 y Ft(MPI)p 275 256 16 2 v 18 w(SHIFT\(gid,)j(direction,)g(steps,)f(pid) p 1027 256 V 19 w(source,)g(pid)p 1295 256 V 19 w(dest\))e Fs(with)75 427 y Ft(IN)i(gid)24 b Fs(group)14 b(id)j(of)d(group)h(with) h(cartesian)f(structure,)75 544 y Ft(IN)i(direction)24 b Fs(co)q(ordinate)16 b(direction)g(of)f(shift,)75 661 y Ft(IN)i(steps)22 b Fs(step)16 b(length)g(\()p Fr(>)c Fs(0:)20 b(up)o(w)o(ards)14 b(shift,)i Fr(<)d Fs(0:)19 b(do)o(wn)o(w)o(ards)14 b(shift\),)75 778 y Ft(OUT)k(pid)p 283 778 V 20 w(source)k Fs(pro)q(cess)15 b(iden)o(ti\014er)i(of)e (source)g(pro)q(cess,)75 896 y Ft(OUT)j(pid)p 283 896 V 20 w(dest)k Fs(pro)q(cess)16 b(iden)o(ti\014er)h(of)d(destination)i (pro)q(cess.)166 1031 y(Dep)q(ending)c(of)f(the)g(p)q(erio)q(dicit)o(y) i(of)d(the)h(cartesian)g(group)f(in)i(the)f(sp)q(eci\014ed)i(co)q (ordinate)e(direction,)75 1088 y(MPI)p 167 1088 14 2 v 16 w(SHIFT)j(pro)o(vides)f(the)g(iden)o(ti\014ers)i(for)d(a)h (circular)h(or)e(an)h(end{o\013)g(shift.)19 b(In)14 b(the)f(case)g(of)g (an)g(end{)75 1144 y(o\013)h(shift,)h(at)f(the)g(b)q(oundary)h(pro)q (cess)g(one)g(of)f(the)h(iden)o(ti\014ers)h(is)g(set)e(to)g(MPI)p 1431 1144 V 16 w(NORANK,)i(indicating)75 1201 y(that)e(there)i(is)f(no) h(pro)q(cess)f(in)h(that)f(direction)h(to)f(comm)o(unicate)g(with.)166 1263 y(In)22 b(a)g(p)q(ortable)g(program)f(the)h(n)o(um)o(b)q(er)g(of)f (a)o(v)m(ailable)j(pro)q(cessors)d(usually)i(is)g(not)e(kno)o(wn)h(at) 75 1319 y(compile)g(time.)35 b(In)21 b(the)g(case)f(of)g(a)g(cartesian) g(top)q(ology)g(the)h(function)g(MPI)p 1478 1319 V 16 w(MAKDIM)g(helps)g(in)75 1376 y(setting)i(the)f(n)o(um)o(b)q(ers)h(of)f (pro)q(cesses)h(p)q(er)g(co)q(o)q(dinate)g(direction,)j(dep)q(ending)e (on)f(the)g(n)o(um)o(b)q(er)f(of)75 1432 y(pro)q(cesses)16 b(in)g(the)f(group)g(and)g(optional)h(constrain)o(ts)f(whic)o(h)h(can)f (b)q(e)h(sp)q(eci\014ed)h(b)o(y)e(the)h(user:)166 1530 y Ft(MPI)p 275 1530 16 2 v 18 w(MAKDIM\(gid,)h(ndims,)f(dims\))f Fs(with)75 1701 y Ft(IN)i(gid)24 b Fs(group)14 b(id)j(of)d(group)h (whic)o(h)h(is)g(to)f(b)q(e)g(structured,)75 1818 y Ft(IN)i(ndims)23 b Fs(n)o(um)o(b)q(er)15 b(of)g(cartesian)g(dimensions,)75 1935 y Ft(IN/OUT)j(dims)23 b Fs(n)o(um)o(b)q(er)15 b(of)g(pro)q(cesses) g(in)i(eac)o(h)e(co)q(ordinate)g(direction.)166 2071 y(The)d(pro)q(cesses)g(of)f(the)h(group)f(with)i(ID)e(n)o(um)o(b)q(er)h Fl(gid)g Fs(are)f(divided)j(as)d(ev)o(enly)i(as)e(p)q(ossible)j(among) 75 2127 y Fl(ndims)i Fs(dimensions,)i(sub)s(ject)e(to)g(the)g (constrain)o(t)g(that)g(all)i(pro)q(cesses)e(in)i(the)e(group)g(are)g (used.)25 b(The)75 2184 y(caller)19 b(ma)o(y)e(further)h(constrain)g (the)g(op)q(eration)g(of)g(this)g(routine)g(b)o(y)g(sp)q(ecifying)i (elemen)o(ts)f(of)e(arra)o(y)75 2240 y Fl(dims)p Fs(.)23 b(If)17 b Fl(dims\(i\))f Fs(is)h(set)f(to)g(a)h(p)q(ositiv)o(e)g(n)o (um)o(b)q(er,)g(the)g(routine)g(will)h(\014x)f(the)f(n)o(um)o(b)q(er)h (of)g(pro)q(cesses)75 2297 y(in)g(co)q(ordinate)g(direction)g Fl(i)f Fs(to)g Fl(dims\(i\))p Fs(.)22 b(Pro)q(cesses)16 b(will)i(then)f(b)q(e)f(distributed)i(among)e(directions)75 2353 y(where)f Fl(dims\(i\))g Fs(is)g(set)g(to)g(0,)f(sub)s(ject)h(to)g (the)g(ab)q(o)o(v)o(e)g(constrain)o(ts.)166 2415 y(Note)k(that)h(an)f (error)g(will)j(o)q(ccur)e(if)h Fl(dims\(i\))d Fs(is)j(p)q(ositiv)o(e)g (and)f(do)q(es)g(not)f(ev)o(enly)i(divide)h(the)75 2472 y(n)o(um)o(b)q(er)13 b(of)f(pro)q(cesses)g(in)h(the)g(group.)19 b(Negativ)o(e)12 b(input)h(v)m(alues)h(of)e Fl(dims\(i\))f Fs(will)j(also)e(cause)h(an)f(error.)75 2528 y(All)k(elemen)o(ts)g(of)f Fl(dims\(*\))f Fs(m)o(ust)h(b)q(e)h(set)f(to)f(0)h(for)g(completely)h (automatic)f(pro)q(cess)g(distribution.)166 2591 y(When)c(the)g (function)h(returns,)f Fl(dims\(i\))f Fs(will)j(b)q(e)e(set)g(to)f(the) h(n)o(um)o(b)q(er)g(of)g(pro)q(cesses)g(in)h(co)q(ordinate)75 2647 y(direction)20 b Fl(i)p Fs(.)29 b(F)l(or)18 b Fl(dims\(i\))g Fs(not)g(set)g(b)o(y)h(the)f(user,)h Fl(dims\(i\))f Fs(will)i(b)q(e)f (ordered)g(\(MAXIMUM,)e(...,)75 2704 y(MINIMUM\).)d(Arra)o(y)h Fl(dims\(*\))f Fs(is)i(suitable)g(for)f(use)g(as)g(input)h(to)f (routine)g(MPI)p 1503 2704 14 2 v 17 w(CAR)l(T.)p eop %%Page: 68 73 68 72 bop 75 -100 a Fs(68)905 b Fq(CHAPTER)15 b(4.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fm(4.5)70 b(A)22 b(simple)f(example)75 147 y Fs(The)12 b(example)h(in)g(\014gure)f(4.1)f (sho)o(ws)g(ho)o(w)h(the)g(grid)g(de\014nition)i(and)e(inquiry)i (functions)f(can)f(b)q(e)h(used)f(in)75 203 y(an)k(application)h (program.)k(A)16 b(partial)g(di\013eren)o(tial)h(equation,)f(for)f (example)h(the)g(P)o(oisson)g(equation,)75 259 y(is)g(to)f(b)q(e)h (solv)o(ed)f(on)h(a)f(rectangular)g(domain.)21 b(First)15 b(the)g(pro)q(cesses)h(organize)f(themselv)o(es)h(in)h(a)e(t)o(w)o(o{) 75 316 y(dimensional)j(structure.)i(Eac)o(h)15 b(pro)q(cess)h(then)g (inquires)h(for)e(the)h(PID)f(n)o(um)o(b)q(ers)h(of)f(its)h(neigh)o(b)q (ors)g(in)75 372 y(the)f(four)g(directions.)21 b(The)15 b(n)o(umerical)h(problem)g(is)g(solv)o(ed)f(b)o(y)g(an)g(iterativ)o(e)g (metho)q(d,)g(the)g(details)h(of)75 429 y(whic)o(h)g(are)f(hidden)i(in) f(the)f(subroutine)h Fl(relax)p Fs(.)166 485 y(In)e(eac)o(h)f (relaxation)g(step)g(eac)o(h)g(pro)q(cess)h(computes)f(new)g(v)m(alues) h(for)f(the)g(solution)h(grid)f(function)75 542 y(at)19 b(all)i(p)q(oin)o(ts)f(o)o(wned)f(b)o(y)h(the)g(pro)q(cess.)33 b(Then)21 b(the)e(v)m(alues)i(at)e(in)o(terpro)q(cess)h(b)q(oundaries)h (ha)o(v)o(e)f(to)75 598 y(b)q(e)h(exc)o(hanged)h(with)f(neigh)o(b)q (oring)h(pro)q(cesses.)37 b(F)l(or)20 b(example,)j(the)d(exc)o(hange)h (subroutine)h(migh)o(t)75 655 y(con)o(tain)13 b(a)g(call)i(lik)o(e)f (MPI)p 524 655 14 2 v 16 w(SENDC\(...,neigh)p 873 655 V 16 w(pid\(1\),...\))k(to)12 b(send)i(up)q(dated)g(v)m(alues)h(to)d (the)h(left{hand)75 711 y(neigh)o(b)q(or)j(\(i-1,j\).)j(Note)c(that)g (there)g(is)h(only)f(lo)q(cal)i(comm)o(unication)f(in)g(this)f(exc)o (hange)h(op)q(eration.)p eop %%Page: 69 74 69 73 bop 75 -100 a Fq(4.5.)34 b(A)15 b(SIMPLE)h(EXAMPLE)1180 b Fs(69)252 191 y(in)o(teger)15 b(ndims,)h(n)o(um)p 639 191 14 2 v 16 w(neigh)252 247 y(logical)g(reorder)252 304 y(parameter)e(\(ndims=2,)i(n)o(um)p 782 304 V 16 w(neigh=4,)g(reorder=.true.\))252 360 y(in)o(teger)f(gid,)h(gid)p 554 360 V 17 w(cart,)e(dims\(ndims\),)h(neigh)p 1057 360 V 18 w(def\(2\),)f(neigh)p 1325 360 V 17 w(pid\(n)o(um)p 1509 360 V 18 w(neigh\))252 417 y(in)o(teger)h(o)o(wn)p 485 417 V 16 w(p)q(osition\(ndims\),)h(p)q(erio)q(d\(ndims\),)h(i,)e (j,)g(rank)252 473 y(real)p Fo(\003)p Fs(8)g(u\(0:101,0:101)o(\),)d (f\(0:101,0:10)o(1\))252 530 y(data)i(dims)i(/)f(ndims)h Fo(\003)f Fs(0)g(/)75 586 y(C)89 b(Create)15 b(a)g(grid)g(structure)g (in)h(ALL)g(group)f(and)h(inquire)g(ab)q(out)g(o)o(wn)e(p)q(osition)252 642 y(gid)i(=)f(MPI)p 470 642 V 16 w(ALL)75 699 y(C)89 b(Set)15 b(pro)q(cess)h(grid)f(size)i(and)e(p)q(erio)q(dicit)o(y)252 755 y(MPI)p 344 755 V 16 w(MAKDIM\(gid,)g(ndims,)h(dims\))252 812 y(p)q(erio)q(d\(1\))f(=)h(MPI)p 594 812 V 16 w(PER)252 868 y(p)q(erio)q(d\(2\))f(=)h(MPI)p 594 868 V 16 w(PER)252 925 y(MPI)p 344 925 V 16 w(CAR)l(T)g(\(gid,)f(gid)p 671 925 V 17 w(cart,)f(ndims,)i(dims,)f(p)q(erio)q(d,)h(reorder,)f(rank\)) 252 981 y(MPI)p 344 981 V 16 w(INQCAR)l(T)i(\(gid)p 669 981 V 16 w(cart,)d(ndims,)i(dims,)g(p)q(erio)q(d,)g(o)o(wn)p 1294 981 V 16 w(p)q(osition\))75 1038 y(C)89 b(Lo)q(ok)15 b(up)h(the)f(PID)h(n)o(um)o(b)q(ers)f(for)g(the)g(neigh)o(b)q(ors.)21 b(Own)15 b(pro)q(cess)h(co)q(ordinates)f(are)g(\(i,j\).)75 1094 y(C)89 b(Neigh)o(b)q(ors)16 b(are)f(\(i{1,j\),)f(\(i+1,j\),)g (\(i,j{1\),)g(\(i,j+1\))252 1151 y(i)i(=)f(o)o(wn)p 413 1151 V 16 w(p)q(osition\(1\))252 1207 y(j)g(=)g(o)o(wn)p 414 1207 V 16 w(p)q(osition\(2\))252 1263 y(neigh)p 360 1263 V 17 w(def\(1\))g(=)g(i{1)252 1320 y(neigh)p 360 1320 V 17 w(def\(2\))g(=)g(j)252 1376 y(MPI)p 344 1376 V 16 w(INQPID)h(\(gid,)f(neigh)p 754 1376 V 18 w(def,)g(neigh)p 964 1376 V 17 w(pid\(1\)\))252 1433 y(neigh)p 360 1433 V 17 w(def\(1\))g(=)g(i+1)252 1489 y(neigh)p 360 1489 V 17 w(def\(2\))g(=)g(j)252 1546 y(MPI)p 344 1546 V 16 w(INQPID)h(\(gid,)f(neigh)p 754 1546 V 18 w(def,)g(neigh)p 964 1546 V 17 w(pid\(2\)\))252 1602 y(neigh)p 360 1602 V 17 w(def\(1\))g(=)g(i)252 1659 y(neigh)p 360 1659 V 17 w(def\(2\))g(=)g(j{1)252 1715 y(MPI)p 344 1715 V 16 w(INQPID)h(\(gid,)f(neigh)p 754 1715 V 18 w(def,)g(neigh)p 964 1715 V 17 w(pid\(3\)\))252 1772 y(neigh)p 360 1772 V 17 w(def\(1\))g(=)g(i)252 1828 y(neigh)p 360 1828 V 17 w(def\(2\))g(=)g(j+1)252 1884 y(MPI)p 344 1884 V 16 w(INQPID)h(\(gid,)f(neigh)p 754 1884 V 18 w(def,)g(neigh)p 964 1884 V 17 w(pid\(4\)\))75 1941 y(C)89 b(Initialize)18 b(the)e(grid)f(functions)h(and)g(start)d(the)j(iteration)252 1997 y(call)g(init)g(\(u,)f(f)t(\))252 2054 y(do)g(10)g(it=1,100)322 2110 y(call)i(relax)e(\(u,)g(f)t(\))75 2167 y(C)89 b(Exc)o(hange)15 b(data)g(with)g(neigh)o(b)q(or)h(pro)q(cesses)322 2223 y(call)h(exc)o(hange)e(\(u,)g(neigh)p 779 2223 V 17 w(pid,)h(n)o(um)p 974 2223 V 17 w(neigh\))75 2280 y(10)131 b(con)o(tin)o(ue)252 2336 y(call)16 b(output)f(\(u\))252 2393 y(end)147 2597 y(Figure)g(4.1:)k(Set{up)d(of)e(pro)q(cess)i(structure)f(for)f(t)o(w)o (o{dimensional)i(parallel)h(P)o(oisson)e(solv)o(er.)p eop %%Page: 70 75 70 74 bop 75 357 a Fn(Chapter)34 b(5)75 566 y Fu(Language)41 b(Binding)817 789 y Fs(Scott)14 b(Berryman)857 846 y(Ewing)i(Lusk)75 994 y Fm(5.1)70 b(In)n(tro)r(duction)75 1097 y Fs(This)21 b(c)o(hapter)g(de\014nes)h(the)f(rules)g(for)g(MPI)f(language)h (binding)i(in)f(general)f(and)g(for)f(F)l(ortran)g(77,)75 1154 y(F)l(ortran)15 b(90,)g(ANSI)i(C,)e(and)i(ANSI)f(C++)h(in)g (particular.)23 b(De\014ned)17 b(here)f(are)g(v)m(arious)g(ob)s(ject)g (repre-)75 1210 y(sen)o(tations,)d(as)h(w)o(ell)g(as)f(the)h(naming)g (con)o(v)o(en)o(tions)g(used)g(for)f(expressing)i(this)f(standard.)19 b(The)14 b(actual)75 1267 y(calling)j(sequences)f(are)f(de\014ned)i (elsewhere.)166 1324 y(It)23 b(is)g(exp)q(ected)h(that)e(an)o(y)g(F)l (ortran)g(90)g(and)h(C++)g(implemen)o(tations)h(use)f(the)g(F)l(ortran) f(77)75 1381 y(and)f(ANSI)h(C)e(bindings,)k(resp)q(ectiv)o(ely)l(.)39 b(Although)22 b(w)o(e)e(consider)i(it)f(premature)g(to)f(de\014ne)i (other)75 1437 y(bindings)e(to)d(F)l(ortran)g(90)g(and)h(C++,)h(the)f (curren)o(t)f(bindings)j(are)e(designed)h(to)e(encourage,)i(rather)75 1493 y(than)c(discourage,)g(exp)q(erimen)o(tation)i(with)e(b)q(etter)g (bindings)i(whic)o(h)f(migh)o(t)f(b)q(e)h(adopted)g(later.)166 1551 y(There)i(are)g(sev)o(eral)g(imp)q(ortan)o(t)f(language)h(binding) i(issues)f(not)e(addressed)i(b)o(y)e(this)i(standard.)75 1607 y(It)f(is)g(b)q(ey)o(ond)h(the)f(scop)q(e)g(of)g(this)g(standard)g (to)f(discuss)i(the)f(in)o(terop)q(erabilit)o(y)i(of)d(message)h (passing)75 1664 y(b)q(et)o(w)o(een)c(languages.)19 b(Although)c(it)e (w)o(ould)i(b)q(e)f(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)h(to)e (guaran)o(tee)g(that)f(messages)75 1720 y(sen)o(t)k(b)q(et)o(w)o(een)h (languages)f(w)o(ere)h(prop)q(erly)g(in)o(terpreted,)g(the)f(matc)o (hing)h(of)f(t)o(yp)q(es)g(across)g(languages)75 1777 y(prev)o(en)o(ts)f(the)h(inclusion)i(of)d(suc)o(h)h(a)f(guaran)o(tee)g (in)i(this)f(standard.)k(It)c(is)g(fully)h(exp)q(ected)g(that)d(man)o (y)75 1833 y(implemen)o(tations)i(will)g(ha)o(v)o(e)f(suc)o(h)g (features,)f(and)h(that)f(suc)o(h)h(features)g(are)f(a)h(sign)g(of)f (the)h(qualit)o(y)g(of)75 1890 y(the)g(implemen)o(tation.)166 2030 y Fk(Discussion:)f Fj(There)f(w)o(as)e(some)g(informal)d (discussion)k(of)f(this)g(issue)h(at)g(the)g(last)f(meeting.)16 b(Is)c(it)f(p)q(ossible)75 2086 y(to)j(sa)o(y)f(something)g(stronger?) 75 2317 y Fm(5.2)70 b(Data)23 b(T)n(yp)r(es)g(and)g(Naming)f(Con)n(v)n (en)n(tions)75 2420 y Fs(All)15 b(named)f(constan)o(ts,)f(routine)h (names,)g(and)g(v)m(ariable)h(names)f(will)h(b)q(egin)g(with)f(the)g (letters)g(\\MPI".)75 2477 y(In)21 b(the)f(C)g(binding,)j(these)d(will) i(b)q(e)f(upp)q(er)g(case,)g(with)f(all)h(remaining)h(letters)e(in)h (lo)o(w)o(er)f(case.)34 b(In)75 2533 y(F)l(ortran,)14 b(the)h(en)o(tire)h(name)f(will)i(b)q(e)f(lo)o(w)o(er)e(case.)166 2591 y(Handles)e(are)g(represen)o(ted)f(as)h(in)o(tegers)f(in)h(F)l (ortran)f(and)g(as)g(p)q(oin)o(ters)h(to)f(structures)g(in)i(C.)d(T)o (yp)q(ed)75 2647 y(comm)o(unication)20 b(routines)g(supp)q(ort)g Ft(string)p Fs(s,)g Ft(double)j(precision)p Fs(,)e Ft(real)p Fs(,)f(and)g Ft(in)o(teger)f Fs(data)g(in)75 2704 y(F)l(ortran.)34 b(Null-terminated)23 b Ft(string)p Fs(s,)e Ft(double)p Fs(,)h Ft(real)p Fs(,)g Ft(short)p Fs(,)f Ft(c)o(har)p Fs(,)g(and)g Ft(long)g Fs(are)f(supp)q(orted)952 2828 y(70)p eop %%Page: 71 76 71 75 bop 75 -100 a Fq(5.3.)34 b(F)o(OR)l(TRAN)16 b(77)f(BINDING)g (ISSUES)971 b Fs(71)290 81 y Fl(double)23 b(precision)f(a)290 138 y(call)h(foo\(a\))385 194 y(.)385 251 y(.)385 307 y(.)290 363 y(subroutine)f(foo\(b\))290 420 y(integer)h(b)75 568 y Fs(Figure)e(5.1:)29 b(This)21 b(is)g(an)f(example)h(of)f(calling) i(a)e(routine)h(with)g(mismatc)o(hed)f(formal)g(and)h(actual)75 624 y(parameters.)75 758 y(in)f(C.)e(In)i(addition,)g(a)f(sp)q(ecial)h Ft(b)o(yte)f Fs(t)o(yp)q(e)g(is)g(supp)q(orted)g(in)h(b)q(oth)f (languages.)31 b(This)20 b(t)o(yp)q(e)f(allo)o(ws)75 814 y(comm)o(unication)h(to)f(b)q(e)h(p)q(erformed)g(with)g(no)f(t)o (yp)q(e)h(con)o(v)o(ersion)g(at)f(all.)33 b(In)21 b(C,)d(the)i (de\014nitions)h(of)75 871 y(named)14 b(constan)o(ts,)f(function)i (protot)o(yp)q(es,)f(and)g(aggregate)f(t)o(yp)q(e)h(information)g(m)o (ust)g(b)q(e)h(supplied)h(in)75 927 y(an)f(include)j(\014le)e Fl(mpi.h)p Fs(.)75 1071 y Fm(5.3)70 b(F)-6 b(ortran)24 b(77)g(Binding)e(Issues)75 1173 y Fs(There)17 b(are)g(sev)o(eral)g(p)q (oin)o(ts)h(w)o(ere)f(this)g(standard)g(div)o(erges)h(from)e(the)h (ANSI)h(F)l(ortran)e(77)h(standard.)75 1229 y(Unless)j(explicitly)i (stated,)d(the)h(MPI)f(F77)f(binding)j(is)f(consisten)o(t)f(with)g (ANSI)h(standard)f(F)l(ortran)75 1286 y(77.)26 b(These)18 b(exceptions)g(are)g(consisten)o(t)f(with)h(common)f(practice)h(in)h (the)e(F)l(ortran)f(comm)o(unit)o(y)l(.)27 b(In)75 1342 y(particular:)143 1436 y Fo(\017)c Fs(Iden)o(ti\014ers)16 b(are)f(limited)i(to)e(31,)f(not)h(six,)g(signi\014can)o(t)h(c)o (haracters.)143 1531 y Fo(\017)23 b Fs(Iden)o(ti\014ers)16 b(ma)o(y)f(con)o(tain)g(underscores)h(after)e(the)h(\014rst)g(c)o (haracter.)143 1625 y Fo(\017)23 b Fs(The)14 b(t)o(yp)q(es)f(of)g(the)h (formal)f(parameter)g(list)i(ma)o(y)e(not)g(matc)o(h)g(the)h(calling)h (parameter)e(list.)20 b(\(An)189 1681 y(example)c(is)f(giv)o(en)h(in)g (\014gure)f(5.3.\))143 1776 y Fo(\017)23 b Fs(Although)16 b(not)g(required,)g(it)h(is)f(strongly)g(suggested)f(that)h(named)g (constan)o(ts)f(b)q(e)h(pro)o(vided)h(in)189 1832 y(an)e(include)i (\014le.)75 2014 y Fm(5.4)70 b(C)22 b(Binding)g(Issues)75 2115 y Fs(W)l(e)h(use)g(the)f(ANSI)i(C)e(declaration)h(format.)41 b(All)24 b(MPI)f(names)f(ha)o(v)o(e)g(an)h Fl(MPI)p 1548 2115 15 2 v 39 w Fs(pre\014x,)i(de\014ned)75 2172 y(constan)o(ts)12 b(are)h(in)h(all)h(capital)f(letters,)f(and)g(de\014ned)i(t)o(yp)q(es)e (ha)o(v)o(e)g(one)h(capital)g(letter)f(after)g(the)g(pre\014x.)75 2316 y Fm(5.5)70 b(Sp)r(eci\014c)21 b(Bindings)75 2500 y Fk(Discussion:)13 b Fj(What)c(w)o(ould)g(b)q(e)h(the)g(b)q(est)h(w)o (a)o(y)e(to)g(presen)o(t)i(the)f(follo)o(wing)d(information?)14 b(One)c(big)f(alphab)q(etical)75 2556 y(list?)16 b(In)10 b(order)h(of)e(presen)o(tation)i(in)e(the)i(individual)d(c)o(hapters?) 18 b(By)10 b(c)o(hapter)h(but)f(alphab)q(etical)f(within)g(c)o(hapter?) 75 2613 y(C)14 b(and)g(F)m(ortran)f(mixed)g(or)g(separate?)p eop %%Page: 72 77 72 76 bop 75 -100 a Fs(72)954 b Fq(CHAPTER)16 b(5.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fi(5.5.1)55 b(C)19 b(bindings)g(for)g(P)n(oin)n(t-to-P)n(oin)n(t)g(Comm)n(unication)75 138 y Fs(These)d(are)e(in)i(alphab)q(etical)i(order.)75 199 y Fl(int)23 b(MPI)p 245 199 15 2 v 17 w(append)p 406 199 V 17 w(contiguous\()f(MPI)p 781 199 V 17 w(Bd)h(bd,)h(void*)f (start,)g(int)g(count,)393 255 y(MPI)p 468 255 V 17 w(Datatype)g (datatype)f(\))75 345 y(int)h(MPI)p 245 345 V 17 w(append)p 406 345 V 17 w(hindexed\()f(MPI)p 733 345 V 17 w(Bd)i(bd,)f(void*)g (start,)g(int)h(count,)393 402 y(int)g(*index)p 636 402 V 16 w(array,)f(MPI)p 891 402 V 17 w(Datatype)g(datatype)f(\))75 492 y(int)h(MPI)p 245 492 V 17 w(append)p 406 492 V 17 w(hvec\()g(MPI)p 638 492 V 17 w(Bd)g(bd,)h(void*)f(start,)g(int)g (count,)g(int)h(stride,)393 548 y(int)g(lenblk,)f(MPI)p 755 548 V 16 w(Datatype)g(datatype)g(\))75 639 y(int)g(MPI)p 245 639 V 17 w(append)p 406 639 V 17 w(indexed\()f(MPI)p 709 639 V 17 w(Bd)i(bd,)f(void*)g(start,)g(int)h(count,)f(int)g(*index) p 1705 639 V 17 w(array,)393 695 y(MPI)p 468 695 V 17 w(Datatype)g(datatype)f(\))75 785 y(int)h(MPI)p 245 785 V 17 w(append)p 406 785 V 17 w(vec\()g(MPI)p 614 785 V 17 w(Bd)g(bd,)h(int)f(count,)g(int)h(stride,)f(int)g(lenblk,)393 842 y(MPI)p 468 842 V 17 w(Datatype)g(datatype)f(\))75 932 y(int)h(MPI)p 245 932 V 17 w(associated\(MPI)p 598 932 V 15 w(Handle)h(handle,)e(MPI)p 1043 932 V 17 w(Handle)h(handle)p 1371 932 V 17 w(type)g(\))75 1022 y(int)g(MPI)p 245 1022 V 17 w(cancel\()g(MPI)p 525 1022 V 17 w(Handle)g(handle,)g(int)g(flag)h (\))75 1113 y(int)f(MPI)p 245 1113 V 17 w(commit)p 406 1113 V 17 w(buffer\()f(MPI)p 685 1113 V 17 w(Bd)i(bd)f(\))75 1203 y(int)g(MPI)p 245 1203 V 17 w(create)p 406 1203 V 17 w(status\()f(MPI)p 685 1203 V 17 w(Handle)h(handle)g(\))75 1293 y(int)g(MPI)p 245 1293 V 17 w(create)p 406 1293 V 17 w(buffer\()f(MPI)p 685 1293 V 17 w(Bd)i(*bd,)f(MPI)p 965 1293 V 17 w(LIFETIME)g(persistence)f(\))75 1383 y(int)h(MPI)p 245 1383 V 17 w(free\()g(MPI)p 477 1383 V 17 w(Handle)g(handle)g(\))75 1474 y(int)g(MPI)p 245 1474 V 17 w(get)p 334 1474 V 17 w(len\()g(int)h(count,)f(MPI)p 805 1474 V 17 w(Status)g(*status,)f(MPI) p 1275 1474 V 17 w(Bd)i(bd)f(\))75 1564 y(int)g(MPI)p 245 1564 V 17 w(init)p 358 1564 V 17 w(recv)g(\()h(MPI)p 614 1564 V 17 w(Handle)f(handle,)g(MPI)p 1061 1564 V 16 w(Bd)h(bd,)f(int)h(source,)f(MPI)p 1603 1564 V 17 w(Tag)g(tag,)393 1620 y(MPI)p 468 1620 V 17 w(Communicator)f(context,)h (MPI)p 1082 1620 V 17 w(LIFETIME)f(persistence)h(\))75 1710 y(int)g(MPI)p 245 1710 V 17 w(init)p 358 1710 V 17 w(rsend)g(\()h(MPI)p 638 1710 V 17 w(Handle)f(handle,)f(MPI)p 1084 1710 V 17 w(Bd)i(bd,)f(int)h(dest,)f(MPI)p 1579 1710 V 17 w(Tag)g(tag,)393 1767 y(MPI)p 468 1767 V 17 w(Communicator)f(context,)h(MPI)p 1082 1767 V 17 w(LIFETIME)f (persistence)h(\))75 1857 y(int)g(MPI)p 245 1857 V 17 w(init)p 358 1857 V 17 w(send)g(\()h(MPI)p 614 1857 V 17 w(Handle)f(handle,)g(MPI)p 1061 1857 V 16 w(Bd)h(bd,)f(int)h(dest,)f (MPI)p 1555 1857 V 17 w(Tag)g(tag,)393 1914 y(MPI)p 468 1914 V 17 w(COMunicator)f(context,)h(MPI)p 1058 1914 V 17 w(LIFETIME)g(persistence)f(\))75 2004 y(int)h(MPI)p 245 2004 V 17 w(init)p 358 2004 V 17 w(ssend)g(\()h(MPI)p 638 2004 V 17 w(Handle)f(handle,)f(MPI)p 1084 2004 V 17 w(Bd)i(bd,)f(int)h(dest,)f(MPI)p 1579 2004 V 17 w(Tag)g(tag,)393 2060 y(MPI)p 468 2060 V 17 w(Communicator)f(context,)h(MPI)p 1082 2060 V 17 w(LIFETIME)f(persistence)h(\))75 2151 y(int)g(MPI)p 245 2151 V 17 w(iprobe\()g(int)h(source,)e(MPI)p 811 2151 V 17 w(Tag)i(tag,)f(MPI)p 1115 2151 V 17 w(Communicator)f (context,)h(int)g(flag,)393 2207 y(type,)g(MPI)p 611 2207 V 17 w(Status)g(*status)g(\))75 2297 y(int)g(MPI)p 245 2297 V 17 w(irecv\()g(MPI)p 501 2297 V 17 w(Handle)g(handle,)g(MPI) p 948 2297 V 17 w(Bd)g(bd,)h(int)f(source,)g(MPI)p 1490 2297 V 17 w(Tag)g(tag,)393 2354 y(MPI)p 468 2354 V 17 w(Communicator)f(context)h(\))75 2444 y(int)g(MPI)p 245 2444 V 17 w(irecvc\()g(MPI)p 525 2444 V 17 w(Handle)g(handle,)g(void*)g (start,)g(int)h(count,)393 2500 y(MPI)p 468 2500 V 17 w(Datatype)f(datatype,)f(int)i(source,)f(MPI)p 1297 2500 V 16 w(Tag)h(tag,)393 2557 y(MPI)p 468 2557 V 17 w(Communicator)e (context,)h(MPI)p 1082 2557 V 17 w(Status)g(*status)g(\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(irsend)g(\()h(MPI)p 549 2647 V 17 w(Handle)f(handle,)g(MPI)p 996 2647 V 16 w(Bd)h(bd,)g(int)f(dest,)g(MPI)p 1490 2647 V 17 w(Tag)g(tag,)393 2704 y(MPI)p 468 2704 V 17 w(Communicator)f(context)h(\))p eop %%Page: 73 78 73 77 bop 75 -100 a Fq(5.5.)34 b(SPECIFIC)15 b(BINDINGS)1191 b Fs(73)75 45 y Fl(int)23 b(MPI)p 245 45 15 2 v 17 w(irsendc)g(\()h (MPI)p 573 45 V 17 w(Handle)f(handle,)g(void*)g(start,)g(int)g(count,) 393 102 y(MPI)p 468 102 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f (MPI)p 1249 102 V 17 w(Tag)g(tag,)393 158 y(MPI)p 468 158 V 17 w(Communicator)f(context)h(\))75 246 y(int)g(MPI)p 245 246 V 17 w(isend)g(\()h(MPI)p 525 246 V 17 w(Handle)f(handle,)g (MPI)p 972 246 V 17 w(Bd)g(bd,)h(int)f(dest,)g(MPI)p 1466 246 V 17 w(Tag)h(tag,)393 302 y(MPI)p 468 302 V 17 w(Communicator)e(context)h(\))75 390 y(int)g(MPI)p 245 390 V 17 w(isendc)g(\()h(MPI)p 549 390 V 17 w(Handle)f(handle,)g (void*)g(start,)g(int)g(count,)393 446 y(MPI)p 468 446 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f(MPI)p 1249 446 V 17 w(Tag)g(tag,)393 503 y(MPI)p 468 503 V 17 w(Communicator)f (context)h(\))75 591 y(int)g(MPI)p 245 591 V 17 w(issend)g(\()h(MPI)p 549 591 V 17 w(Handle)f(handle,)g(MPI)p 996 591 V 16 w(Bd)h(bd,)g(int)f(dest,)g(MPI)p 1490 591 V 17 w(Tag)g(tag,)393 647 y(MPI)p 468 647 V 17 w(Communicator)f(context)h(\))75 735 y(int)g(MPI)p 245 735 V 17 w(issendc)g(\()h(MPI)p 573 735 V 17 w(Handle)f(handle,)g(void*)g(start,)g(int)g(count,)393 791 y(MPI)p 468 791 V 17 w(Datatype)g(datatype,)f(int)i(dest,)f(MPI)p 1249 791 V 17 w(Tag)g(tag,)393 848 y(MPI)p 468 848 V 17 w(Communicator)f(context)h(\))75 936 y(int)g(MPI)p 245 936 V 17 w(probe\()g(int)h(source,)f(MPI)p 788 936 V 16 w(Tag)h(tag,)f(MPI)p 1091 936 V 17 w(Communicator)f(context,)393 992 y(MPI)p 468 992 V 17 w(Datatype)h(datatype,)f(MPI)p 1010 992 V 17 w(Status)h(*status)g(\))75 1080 y(int)g(MPI)p 245 1080 V 17 w(query\()g(MPI)p 501 1080 V 17 w(Handle)g(handle,)g(int) g(count,)g(int)h(source,)f(MPI)p 1497 1080 V 17 w(Tag)g(tag\))75 1168 y(int)g(MPI)p 245 1168 V 17 w(recv\()g(MPI)p 477 1168 V 17 w(Bd)h(bd,)f(int)h(source,)f(MPI)p 1020 1168 V 16 w(Tag)h(tag,)f(MPI)p 1323 1168 V 17 w(Communicator)f(context,)393 1224 y(MPI)p 468 1224 V 17 w(Status)h(*status)g(\))75 1312 y(int)g(MPI)p 245 1312 V 17 w(recvc)g(\()h(void*)f(start,)g(int)h (count,)f(MPI)p 1098 1312 V 17 w(Datatype)f(datatype,)h(int)g(source,) 393 1368 y(MPI)p 468 1368 V 17 w(Tag)g(tag,)h(MPI)p 772 1368 V 16 w(Communicator)f(context,)f(MPI)p 1385 1368 V 17 w(Status)h(*status)g(\))75 1456 y(int)g(MPI)p 245 1456 V 17 w(rsend)g(\()h(MPI)p 525 1456 V 17 w(Bd)f(bd,)h(int)f(dest,)h (MPI)p 1020 1456 V 16 w(Tag)g(tag,)f(MPI)p 1323 1456 V 17 w(Communicator)f(context)h(\))75 1544 y(int)g(MPI)p 245 1544 V 17 w(rsendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1122 1544 V 16 w(Datatype)g(datatype,)g(int)g(dest,)393 1600 y(MPI)p 468 1600 V 17 w(Tag)g(tag,)h(MPI)p 772 1600 V 16 w(Communicator)f(context)g(\))75 1688 y(int)g(MPI)p 245 1688 V 17 w(send)h(\()f(MPI)p 501 1688 V 17 w(Bd)h(bd,)f(int)h (dest,)f(MPI)p 996 1688 V 16 w(Tag)h(tag,)f(MPI)p 1299 1688 V 17 w(Communicator)f(context)h(\))75 1776 y(int)g(MPI)p 245 1776 V 17 w(sendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1098 1776 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 1832 y(MPI)p 468 1832 V 17 w(Tag)g(tag,)h(MPI)p 772 1832 V 16 w(Communicator)f(context)g(\))75 1920 y(int)g(MPI)p 245 1920 V 17 w(ssend)g(\()h(MPI)p 525 1920 V 17 w(Bd)f(bd,)h(int)f (dest,)h(MPI)p 1020 1920 V 16 w(Tag)g(tag,)f(MPI)p 1323 1920 V 17 w(Communicator)f(context)h(\))75 2008 y(int)g(MPI)p 245 2008 V 17 w(ssendc)g(\()h(void*)f(start,)g(int)h(count,)f(MPI)p 1122 2008 V 16 w(Datatype)g(datatype,)g(int)g(dest,)393 2064 y(MPI)p 468 2064 V 17 w(Tag)g(tag,)h(MPI)p 772 2064 V 16 w(Communicator)f(context)g(\))75 2152 y(int)g(MPI)p 245 2152 V 17 w(start\()g(MPI)p 501 2152 V 17 w(Handle)g(handle)g(\))75 2240 y(int)g(MPI)p 245 2240 V 17 w(Status)g(\()h(MPI)p 549 2240 V 17 w(Handle)f(handle,)g(flag,)g(MPI)p 1139 2240 V 17 w(Status)g(*status)g(\))75 2327 y(int)g(MPI)p 245 2327 V 17 w(Statusany)g(\()h(MPI)p 621 2327 V 16 w(Handle)f(*handle)p 972 2327 V 17 w(list,)g(int)g(count,)g(int)h (index,)393 2384 y(MPI)p 468 2384 V 17 w(Status)f(*status)g(\))75 2472 y(int)g(MPI)p 245 2472 V 17 w(wait)h(\()f(MPI)p 501 2472 V 17 w(Handle)g(handle,)g(MPI)p 948 2472 V 17 w(Status)g(*status)g(\))75 2559 y(int)g(MPI)p 245 2559 V 17 w(waitall\()g(MPI)p 549 2559 V 17 w(Handle)g(*handle)p 901 2559 V 16 w(list,)g(int)h(count,)f(list)p 1419 2559 V 16 w(of)p 1483 2559 V 17 w(return)p 1644 2559 V 17 w(handles\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(waitany)g(\()h(MPI)p 573 2647 V 17 w(Handle)f(*handle)p 925 2647 V 16 w(list,)g(int)h (count,)f(int)g(index,)393 2704 y(MPI)p 468 2704 V 17 w(Status)g(*status)g(\))p eop %%Page: 74 79 74 78 bop 75 -100 a Fs(74)954 b Fq(CHAPTER)16 b(5.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fi(5.5.2)55 b(C)19 b(Bindings)g(for)f(Collectiv)n(e)f(Comm)n(unication)g(Chapter)75 135 y Fs(These)f(are)e(in)i(order)f(of)g(presen)o(tation)g(in)h(the)g (Collectiv)o(e)g(Comm)o(unication)g(c)o(hapter.)75 194 y Fl(int)23 b(MPI)p 245 194 15 2 v 17 w(comm)p 358 194 V 17 w(init\()g(MPI)p 590 194 V 17 w(Group)g(group)g(\))75 283 y(int)g(MPI)p 245 283 V 17 w(rank\()g(MPI)p 477 283 V 17 w(Group)g(group,)g(int)h(rank)f(\))75 372 y(int)g(MPI)p 245 372 V 17 w(gsize\()g(MPI)p 501 372 V 17 w(Group)g(group,)g(int)h (size)f(\))75 461 y(int)g(MPI)p 245 461 V 17 w(barrier\()g(MPI)p 549 461 V 17 w(Group)g(group)g(\))75 549 y(int)g(MPI)p 245 549 V 17 w(synch\()g(MPI)p 501 549 V 17 w(Group)g(group,)g(MPI)p 900 549 V 17 w(MAP)g(map)h(\))75 638 y(int)f(MPI)p 245 638 V 17 w(bcast\()g(MPI)p 501 638 V 17 w(Bd)h(bd,)f(MPI)p 757 638 V 17 w(Group)g(group,)g(int)h(root)f(\))75 727 y(int)g(MPI)p 245 727 V 17 w(bcastc\()g(buf,)g(int)h(len,)f(MPI)p 859 727 V 17 w(Datatype)g(datatype,)f(MPI)p 1401 727 V 17 w(Group)h(group,)393 783 y(int)h(root)f(\))75 872 y(int)g(MPI)p 245 872 V 17 w(gather\()g(MPI)p 525 872 V 17 w(Bd)g(inbd,)h(list)p 853 872 V 16 w(of)p 917 872 V 17 w(MPI)p 1006 872 V 17 w(Bd)g(outbds,)e(MPI)p 1357 872 V 17 w(Group)h(group,)g(int)h(root)393 929 y(\))75 1017 y(int)f(MPI)p 245 1017 V 17 w(gatherc\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)f(MPI)p 1408 1017 V 17 w(Datatype)g(datatype,)393 1074 y(MPI)p 468 1074 V 17 w(Group)g(group,)g(int)h(root)f(\))75 1163 y(int)g(MPI)p 245 1163 V 17 w(scatter\()g(list)p 573 1163 V 17 w(of)p 638 1163 V 17 w(MPI)p 727 1163 V 16 w(Bd)h(inbds,)f(MPI)p 1054 1163 V 17 w(Bd)g(outbd,)g(MPI)p 1381 1163 V 17 w(Group)g(group,)393 1219 y(int)h(root)f(\))75 1308 y(int)g(MPI)p 245 1308 V 17 w(scatterc\()g(void*)g(inbuf,)g(void*) g(outbuf,)g(int)h(len,)393 1364 y(MPI)p 468 1364 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1364 V 17 w(Group)h(group,)g(int)h (root)f(\))75 1453 y(int)g(MPI)p 245 1453 V 17 w(allscatter\()g(MPI)p 621 1453 V 16 w(Bd)h(inbds[],)f(MPI)p 996 1453 V 16 w(Bd)h(outbds[],)f (MPI)p 1395 1453 V 16 w(Group)h(group)f(\))75 1542 y(int)g(MPI)p 245 1542 V 17 w(allscatterc\()f(void*)i(inbuf,)f(void*)g(outbuf,)g(int) g(len,)393 1598 y(MPI)p 468 1598 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1598 V 17 w(Group)h(group)g(\))75 1687 y(int)g(MPI)p 245 1687 V 17 w(allcast\()g(MPI)p 549 1687 V 17 w(Bd)g(inbds[],)g(MPI)p 924 1687 V 17 w(Bd)g(outbds[],)g(MPI)p 1323 1687 V 17 w(Group)g(group)g(\))75 1776 y(int)g(MPI)p 245 1776 V 17 w(allcastc\()g(void*)g(inbuf,)g(void*)g(outbuf,)g(int)h(len,)393 1832 y(MPI)p 468 1832 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1832 V 17 w(Group)h(group)g(\))75 1921 y(int)g(MPI)p 245 1921 V 17 w(reduce\()g(MPI)p 525 1921 V 17 w(Bd)g(inbd,)h(MPI)p 829 1921 V 16 w(Bd)g(outbd,)f(MPI)p 1156 1921 V 17 w(Group)g(group,)g (int)h(root,)393 1977 y(MPI)p 468 1977 V 17 w(Op)g(op)f(\))75 2066 y(int)g(MPI)p 245 2066 V 17 w(reducec\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)f(MPI)p 1408 2066 V 17 w(Datatype)g(datatype,)393 2123 y(MPI)p 468 2123 V 17 w(Group)g(group,)g(int)h(root,)f(MPI)p 1106 2123 V 17 w(Op)g(op)h(\))75 2211 y(int)f(MPI)p 245 2211 V 17 w(user)p 358 2211 V 17 w(reduce\()g(MPI)p 638 2211 V 17 w(Bd)g(inbd,)g(MPI)p 941 2211 V 17 w(Bd)h(outbd,)f(MPI)p 1269 2211 V 16 w(Group)h(group,)f(int)g(root,)393 2268 y(void)g(*userfunc\(\),)g(int)g(unitsize)g(\))75 2357 y(int)g(MPI)p 245 2357 V 17 w(user)p 358 2357 V 17 w(reducec\()g(void*) g(inbuf,)g(void*)g(outbuf,)g(int)g(len,)393 2413 y(MPI)p 468 2413 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2413 V 17 w(Group)h(group,)g(int)h(root,)393 2470 y(void)f(*userfunc\(\),)g (unitsize)f(\))75 2558 y(int)h(MPI)p 245 2558 V 17 w(allreduce\()g(MPI) p 597 2558 V 16 w(Bd)h(inbd,)f(MPI)p 900 2558 V 17 w(Bd)h(outbd,)f(MPI) p 1228 2558 V 16 w(Group)h(group,)f(MPI)p 1627 2558 V 16 w(Op)h(op)g(\))75 2647 y(int)f(MPI)p 245 2647 V 17 w(allreducec\()g(void*)g(inbuf,)g(void*)g(outbuf,)g(int)g(len,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2704 V 17 w(Group)h(group,)g(MPI)p 1409 2704 V 17 w(Op)h(op)f(\))p eop %%Page: 75 80 75 79 bop 75 -100 a Fq(5.5.)34 b(SPECIFIC)15 b(BINDINGS)1191 b Fs(75)75 45 y Fl(int)23 b(MPI)p 245 45 15 2 v 17 w(user)p 358 45 V 17 w(allreduce\()f(MPI)p 709 45 V 17 w(Bd)i(inbd,)f(MPI)p 1013 45 V 17 w(Bd)g(outbd,)g(MPI)p 1340 45 V 17 w(Group)g(group,)393 102 y(void)g(*userfunc\(\)\))75 189 y(int)g(MPI)p 245 189 V 17 w(user)p 358 189 V 17 w(allreducec\()f(void*)h(inbuf,)g(void*) h(outbuf,)f(int)g(len,)393 246 y(MPI)p 468 246 V 17 w(Datatype)g (datatype,)f(MPI)p 1010 246 V 17 w(Group)h(group,)g(void)h (*userfunc\(\)\))75 333 y(int)f(MPI)p 245 333 V 17 w(scan\()g(MPI)p 477 333 V 17 w(Bd)h(inbd,)f(MPI)p 781 333 V 17 w(Bd)g(outbd,)g(MPI)p 1108 333 V 17 w(Group)g(group,)g(MPI)p 1507 333 V 17 w(Op)h(op)f(\))75 421 y(int)g(MPI)p 245 421 V 17 w(scanc\()g(void*)g (inbuf,)g(void*)h(outbuf,)f(int)g(len,)g(MPI)p 1360 421 V 17 w(Datatype)g(datatype,)393 477 y(MPI)p 468 477 V 17 w(Group)g(group,)g(MPI)p 867 477 V 17 w(Op)h(op)f(\))75 565 y(int)g(MPI)p 245 565 V 17 w(user)p 358 565 V 17 w(scan\()g(MPI)p 590 565 V 17 w(Bd)g(inbd,)h(MPI)p 894 565 V 16 w(Bd)g(outbd,)f(MPI)p 1221 565 V 17 w(Group)g(group,)393 621 y(void)g(*userfunc\(\))g(\))75 709 y(int)g(MPI)p 245 709 V 17 w(user)p 358 709 V 17 w(scanc\()g(void*)g(inbuf,)g(void*)g (outbuf,)g(int)h(len,)393 765 y(MPI)p 468 765 V 17 w(Datatype)f (datatype,)f(MPI)p 1010 765 V 17 w(Group)h(group,)g(void)h (*userfunc\(\)\))75 924 y Fi(5.5.3)55 b(F)-5 b(ortran)19 b(Bindings)g(for)f(P)n(oin)n(t-to-P)n(oin)n(t)i(Routines)75 1012 y Fl(mpi)p 150 1012 V 17 w(append)p 311 1012 V 16 w(contiguous\()i(bd,)i(start,)f(count,)g(datatype,)g(rc)g(\))170 1068 y(real)h(start\(*\))170 1125 y(integer)f(bd,)h(count,)f(datatype,) f(rc)75 1212 y(mpi)p 150 1212 V 17 w(append)p 311 1212 V 16 w(hindexed\()h(bd,)g(start,)g(count,)g(index)p 1115 1212 V 17 w(array,)g(datatype,)f(rc)i(\))170 1269 y(real)g(start\(*\)) 170 1325 y(integer)f(index)p 484 1325 V 17 w(array\(*\))170 1382 y(integer)g(bd,)h(count,)f(datatype,)f(rc)75 1469 y(mpi)p 150 1469 V 17 w(append)p 311 1469 V 16 w(hvec\()h(bd,)h(start,) f(count,)g(stride,)g(lenblk,)g(datatype,)f(rc)i(\))170 1526 y(real)g(start\(*\))170 1582 y(integer)f(bd,)h(count,)f(stride,)g (lenblk,)g(datatype,)f(rc)75 1670 y(mpi)p 150 1670 V 17 w(append)p 311 1670 V 16 w(indexed\()h(bd,)g(start,)g(count,)g (index)p 1091 1670 V 17 w(array,)g(datatype,)g(rc)g(\))170 1726 y(real)h(start\(*\))170 1783 y(integer)f(index)p 484 1783 V 17 w(array\(*\))170 1839 y(integer)g(bd,)h(count,)f (datatype,)f(rc)75 1927 y(mpi)p 150 1927 V 17 w(append)p 311 1927 V 16 w(vec\()h(bd,)h(count,)f(stride,)g(lenblk,)g(datatype,)f (rc)i(\))170 1983 y(integer)f(bd,)h(count,)f(stride,)g(lenblk,)g (datatype,)f(rc)75 2071 y(mpi)p 150 2071 V 17 w(associated\()g(handle,) h(handle)p 788 2071 V 16 w(type,)h(rc)f(\))170 2127 y(integer)g (handle,)g(handle)p 699 2127 V 17 w(type,)g(rc)75 2215 y(mpi)p 150 2215 V 17 w(cancel\()g(handle,)g(flag,)g(rc)g(\))170 2271 y(integer)g(handle,)g(flag,)g(rc)75 2359 y(mpi)p 150 2359 V 17 w(commit)p 311 2359 V 16 w(buffer\()g(bd,)g(rc)h(\))170 2415 y(integer)f(bd,)h(rc)75 2503 y(mpi)p 150 2503 V 17 w(create)p 311 2503 V 16 w(buffer\()f(bd,)g(persistence,)g(rc)g(\)) 170 2560 y(integer)g(bd,)h(persistence,)e(rc)75 2647 y(mpi)p 150 2647 V 17 w(create)p 311 2647 V 16 w(status\()h(handle,)g (rc)h(\))170 2704 y(integer)f(handl,)g(rce)p eop %%Page: 76 81 76 80 bop 75 -100 a Fs(76)954 b Fq(CHAPTER)16 b(5.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fl(mpi)p 150 45 15 2 v 17 w(free\()23 b(handle,)g(rc)g(\))170 102 y(integer)g(handl,)g (rce)75 190 y(mpi)p 150 190 V 17 w(get)p 239 190 V 17 w(len\()g(count,)g(status,)g(bd,)g(rc)h(\))170 247 y(integer)f(count,)g (status,)g(bd,)h(rc)75 335 y(mpi)p 150 335 V 17 w(init)p 263 335 V 16 w(recv)g(\()f(handle,)g(bd,)h(source,)f(tag,)g(context,)g (persistence,)f(rc)i(\))170 392 y(integer)f(handle,)g(bd,)h(source,)f (tag,)g(context,)g(persistence,)f(rc)75 480 y(mpi)p 150 480 V 17 w(init)p 263 480 V 16 w(rsend)i(\()f(handle,)g(bd,)h(dest,)f (tag,)g(context,)g(persistence,)f(rc)i(\))170 537 y(integer)f(handle,)g (bd,)h(dest,)f(tag,)g(context,)g(persistence,)f(rc)75 625 y(mpi)p 150 625 V 17 w(init)p 263 625 V 16 w(send)i(\()f(handle,)g (bd,)h(dest,)f(tag,)g(context,)g(persistence,)f(rc)i(\))170 681 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g(persistence,) f(rc)75 770 y(mpi)p 150 770 V 17 w(init)p 263 770 V 16 w(ssend)i(\()f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g(persistence,)f (rc)i(\))170 826 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g (persistence,)f(rc)75 915 y(mpi)p 150 915 V 17 w(iprobe\()h(source,)g (tag,)g(context,)g(flag,)g(type,)g(status,)g(rc)h(\))170 971 y(integer)f(source,)g(tag,)h(context,)e(flag,)i(type,)f(status,)g (rc)75 1060 y(mpi)p 150 1060 V 17 w(irecv\()g(handle,)g(bd,)g(source,)g (tag,)g(context,)g(rc)h(\))170 1116 y(integer)f(handle,)g(bd,)h (source,)f(tag,)g(context,)g(rc)75 1205 y(mpi)p 150 1205 V 17 w(irecvc\()g(handle,)g(start,)g(count,)g(datatype,)f(source,)h (tag,)h(context,)e(status,)393 1261 y(rc)i(\))170 1318 y(real)g(start\(*\))170 1374 y(integer)f(handle,)g(count,)g(datatype,)g (source,)g(tag,)g(context,)g(status,)g(rc)75 1463 y(mpi)p 150 1463 V 17 w(irsend)g(\()h(handle,)e(bd,)i(dest,)f(tag,)g(context,)g (rc)h(\))170 1519 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g (rc)75 1608 y(mpi)p 150 1608 V 17 w(irsendc)g(\()g(handle,)g(start,)g (count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)i(\))170 1664 y(real)g(start\(*\))170 1721 y(integer)f(handle,)g(count,)g (datatype,)g(dest,)g(tag,)g(context,)g(rc)75 1809 y(mpi)p 150 1809 V 17 w(isend)g(\()h(handle,)f(bd,)g(dest,)g(tag,)h(context,)e (rc)i(\))170 1866 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(context,)g (rc)75 1954 y(mpi)p 150 1954 V 17 w(isendc)g(\()h(handle,)e(start,)h (count,)h(datatype,)e(dest,)h(tag,)h(context,)e(rc)i(\))170 2011 y(real)g(start\(*\))170 2067 y(integer)f(handle,)g(count,)g (datatype,)g(dest,)g(tag,)g(context,)g(rc)75 2156 y(mpi)p 150 2156 V 17 w(issend)g(\()h(handle,)e(bd,)i(dest,)f(tag,)g(context,)g (rc)h(\))170 2212 y(integer)f(handle,)g(bd,)h(dest,)f(tag,)g(contex,)g (rct)75 2301 y(mpi)p 150 2301 V 17 w(issendc)g(\()g(handle,)g(start,)g (count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)i(\))170 2357 y(real)g(start\(*\))170 2414 y(integer)f(handle,)g(count,)g (datatype,)g(dest,)g(tag,)g(context,)g(rc)75 2502 y(mpi)p 150 2502 V 17 w(probe\()g(source,)g(tag,)g(context,)g(datatype,)f (status,)h(rc)h(\))170 2559 y(integer)f(source,)g(tag,)h(context,)e (datatype,)h(status,)g(rc)75 2647 y(mpi)p 150 2647 V 17 w(query\()g(handle,)g(count,)g(source,)g(tag,)g(rc)h(\))170 2704 y(integer)f(handle,)g(count,)g(source,)g(tag,)h(rc)p eop %%Page: 77 82 77 81 bop 75 -100 a Fq(5.5.)34 b(SPECIFIC)15 b(BINDINGS)1191 b Fs(77)75 45 y Fl(mpi)p 150 45 15 2 v 17 w(recv\()23 b(bd,)g(source,)g(tag,)h(context,)e(status,)h(rc)h(\))170 102 y(integer)f(bd,)h(source,)f(tag,)g(context,)g(status,)g(rc)75 188 y(mpi)p 150 188 V 17 w(recvc)g(\(start,)g(count,)g(datatype,)g (source,)f(tag,)i(context,)f(statu,)g(rcs\))170 244 y(real)h (start\(*\))170 301 y(integer)f(count,)g(datatype,)g(source,)g(tag,)g (context,)g(status,)g(rc)75 387 y(mpi)p 150 387 V 17 w(rsend)g(\()h(bd,)f(dest,)g(tag,)h(context,)e(rc)i(\))170 444 y(integer)f(bd,)h(dest,)f(tag,)g(context,)g(rc)75 530 y(mpi)p 150 530 V 17 w(rsendc)g(\(start,)g(count,)g(datatype,)f (dest,)i(tag,)f(context,)g(rc)g(\))170 587 y(real)h(start\(*\))170 643 y(integer)f(count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)75 730 y(mpi)p 150 730 V 17 w(send)h(\()h(bd,)f(dest,)g(tag,)h(context,)f (rc)g(\))170 786 y(integer)g(bd,)h(dest,)f(tag,)g(context,)g(rc)75 873 y(mpi)p 150 873 V 17 w(sendc)g(\(start,)g(count,)g(datatype,)g (dest,)g(tag,)g(context,)g(rc)g(\))170 929 y(real)h(start\(*\))170 985 y(integer)f(count,)g(datatype,)g(dest,)g(tag,)h(context,)e(rc)75 1072 y(mpi)p 150 1072 V 17 w(ssend)h(\()h(bd,)f(dest,)g(tag,)h (context,)e(rc)i(\))170 1128 y(integer)f(bd,)h(dest,)f(tag,)g(context,) g(rc)75 1215 y(mpi)p 150 1215 V 17 w(ssendc)g(\(start,)g(count,)g (datatype,)f(dest,)i(tag,)f(context,)g(rc)g(\))170 1271 y(real)h(start\(*\))170 1328 y(integer)f(count,)g(datatype,)g(dest,)g (tag,)h(context,)e(rc)75 1414 y(mpi)p 150 1414 V 17 w(start\()h (handle,)g(rc)g(\))170 1471 y(integer)g(handle,)g(rc)75 1557 y(mpi)p 150 1557 V 17 w(status)g(\()h(handle,)e(flag,)i(status,)f (rc)g(\))170 1614 y(integer)g(handle,)g(flag,)g(status,)g(rc)75 1700 y(mpi)p 150 1700 V 17 w(statusany)f(\()i(handle)p 597 1700 V 16 w(list,)g(count,)f(index,)g(status,)g(rc)g(\))170 1756 y(integer)g(handle)p 508 1756 V 17 w(list\(*\))170 1813 y(integer)g(count,)g(index,)g(status,)g(rc)75 1899 y(mpi)p 150 1899 V 17 w(wait)g(\()h(handle,)f(status,)g(rc)g(\))170 1956 y(integer)g(handle,)g(status,)g(rc)75 2042 y(mpi)p 150 2042 V 17 w(waitall\()g(handle)p 526 2042 V 16 w(list,)g(count,)g (list)p 948 2042 V 17 w(of)p 1013 2042 V 17 w(return)p 1174 2042 V 16 w(handles,)g(rc)g(\))170 2099 y(integer)g(handle)p 508 2099 V 17 w(list\(*\))170 2155 y(integer)g(count)170 2212 y(integer)g(list)p 460 2212 V 17 w(of)p 525 2212 V 17 w(return)p 686 2212 V 16 w(handles\(*,)g(rc\))75 2298 y(mpi)p 150 2298 V 17 w(waitany)g(\()g(handle)p 549 2298 V 17 w(list,)g(count,)g(index,)g(status,)g(rc)h(\))170 2355 y(integer)f(handle)p 508 2355 V 17 w(list\(*\))170 2411 y(integer)g(count,)g(index,)g(status,)g(rc)75 2561 y Fi(5.5.4)55 b(F)-5 b(ortran)19 b(Bindings)g(for)f(Collectiv)n(e)f (Comm)n(unciation)g(Chapter)75 2647 y Fs(These)f(are)e(not)h(done)h(y)o (et.)75 2704 y Fl(mpi)p 150 2704 V 17 w(comm)p 263 2704 V 16 w(init\()24 b(group,)f(rc)g(\))p eop %%Page: 78 83 78 82 bop 75 -100 a Fs(78)954 b Fq(CHAPTER)16 b(5.)34 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fl(mpi)p 150 45 15 2 v 17 w(rank\()23 b(group,)g(rank,)g(rc)h(\))75 132 y(mpi)p 150 132 V 17 w(gsize\()f(group,)g(size,)g(rc)h(\))75 218 y(mpi)p 150 218 V 17 w(barrier\()f(group,)g(rc)g(\))75 304 y(mpi)p 150 304 V 17 w(synch\()g(group,)g(map,)g(rc)h(\))75 391 y(mpi)p 150 391 V 17 w(bcast\()f(buffer)p 478 391 V 16 w(handle,)g(group,)g(root,)g(rc)h(\))75 477 y(mpi)p 150 477 V 17 w(bcastc\()f(buf,)g(len,)g(type,)g(group,)h(root,)f(rc)g (\))75 564 y(mpi)p 150 564 V 17 w(gather\()g(inhandle,)f(list)p 692 564 V 17 w(of)p 757 564 V 17 w(outhandles,)g(group,)h(root,)g(rc)h (\))75 650 y(mpi)p 150 650 V 17 w(gatherc\()f(inbuf,)g(outbuf,)f (inlen,)i(type,)f(group,)g(root,)g(rc)h(\))75 737 y(mpi)p 150 737 V 17 w(scatter\()f(list)p 478 737 V 16 w(of)p 542 737 V 17 w(inhandles,)g(outhandle,)f(group,)h(root,)g(rc)h(\))75 823 y(mpi)p 150 823 V 17 w(scatterc\()e(inbuf,)h(outbuf,)g(len,)h (type,)f(group,)g(root,)g(rc)h(\))75 910 y(mpi)p 150 910 V 17 w(allscatter\()e(list)p 549 910 V 17 w(of)p 614 910 V 17 w(inhandles,)g(list)p 989 910 V 17 w(of)p 1054 910 V 17 w(outhandles,)g(group,)h(rc)h(\))75 996 y(mpi)p 150 996 V 17 w(allscatterc\()e(inbuf,)h(outbuf,)g(len,)g(type,) g(grou,)h(rcp\))75 1083 y(mpi)p 150 1083 V 17 w(allcast\()f(inhandle,)f (list)p 716 1083 V 17 w(of)p 781 1083 V 17 w(outhandles,)g(group,)h(rc) h(\))75 1169 y(mpi)p 150 1169 V 17 w(allcastc\()e(inbuf,)h(outbuf,)g (len,)h(type,)f(grou,)g(rcp\))75 1255 y(mpi)p 150 1255 V 17 w(reduce\()g(inhandle,)f(outhandle,)h(group,)g(root,)g(op,)g(rc)h (\))75 1342 y(mpi)p 150 1342 V 17 w(reducec\()f(inbuf,)g(outbuf,)f (len,)i(type,)f(group,)g(root,)g(op,)h(rc)f(\))75 1428 y(mpi)p 150 1428 V 17 w(user)p 263 1428 V 16 w(reduce\()g(inhandle,)g (outhandle,)f(group,)h(root,)h(function,)e(unitsize,)h(rc)h(\))75 1515 y(mpi)p 150 1515 V 17 w(user)p 263 1515 V 16 w(reducec\()f(inbuf,) g(outbuf,)g(len,)g(type,)h(group,)f(root,)g(function,)393 1571 y(unitsize,)g(rc)g(\))75 1658 y(mpi)p 150 1658 V 17 w(allreduce\()f(inhandle,)h(outhandle,)f(group,)h(op,)h(rc)f(\))75 1744 y(mpi)p 150 1744 V 17 w(allreducec\()f(inbuf,)h(outbuf,)g(len,)g (type,)h(group,)f(op,)g(rc)h(\))75 1831 y(mpi)p 150 1831 V 17 w(user)p 263 1831 V 16 w(allreduce\()f(inhandle,)g(outhandle,)f (group,)h(function,)g(rc)g(\))75 1917 y(mpi)p 150 1917 V 17 w(user)p 263 1917 V 16 w(allreducec\()g(inbuf,)g(outbuf,)g(len,)g (type,)g(group,)g(function,)g(rc)g(\))75 2004 y(mpi)p 150 2004 V 17 w(scan\()g(inhandle,)g(outhandle,)f(group,)h(op,)h(rc)f (\))75 2090 y(mpi)p 150 2090 V 17 w(scanc\()g(inbuf,)g(outbuf,)g(len,)g (type,)g(group,)g(op,)h(rc)f(\))75 2177 y(mpi)p 150 2177 V 17 w(user)p 263 2177 V 16 w(scan\()h(inhandle,)e(outhandle,)h(group,) g(function,)f(rc)i(\))75 2263 y(mpi)p 150 2263 V 17 w(user)p 263 2263 V 16 w(scanc\()f(inbuf,)g(outbuf,)g(len,)h(type,)f(group,)g (function,)g(rc)g(\))p eop %%Page: 79 84 79 83 bop 75 356 a Fn(Chapter)34 b(6)75 564 y Fu(Correctness)40 b(Issues)857 786 y Fs(Ewing)16 b(Lusk)166 880 y(This)k(c)o(hapter)g (will)i(ev)o(olv)o(e)e(to)f(include)j(discussions)g(and)e(examples)h (related)f(to)g(the)g(issue)g(of)75 937 y(correct)e(and)h(safe)g(MPI)g (programs.)29 b(It)19 b(ma)o(y)f(ev)o(en)o(tually)i(include)h(a)e (formal)f(sp)q(eci\014cation)j(for)d(the)75 993 y(seman)o(tics)f(of)f (MPI)g(routines.)25 b(F)l(or)16 b(the)g(time)h(b)q(eing,)h(in)g (includes)g(discussions)h(and)d(examples)i(that)75 1050 y(ha)o(v)o(e)d(up)g(to)g(no)o(w)g(app)q(eared)h(in)g(other)e(c)o (hapters.)75 1170 y Fi(6.0.5)55 b(Order)75 1256 y Fs(MPI)17 b(preserv)o(es)f(the)h(order)g(of)f(messages)g(b)q(et)o(w)o(een)h(an)o (y)g(\014xed)g(pair)g(of)g(pro)q(cesses.)24 b(In)18 b(other)e(w)o (ords,)75 1312 y(if)22 b(pro)q(cess)f(A)h(executes)f(t)o(w)o(o)f (successiv)o(e)j(send)f Fl(start)e Fs(sub)q(op)q(erations,)j(pro)q (cess)f(B)f(executes)h(t)o(w)o(o)75 1369 y(successiv)o(e)g(receiv)o(e)f Fl(start)f Fs(op)q(erations,)h(and)g(b)q(oth)g(receiv)o(es)g(matc)o(h)f (either)h(send,)h(then)f(the)f(\014rst)75 1425 y(receiv)o(e)e(will)h (receiv)o(e)g(the)e(message)g(sen)o(t)g(b)o(y)h(the)f(\014rst)g(send,)i (and)e(the)h(second)g(receiv)o(e)g(will)h(receiv)o(e)75 1482 y(the)h(message)g(sen)o(t)g(b)o(y)g(the)h(second)g(send.)35 b(Th)o(us,)21 b(if)g(t)o(w)o(o)e(messages)h(from)f(the)i(same)e(source) i(can)75 1538 y(satisfy)d(a)g(p)q(ending)i(receiv)o(e,)g(the)e(\014rst) g(message)g(sen)o(t)g(is)g(accepted;)i(if)f(a)f(message)g(can)g (satisfy)g(t)o(w)o(o)75 1594 y(p)q(ending)f(receiv)o(es,)f(the)f (\014rst)g(receiv)o(e)h(p)q(osted)f(is)h(satis\014ed.)166 1651 y(The)c(last)f(paragraph)g(assumes)g(that)g(the)h(send)g Fl(start)e Fs(op)q(erations)i(are)f(ordered)h(b)o(y)f(the)h(program)75 1707 y(order)20 b(at)f(pro)q(cess)h(A,)g(and)g(the)g(receiv)o(e)h Fl(start)e Fs(op)q(erations)h(are)f(ordered)i(b)o(y)e(the)h(program)f (order)75 1764 y(at)h(pro)q(cess)h(B.)g(If)g(a)g(pro)q(cess)g(is)g(m)o (ultithreaded)h(and)f(the)g(op)q(erations)h(are)e(executed)i(b)o(y)f (distinct)75 1820 y(threads,)d(then)g(the)g(seman)o(tics)f(of)h(the)f (threaded)h(system)g(ma)o(y)f(not)g(de\014ne)i(an)e(order)h(b)q(et)o(w) o(een)g(the)75 1877 y(t)o(w)o(o)c(op)q(erations,)h(in)h(whic)o(h)g (case)f(the)g(condition)i(is)f(v)o(oid.)75 1997 y Fi(6.0.6)55 b(Progress)19 b(and)g(F)-5 b(airness)75 2083 y Fs(W)l(e)20 b(can)g(mo)q(del)g(the)g(execution)h(of)e(MPI)h(programs)e(as)h(an)h (in)o(teraction)g(b)q(et)o(w)o(een)g(executing)h(pro-)75 2139 y(cesses)d(that)f(execute)h(eac)o(h)g(their)g(o)o(wn)f(program,)g (and)g(the)h Ft(comm)o(unication)j(subsystem)p Fs(.)26 b(The)75 2195 y(comm)o(unication)16 b(subsystem)g(ma)o(y)f(ha)o(v)o(e)g (v)m(arious)h(constrain)o(ts)g(on)f(the)h(amoun)o(t)f(of)g(resources)h (it)g(can)75 2252 y(use.)k(E.g.:)166 2308 y(Bounds)e(on)f(the)g(n)o(um) o(b)q(er)h(and)f(total)g(sizes)h(of)f(activ)o(e)h(comm)o(unication)g (ob)s(jects.)25 b(Suc)o(h)18 b(b)q(ound)75 2365 y(can)d(b)q(e)h (global,)g(p)q(er)f(no)q(de,)h(or)e(p)q(er)i(pair)g(of)e(comm)o (unicating)i(no)q(des.)166 2421 y(Bounds)e(on)g(the)g(n)o(um)o(b)q(er)g (and)h(total)e(sizes)i(of)e(messages)g(bu\013ered)i(in)f(the)g(system.) 19 b(Suc)o(h)c(b)q(ound)75 2478 y(can,)k(again,)g(b)q(e)g(global,)g(p)q (er)g(no)q(de,)g(or)f(p)q(er)h(pair)f(of)g(comm)o(unicating)h(no)q(de.) 30 b(In)19 b(addition,)h(a)e(mes-)75 2534 y(sage)f(ma)o(y)g(b)q(e)h (bu\013ered)g(at)f(the)g(sender,)i(at)d(the)i(receiv)o(er,)g(at)f(b)q (oth,)h(or)f(p)q(erhaps)h(at)f(another)g(place)75 2591 y(altogether.)166 2647 y(Th)o(us,)i(it)g(will)i(b)q(e)e(di\016cult)h (to)e(set)h(rules)g(on)g(resource)g(managemen)o(t)f(of)g(the)h(comm)o (unication)75 2704 y(subsystem.)29 b(Ho)o(w)o(ev)o(er,)17 b(it)i(is)g(generally)g(exp)q(ected)g(that)f(implemen)o(ters)h(will)h (pro)o(vide)f(information)952 2828 y(79)p eop %%Page: 80 85 80 84 bop 75 -100 a Fs(80)924 b Fq(CHAPTER)16 b(6.)29 b(CORRECTNESS)17 b(ISSUES)75 45 y Fs(on)e(the)g(mec)o(hanism)g(used)h (for)e(resource)h(allo)q(cation,)h(and)f(that)f(query)i(and)f(set)f (functions)i(will)h(allo)o(w)75 102 y(to)e(query)g(and)g(p)q(ossibly)i (con)o(trol)e(the)g(amoun)o(t)g(of)f(a)o(v)m(ailable)j(resources.)166 158 y(W)l(e)g(pro)o(vide)h(in)g(this)f(section)h(a)f(set)g(of)g (minimal)h(requiremen)o(ts)g(on)f(the)h(comm)o(unication)f(sub-)75 214 y(system.)26 b(Programs)15 b(that)i(execute)h(on)f(an)o(y)g (subsystem)g(that)f(ful\014ls)j(these)f(minimal)g(requiremen)o(ts)75 271 y(are)12 b Ft(safe)h Fs(and)g(will)h(p)q(ort)f(to)f(an)o(y)g(MPI)h (implemen)o(tation.)21 b Ft(Unsafe)13 b Fs(programs)e(ma)o(y)h(execute) i(on)e(some)75 327 y(MPI)k(implemen)o(tations,)h(dep)q(ending)h(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(and)g(the)g (implemen-)75 384 y(tation)k(used)g(for)g(the)g(MPI)g(comm)o(unication) h(subsystem.)34 b(Finally)22 b Ft(erroneous)d Fs(programs)g(nev)o(er)75 440 y(execute)e(correctly)l(.)22 b(\(While)17 b(it)f(is)g(desirable)i (to)d(detect)h(erroneous)g(programs,)e(it)i(is)h(not)e(p)q(ossible)j (to)75 497 y(do)13 b(so)f(at)g(compile)i(time,)g(and)f(often)f (prohibitiv)o(e)j(to)d(do)g(so)h(at)f(run)h(time.)19 b(Th)o(us,)13 b(this)g(do)q(cumen)o(t)h(do)q(es)75 553 y(not)g(sp)q(ecify)h(a)f(b)q(eha)o(vior)h(for)f(erroneous)g(programs,)f (although)h(the)g(desired)i(b)q(eha)o(vior)f(is)f(to)g(return)g(a)75 610 y(useful)i(error)f(message.\))131 716 y(1.)22 b(If)15 b(a)g(pro)q(cess)g(executes)h(an)f Fl(INIT)f Fs(op)q(eration,)h(then)g (the)h(op)q(eration)f(ev)o(en)o(tually)h(succeeds,)g(or)e(a)189 773 y Fh(r)n(esour)n(c)n(e)f(exc)n(eption)f Fs(o)q(ccurs.)19 b(The)13 b(standard)f(do)q(es)g(not)h(sp)q(ecify)g(when)g(a)f(resource) h(exception)g(is)189 829 y(allo)o(w)o(ed)h(to)g(o)q(ccur.)20 b(It)14 b(is)h(exp)q(ected)h(that)d(an)h(op)q(erational)h(de\014nition) h(will)g(b)q(e)f(made)g(a)o(v)m(ailable,)189 885 y(in)20 b(the)g(form)e(of)h(test)g(programs)g(that)f(ha)o(v)o(e)h(to)g(execute) h(with)g(no)g(resource)f(exceptions.)34 b(It)189 942 y(is)21 b(highly)i(desirable)f(to)f(ha)o(v)o(e)f(generous)h(b)q(ounds)h (on)f(the)g(n)o(um)o(b)q(er)h(of)e(concurren)o(tly)i(activ)o(e)189 998 y(comm)o(unication)17 b(ob)s(jects)f(eac)o(h)g(pro)q(cess)h(ma)o(y) f(ha)o(v)o(e,)g(so)g(that,)g(in)i(practice,)f Fl(INIT)f Fs(op)q(erations)189 1055 y(will)g(alw)o(a)o(ys)f(b)q(e)h(guaran)o (teed)f(to)f(succeed.)131 1149 y(2.)22 b(Eac)o(h)16 b(pro)q(cess)g(can) h(initiate)h(a)e(comm)o(unication)h(op)q(eration)g(for)e(eac)o(h)i (activ)o(e)f(comm)o(unication)189 1205 y(ob)s(ject.)j(I.e.)h(correct)15 b Fl(START)f Fs(op)q(erations)i(alw)o(a)o(ys)e(succeed)i(\(ev)o(en)o (tually\).)131 1299 y(3.)22 b(A)e(send)h(op)q(eration)f(is)h Ft(enabled)h Fs(if)e(the)h(sending)g(pro)q(cess)g(has)f(issued)i(a)e Fl(COMPLETE)f Fs(op)q(era-)189 1356 y(tion)f(and)g(the)f(receiving)j (pro)q(cess)e(has)g(issued)g(a)g Fl(START)f Fs(op)q(eration)h(for)f(a)h (matc)o(hing)f(receiv)o(e.)189 1412 y(Symmetrically)l(,)k(a)d(receiv)o (e)i(op)q(eration)f(is)g Ft(enabled)g Fs(if)h(the)e(receiving)j(pro)q (cess)e(has)f(issued)i(a)189 1468 y Fl(COMPLETE)10 b Fs(op)q(eration)i(and)f(the)h(sending)g(pro)q(cess)g(has)f(issued)i(a)e Fl(START)g Fs(op)q(eration)g(for)g(a)g(matc)o(h-)189 1525 y(ing)19 b(send.)31 b(An)19 b(enabled)h(op)q(eration)e(ma)o(y)g(b) q(ecome)i Ft(disabled)f Fs(either)h(b)q(ecause)f(it)g(completes)189 1581 y(successfully)e(or,)e(in)h(the)f(case)h(of)f(a)g(receiv)o(e,)h(b) q(ecause)g(the)g(matc)o(hing)f(message)g(is)h(successfully)189 1638 y(receiv)o(ed)g(b)o(y)f(another)g(receiv)o(e)g(op)q(eration)h(or,) e(in)i(the)f(case)g(of)g(a)f(send,)i(b)q(ecause)g(the)f(matc)o(hing)189 1694 y(receiv)o(e)h(successfully)h(receiv)o(es)f(another)f(message.)189 1770 y Ft(An)24 b(enabled)j(op)q(eration)f(either)g(completes)f (successfully)h(or)e(b)q(ecomes)i(p)q(erma-)189 1826 y(nen)o(tly)17 b(disabled.)189 1901 y Fs(I.e.,)d(an)h(enabled)i(op)q (eration)e(either)h(ev)o(en)o(tually)g(succeeds,)g(or)e(b)q(ecomes)i (disabled)h(\(progress\);)189 1958 y(and)i(an)g(op)q(eration)g(cannot)g (b)q(e)h(enabled)h(in\014nitely)g(man)o(y)e(times)g(without)g(ev)o(er)g (succeeding)189 2014 y(\(fairness\).)131 2108 y(4.)j(A)15 b Fl(FREE)f Fs(op)q(eration)i(alw)o(a)o(ys)e(succeeds)j(\(ev)o(en)o (tually\).)166 2214 y(The)i(four)f(conditions)i(guaran)o(tee)d (progress)h(in)i(the)e(comm)o(unication)i(subsystem.)30 b(The)18 b(third)75 2271 y(condition)e(guaran)o(tee)f(\(w)o(eak\))f (fairness)h(among)g(comp)q(eting)h(comm)o(unication)g(requests.)166 2327 y(Examples)g(\(in)o(v)o(olving)g(t)o(w)o(o)e(pro)q(cessors)h(with) g(ranks)g(0)g(and)g(1\))166 2384 y(The)g(follo)o(wing)h(program)e(is)i (safe,)f(and)g(should)h(alw)o(a)o(ys)f(succeed.)75 2534 y Fl(CALL)23 b(MPI_RANK\(rank,)f(context\))75 2591 y(IF)i (\(rank.EQ.0\))e(THEN)147 2647 y(CALL)h(MPI_SENDB\(sendbuf,)e(len,)j (1,)f(tag,)h(context\))147 2704 y(CALL)f(MPI_RECVB\(recvbuf,)e(len,)j (1,)f(tag,)h(context\))p eop %%Page: 81 86 81 85 bop 1830 -100 a Fs(81)75 45 y Fl(ELSE)95 b(!)24 b(rank.EQ.1)147 102 y(CALL)f(MPI_RECVB\(recvbuf,)e(len,)j(0,)f(tag,)h (context\))147 158 y(CALL)f(MPI_SENDB\(sendbuf,)e(len,)j(0,)f(tag,)h (context\))75 214 y(END)f(IF)166 319 y Fs(The)15 b(follo)o(wing)h (program)e(is)i(erroneous,)f(and)g(should)h(alw)o(a)o(ys)f(fail.)75 499 y Fl(CALL)23 b(MPI_RANK\(rank,)f(context\))75 555 y(IF)i(\(rank.EQ.0\))e(THEN)147 612 y(CALL)h(MPI_RECVB\(recvbuf,)e (len,)j(1,)f(tag,)h(context\))147 668 y(CALL)f(MPI_SENDB\(sendbuf,)e (len,)j(1,)f(tag,)h(context\))75 725 y(ELSE)95 b(!)24 b(rank.EQ.1)147 781 y(CALL)f(MPI_RECVB\(recvbuf,)e(len,)j(0,)f(tag,)h (context\))147 838 y(CALL)f(MPI_SENDB\(sendbuf,)e(len,)j(0,)f(tag,)h (context\))75 894 y(END)f(IF)166 1014 y Fs(The)15 b(receiv)o(e)g(op)q (eration)g(of)f(the)h(\014rst)f(pro)q(cess)h(m)o(ust)f(complete)i(b)q (efore)f(its)f(send,)h(and)g(can)g(com-)75 1071 y(plete)j(only)g(if)g (the)f(matc)o(hing)g(send)h(of)f(the)g(second)h(pro)q(cessor)f(is)h (executed;)g(the)g(receiv)o(e)g(op)q(eration)75 1127 y(of)c(the)h(second)g(pro)q(cess)g(m)o(ust)f(complete)h(b)q(efore)g (its)f(send)i(and)e(can)h(complete)g(only)g(if)g(the)g(matc)o(hing)75 1184 y(send)h(of)f(the)g(\014rst)g(pro)q(cess)g(is)h(executed.)21 b(This)15 b(program)f(will)j(deadlo)q(c)o(k.)166 1243 y(The)10 b(follo)o(wing)h(program)f(is)g(unsafe,)h(and)g(ma)o(y)e (succeed)j(or)e(fail,)h(dep)q(ending)i(on)d(implemen)o(tation.)75 1422 y Fl(CALL)23 b(MPI_RANK\(rank,)f(context\))75 1479 y(IF)i(\(rank.EQ.0\))e(THEN)147 1535 y(CALL)h(MPI_SENDB\(sendbuf,)e (len,)j(1,)f(tag,)h(context\))147 1592 y(CALL)f(MPI_RECVB\(recvbuf,)e (len,)j(1,)f(tag,)h(context\))75 1648 y(ELSE)95 b(!)24 b(rank.EQ.1)147 1705 y(CALL)f(MPI_SENDB\(sendbuf,)e(len,)j(0,)f(tag,)h (context\))147 1761 y(CALL)f(MPI_RECVB\(recvbuf,)e(len,)j(0,)f(tag,)h (context\))75 1817 y(END)f(IF)166 1994 y Fs(The)11 b(message)g(sen)o(t) g(b)o(y)g(eac)o(h)g(pro)q(cess)g(has)g(to)f(b)q(e)i(copied)g(out)f(b)q (efore)g(the)g(send)h(op)q(eration)f(returns)75 2051 y(and)k(the)g(receiv)o(e)h(op)q(eration)f(starts.)k(F)l(or)14 b(the)h(program)f(to)g(complete,)h(it)h(is)f(necessary)g(that)g(at)f (least)75 2107 y(one)i(of)g(the)h(t)o(w)o(o)e(messages)g(sen)o(t)h(is)h (bu\013ered)g(out)f(of)f(either)i(pro)q(cesses')g(address)f(space.)23 b(Th)o(us,)17 b(this)75 2163 y(program)c(can)i(succeed)g(only)g(if)g (the)f(comm)o(unication)h(system)f(has)g(su\016cien)o(t)h(bu\013er)g (space)f(to)g(bu\013er)75 2220 y Fl(len)h Fs(w)o(ords)f(of)h(data.)166 2279 y(If)g(additional)h(requiremen)o(ts)f(will)i(b)q(ecome)e(part)f (of)h(the)g(standard)f(\(e.g.,)f(b)q(ounds)j(on)f(the)f(mini-)75 2336 y(mal)i(n)o(um)o(b)q(er)f(of)g(concurren)o(tly)h(activ)o(e)f (handles)i(that)e(need)h(b)q(e)g(supp)q(orted,)f(then)h(further)f (programs)75 2392 y(b)q(ecome)h(safe.)75 2530 y Fi(6.0.7)55 b(Collectiv)n(e)17 b(Comm)n(unications)75 2704 y Fk(Discussion:)34 b Fj(This)14 b(is)f(still)g(v)o(ery)i(preliminary)p eop %%Page: 82 87 82 86 bop 75 -100 a Fs(82)919 b Fq(CHAPTER)16 b(6.)34 b(CORRECTNESS)17 b(ISSUES)166 45 y Fs(The)i(seman)o(tics)g(of)f(the)h (collectiv)o(e)h(comm)o(unication)g(op)q(erations)f(can)f(b)q(e)i (deriv)o(ed)f(from)f(their)75 102 y(op)q(erational)g(de\014nition)i(in) f(terms)e(of)g(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication.)28 b(It)18 b(is)g(assumed)g(that)f(mes-)75 158 y(sages)e(p)q(ertaining)i (to)e(one)g(op)q(eration)h(cannot)f(b)q(e)i(confused)f(with)g(messages) f(p)q(ertaining)h(to)f(another)75 214 y(op)q(eration.)20 b(Also)14 b(messages)f(p)q(ertaining)i(to)e(t)o(w)o(o)g(distinct)i(o)q (ccurrences)g(of)e(the)h(same)g(op)q(eration)g(can-)75 271 y(not)h(b)q(e)h(confused,)f(if)h(the)g(t)o(w)o(o)d(o)q(ccurrences)k (ha)o(v)o(e)e(distinct)h(parameters.)j(The)d(relev)m(an)o(t)g (parameters)75 327 y(for)e(this)h(purp)q(ose)h(are)e Fl(group)p Fs(,)g Fl(root)g Fs(and)h Fl(op)p Fs(.)k(messages)14 b(p)q(ertaining)i(to)e(another)h(o)q(ccurrence)h(of)e(the)75 384 y(same)k(op)q(eration,)g(with)g(di\013eren)o(t)g(parameters.)27 b(The)18 b(implemen)o(ter)i(can,)e(of)f(course,)i(use)f(another,)75 440 y(more)d(e\016cien)o(t)h(implemen)o(tation,)g(as)f(long)g(as)g(it)g (has)g(the)h(same)f(e\013ect.)166 573 y Fk(Discussion:)166 623 y Fj(This)c(statemen)o(t)h(do)q(es)g(not)g(y)o(et)g(apply)e(to)i (the)g(curren)o(t,)h(incomplete)e(and)g(somewhat)g(careless)i (de\014nitions)75 672 y(I)h(pro)o(vided)g(in)f(this)h(draft.)166 722 y(The)i(de\014nition)f(ab)q(o)o(v)o(e)g(means)g(that)g(messages)h (p)q(ertaining)f(to)g(a)g(collectiv)o(e)h(comm)o(uni)o(cation)d(carry)j (in-)75 772 y(formation)11 b(iden)o(tifying)h(the)j(op)q(eration)e (itself,)g(and)h(the)g(v)n(alues)g(of)f(the)h Ff(group)f Fj(and,)g(where)i(relev)n(an)o(t,)e Ff(root)g Fj(or)75 822 y Ff(op)g Fj(parameters.)19 b(Is)14 b(this)g(acceptable?)166 961 y Fs(A)h(few)g(examples:)75 1050 y Fl(MPI_BCASTC\(buf,)22 b(len,)h(type,)g(group,)g(0\);)75 1106 y(MPI_BCASTC\(buf,)f(len,)h (type,)g(group,)g(1\);)166 1195 y Fs(Tw)o(o)13 b(consecutiv)o(e)h (broadcasts,)f(in)h(the)g(same)f(group,)g(with)h(the)g(same)f(tag,)g (but)g(di\013eren)o(t)h(ro)q(ots.)75 1252 y(Since)k(the)e(op)q (erations)g(are)g(distinguishable,)i(messages)e(from)f(one)h(broadcast) g(cannot)g(b)q(e)g(confused)75 1308 y(with)g(messages)e(from)h(the)g (other)g(broadcast;)f(the)h(program)f(is)i(safe)f(and)g(will)i(execute) f(as)f(exp)q(ected.)75 1397 y Fl(MPI_BCASTC\(buf,)22 b(len,)h(type,)g(group,)g(0\);)75 1453 y(MPI_BCASTC\(buf,)f(len,)h (type,)g(group,)g(0\);)166 1542 y Fs(Tw)o(o)18 b(consecutiv)o(e)i (broadcasts,)f(in)h(the)f(same)g(group,)h(with)f(the)g(same)g(tag)f (and)i(ro)q(ot.)30 b(Since)75 1599 y(p)q(oin)o(t-to-p)q(oin)o(t)14 b(comm)o(unication)g(preserv)o(es)g(the)f(order)h(of)f(messages)g (here,)g(to)q(o,)g(messages)g(from)g(one)75 1655 y(broadcast)g(will)i (not)e(b)q(e)h(confused)g(with)g(messages)f(from)g(the)g(other)g (broadcast;)g(the)h(program)e(is)i(safe)75 1711 y(and)h(will)i(execute) f(as)f(in)o(tended.)75 1800 y Fl(MPI_RANK\(&rank,)22 b(group\))75 1857 y(if)i(\(rank==0\))123 1913 y({)147 1970 y(MPI_BCASTC\(buf,)d(len,)j(type,)f(group,)g(0\);)147 2026 y(MPI_SENDC\(buf,)f(len,)h(type,)g(2,)h(tag,)f(group\);)123 2083 y(})75 2139 y(elseif)g(\(rank==1\))123 2195 y({)147 2252 y(MPI_RECVC\(buf,)f(len,)h(type,)g(MPI_DONTCARE,)f(tag,)h (group\);)147 2308 y(MPI_BCASTC\(buf,)e(len,)j(type,)f(group,)g(0\);) 147 2365 y(MPI_RECVC\(buf,)f(len,)h(type,)g(MPI_DONTCARE,)f(tag,)h (group\);)123 2421 y(})75 2478 y(else)123 2534 y({)147 2591 y(MPI_SENDC\(buf,)f(len,)h(type,)g(2,)h(tag,)f(group\);)147 2647 y(MPI_BCASTC\(buf,)e(len,)j(type,)f(group,)g(0\);)123 2704 y(})p eop %%Page: 83 88 83 87 bop 1830 -100 a Fs(83)166 45 y(Pro)q(cess)17 b(zero)g(executes)h (a)f(broadcast)f(follo)o(w)o(ed)i(b)o(y)f(a)g(send)g(to)g(pro)q(cess)g (one;)h(pro)q(cess)g(t)o(w)o(o)e(ex-)75 102 y(ecutes)h(a)f(send)h(to)f (pro)q(cess)h(one,)f(follo)o(w)o(ed)h(b)o(y)f(a)g(broadcast;)g(and)h (pro)q(cess)g(one)f(executes)h(a)f(receiv)o(e,)75 158 y(a)21 b(broadcast)g(and)g(a)g(receiv)o(e.)39 b(A)21 b(p)q(ossible)i(outcome)e(is)h(for)f(the)g(op)q(erations)h(to)e(b)q(e)i (matc)o(hed)f(as)75 214 y(illustrated)c(b)o(y)e(the)g(diagram)g(b)q (elo)o(w.)170 434 y Fl(0)549 b(1)525 b(2)457 547 y(/)24 b(-)f(>)48 b(receive)285 b(/)24 b(-)g(send)409 603 y(/)597 b(/)75 659 y(broadcast)70 b(/)215 b(broadcast)166 b(/)71 b(broadcast)337 716 y(/)573 b(/)123 772 y(send)71 b(-)310 b(receive)47 b(<)23 b(-)166 992 y Fs(The)14 b(reason)g(is)h(that)f (broadcast)f(is)i(not)f(a)g(sync)o(hronous)g(op)q(eration;)h(the)f (call)h(at)f(a)g(pro)q(cess)h(ma)o(y)75 1048 y(return)j(b)q(efore)g (the)f(other)h(pro)q(cesses)g(ha)o(v)o(e)f(en)o(tered)h(the)g (broadcast.)27 b(Th)o(us,)18 b(the)f(message)h(sen)o(t)f(b)o(y)75 1104 y(pro)q(cess)g(zero)f(can)h(arriv)o(e)g(to)f(pro)q(cess)g(one)h(b) q(efore)g(the)g(message)f(sen)o(t)g(b)o(y)h(pro)q(cess)g(t)o(w)o(o,)e (and)i(b)q(efore)75 1161 y(the)e(call)i(to)d(broadcast)h(on)g(pro)q (cess)g(one.)166 1300 y Fk(Missing:)h Fj(This)d(c)o(hapter)i(should)e (ha)o(v)o(e)g(a)g(discussion)h(of)e(fairness)i(for)f(the)h(MPI)p 1466 1300 13 2 v 15 w(W)-5 b(AIT)p 1595 1300 V 15 w(ANY)14 b(function.)75 1357 y(Do)f(w)o(e)h(w)o(an)o(t)g(MPI)g(to)g(ensure)h (fairness?)k(I)14 b(prop)q(ose)g(not.)p eop %%Page: 84 89 84 88 bop 75 364 a Fn(Chapter)34 b(7)75 579 y Fu(En)m(vironmen)m(tal)41 b(Managemen)m(t)f(and)75 703 y(Inquiry)869 933 y Fs(Bill)17 b(Gropp)75 1118 y Fm(7.1)70 b(In)n(tro)r(duction)75 1233 y Fs(This)17 b(c)o(hapter)e(con)o(tains)h(MPI-sp)q(eci\014c)i(routines) f(for)e(in)o(teracting)i(with)f(the)g(en)o(vironmen)o(t)g(in)h(whic)o (h)75 1290 y(MPI)12 b(programs)e(run.)19 b(The)12 b(c)o(hapter)g(op)q (ens)g(with)g(a)g(discussion)h(of)f(MPI)f(routines)i(for)e (initialization)j(of)75 1346 y(the)g(MPI)h(en)o(vironmen)o(t.)20 b(The)14 b(c)o(hapter)h(includes)h(a)e(large)h(n)o(um)o(b)q(er)f(of)g (enquiry)i(functions,)f(b)o(y)f(whic)o(h)75 1402 y(an)h(MPI)f(program)g (can)h(determine)g(features)g(of)f(the)h(en)o(vironmen)o(t)g(in)g(whic) o(h)h(it)f(is)g(running)h(in)f(order)75 1459 y(to)g(tune)i(itself.)23 b(There)17 b(is)f(also)h(a)e(discussion)j(of)e(functions)h(that)e(are)h (not)g(sp)q(eci\014c)i(to)d(MPI,)h(suc)o(h)g(as)75 1515 y(timing)d(functions,)g(that)e(ma)o(y)h(b)q(e)h(pro)o(vided)g(b)o(y)f (other)g(standards)f(\(e.g.)18 b(POSIX\).)13 b(F)l(or)e(completeness,) 75 1572 y(w)o(e)k(discuss)h(signals,)g(although)f(p)q(erhaps)h(this)g (b)q(elongs)g(in)g(the)f(c)o(hapter)g(on)g(correctness.)75 1735 y Fi(7.1.1)55 b(Program)18 b(Startup)h(Issues)75 1835 y Fs(The)i(MPI)f(standard)g(should)i(pro)o(vide)f(enough)g (information)g(to)e(sp)q(ecify)j(an)f(en)o(tire,)h(source)e(co)q(de)75 1891 y(p)q(ortable)12 b(program.)18 b(T)l(o)12 b(do)f(this,)i(it)f(is)g (necessary)g(to)g(sp)q(ecify)h(ho)o(w)e(parallel)i(programs)e(start)g (and)h(end.)75 1948 y(This)f(is)f(the)h(analogue)f(of)g Fl(PROGRAM)23 b(MAIN)g(...)47 b(END)10 b Fs(in)h(F)l(ortran)e(and)h Fl(main\(argc,argv\))p Fo(f)p Fl(...exit\(rc\))o(;)p Fo(g)p Fs(.)75 2004 y(This)18 b(prop)q(osal)h(do)q(es)f(not)f(deal)i (with)f(metho)q(ds)g(for)g(spa)o(wning)g(tasks)f(during)i(the)f (execution)h(of)e(an)75 2061 y(MPI)g(program,)e(or)h(for)g(handling)i (explicitly)i(MIMD)c(mo)q(dels)h(\(b)q(ey)o(ond)g(the)g(usual)g(w)o(a)o (y)e(of)i(starting)75 2117 y(a)h(T)l(uring)i(mac)o(hine)f(on)g(all)h (no)q(des)f(and)g(sending)h(the)f(appropriate)f(tap)q(e\).)31 b(This)19 b(is)g(not)f(mean)o(t)h(to)75 2174 y(suggest)11 b(that)h(MPI)f(not)h(ha)o(v)o(e)f(suc)o(h)i(in)o(terfaces,)f(just)g (that)f(w)o(e)h(are)f(only)i(prop)q(osing)f(an)g(SPMD)f(mo)q(del.)166 2237 y(It)g(should)i(b)q(e)f(p)q(ossible)h(to)e(write)h(a)f(program)f (that)h(is)h(source-language)g(p)q(ortable)g(to)f(an)o(y)g(system,)75 2294 y(particularly)k(if)f(it)g(requires)h(only)f(a)g(simple)h(in)o (terface)f(to)f(the)h(op)q(erating)g(system.)19 b(An)14 b(example)h(is)f(an)75 2350 y(en)o(vironmen)o(t)i(that)f(starts)g(up)h (a)f(program)g(with)h(a)g(\014xed)g(n)o(um)o(b)q(er)g(of)g(pro)q (cessors.)21 b(Without)16 b(this,)g(it)75 2407 y(is)g(not)f(p)q (ossible)h(to)f(write)g Fh(any)k Fs(p)q(ortable)d(programs.)166 2471 y(Ho)o(w)o(ev)o(er,)d(no)i(standard)g(should)h(restrict)e (programs)g(to)g(a)h(particular)g(in)o(terface.)20 b(F)l(or)14 b(example,)75 2527 y(some)h(applications)i(ma)o(y)e(w)o(an)o(t)f(to)h (dynamically)i(add)f(and)g(remo)o(v)o(e)e(tasks)h(during)h(the)g (execution)g(of)75 2583 y(an)d(application.)21 b(W)l(e)13 b(lea)o(v)o(e)h(it)f(to)f(others)h(on)g(the)h(committee)f(to)f(propse)i (a)e(mec)o(hanism)i(for)f(this)g(case.)166 2647 y(Belo)o(w,)i(w)o(e)g (discuss)h(three)f(di\013eren)o(t)g(approac)o(hes;)f(w)o(e)h(note)g (that)f(all)i(of)f(these)g(ha)o(v)o(e)f(b)q(een)i(tried)75 2704 y(in)g(some)f(existing)h(system)f(or)g(systems.)952 2828 y(84)p eop %%Page: 85 90 85 89 bop 75 -100 a Fq(7.1.)34 b(INTR)o(ODUCTION)1279 b Fs(85)75 45 y Ft(MPI)p 184 45 16 2 v 18 w(INIT)75 134 y Fs(The)11 b(seemingly)i(simplest)f(approac)o(h)f(is)g(to)f(use)i(an)f (explicit)i(initialization)h(routine,)e Fl(MPI)p 1618 134 15 2 v 17 w(Init)p Fs(.)18 b(There)75 191 y(are)h(sev)o(eral)g (problems)h(with)f(this)h(approac)o(h.)31 b(The)20 b(ma)s(jor)d(one)j (is)f(that)g(the)g(parallel)i(part)d(of)h(the)75 247 y(program)c(is)j(the)e(co)q(de)h(that)f(\\follo)o(ws")g(the)h Fl(MPI)p 942 247 V 17 w(Init)p Fs(.)23 b(This)17 b(is)h(p)q(oten)o (tially)g(error)d(prone;)i(further,)75 304 y(it)e(can)h(b)q(e)g (di\016cult)g(to)f(implemen)o(t)h(on)f(some)g(systems.)75 433 y Ft(MPI)p 184 433 16 2 v 18 w(MAIN)75 523 y Fs(Another)i(approac)o (h)g(is)h(to)f(insist)h(that)e(the)i(en)o(tire)f(program)g(b)q(e)h(run) f(in)h(parallel;)i(a)d(simple)h(w)o(a)o(y)f(to)75 579 y(do)h(this)h(is)h(to)d(replace)j(the)f(\\main")f(program)f(with)i(a)g (routine.)30 b(In)19 b(C,)f(this)h(could)h(b)q(e)f Fl(MPI)p 1742 579 15 2 v 17 w(Main\()75 636 y(argc,)k(argv)g(\))p Fs(.)d(The)15 b(linking)i(step)d(then)h(pro)o(vides)g(a)g Fl(main)f Fs(that)g(handles)h(all)h(appropriate)f(startup)75 692 y(\(including)h(the)e(pro)q(cessing)g(of)f(command)g(line)j (argumen)o(ts)c(where)i(appropriate)f(for)g(information)h(on)75 748 y(the)h(n)o(um)o(b)q(er)h(of)f(tasks)f(etc.\))20 b(and)15 b(then)g(calls)i(the)e(user's)g Fl(MPI)p 1168 748 V 17 w(main)f Fs(routine.)166 807 y(Note)h(that)f(the)i Fl(MPI)p 528 807 V 16 w(main)f Fs(in)o(terface)g(can)h(b)q(e)g (implemen)o(ted)h(with)e Fl(MPI)p 1429 807 V 17 w(call)f Fs(through)147 923 y Fl(int)23 b(main\()g(argc,)g(argv)h(\))147 980 y(int)f(argc;)147 1036 y(char)g(**argv;)147 1093 y({)147 1149 y(return)g(MPI_call\()f(argc,)h(argv,)h(MPI_main)e(\);)147 1205 y(})75 1335 y Ft(MPI)p 184 1335 16 2 v 18 w(CALL)75 1424 y Fs(Another)17 b(approac)o(h)f(is)h(to)f(pro)o(vide)h(a)f (routine)h(that)f(runs)h(a)f(user-de\014ned)j(routine)e(in)g(parallel.) 26 b(F)l(or)75 1481 y(example,)19 b Fl(MPI)p 344 1481 15 2 v 17 w(Call\()k(argc,)g(argv,)g(routine)g(\))18 b Fs(can)g(b)q(e)h(used)f(to)f(run)i Fl(routine\()j(argc,)i(argv)75 1537 y(\))15 b Fs(in)i(parallel.)23 b(A)16 b(generalization)h(of)e (this)h(w)o(ould)g(allo)o(w)g(the)g(running)h(of)e(a)g(particular)h (routine)h(on)e(a)75 1594 y(de\014ned)i(group:)k Fl(MPI)p 455 1594 V 17 w(Call\()i(groupid,)g(argc,)g(argv,)g(routine)g(\))p Fs(.)f(This)16 b(is)h(more)e(\015exible)j(than)75 1650 y(the)g Fl(MPI)p 231 1650 V 17 w(Main)g Fs(approac)o(h)g(but)g(retains) h(the)f(simplicit)o(y)j(of)d(running)h(a)f(w)o(ell-de\014ned)j(program) c(unit)75 1707 y(\(the)e(routine\))g(in)h(parallel.)166 1765 y(Note)g(that)f(in)i(this)f(case,)g(program)f(termination)h(is)g (handled)i(b)o(y)e(ha)o(ving)g(\(a\))f(the)h(routine)h(exit)75 1821 y(and)e(\(b\))g(the)g(calling)i(routine)f(exit.)k(F)l(or)15 b(example)170 1924 y Fl(irc)24 b(=)g(MPI_Call\()e(argc,)h(argv,)h (worker)f(\);)170 1980 y(exit\()h(irc)f(\);)75 2037 y(...)170 2093 y(int)h(worker\()f(argc,)g(argv)g(\))170 2150 y(int)h(argc;)f (char)g(**argv;)170 2206 y({)75 2262 y(...)170 2319 y(if)h(\(error)f (condition\))g(return)g(1;)170 2375 y(...)170 2432 y(return)g(0;)170 2488 y(})166 2589 y Fs(All)16 b(three)g(of)e(these)i(approac)o(hes)f (are)g(curren)o(tly)g(in)h(use.)166 2647 y(These)i(are)g(not)g(the)g (most)f(general)i(metho)q(ds.)29 b(F)l(or)17 b(example,)i(these)g(pro)o (vide)f(no)g(mec)o(hanism)75 2704 y(for)c(dynamically)i(creating)f (\(remote)e(spa)o(wn\))h(tasks)f(on)i(other)f(pro)q(cessors.)19 b(Also,)c(there)f(is)h(no)f(direct)p eop %%Page: 86 91 86 90 bop 75 -100 a Fs(86)326 b Fq(CHAPTER)16 b(7.)34 b(ENVIR)o(ONMENT)l(AL)17 b(MANA)o(GEMENT)d(AND)h(INQUIR)l(Y)75 45 y Fs(supp)q(ort)d(for)f(MIMD)g(programming.)18 b(Finally)l(,)c (there)e(is)g(no)f(supp)q(ort)h(for)f(adding)i(an)e(already-running)75 102 y(pro)q(cess)k(to)g(the)g(MPI)g(collection)i(of)e(pro)q(cesses.)166 158 y(In)f(the)f(case)g(of)g(heterogeneous)g(en)o(vironmen)o(ts,)g(it)h (is)f(necessary)h(to)e(sp)q(ecify)j(the)e(set)g(of)f(w)o(orksta-)75 214 y(tions)i(to)f(b)q(e)h(used.)20 b(Our)14 b(only)h(requiremen)o(t)f (is)g(that)f(this)i(b)q(e)f(done,)g(in)h(the)e(C/POSIX)i(case,)e (through)75 271 y(command-line)18 b(argumen)o(ts)e(\(th)o(us)g(main)o (taining)h(source-co)q(de)g(p)q(ortabilit)o(y\).)24 b(W)l(e)17 b(could)g(sp)q(ecify)h(an)75 327 y(extensible,)j(common)d(format)f(for) h(sp)q(ecifying)i(the)f(mac)o(hines,)h(executables,)g(and)e(resource)h (limits,)75 384 y(but,)h(to)f(a)o(v)o(oid)g(additional)i(debate,)f(w)o (e)f(suggest)g(only)h(that)e(the)i(command)f(line)i(option)f Fl(-mpi)e Fs(b)q(e)75 440 y(reserv)o(ed)d(for)g(future)g(extensions.)75 583 y Fm(7.2)70 b(MPI)23 b(Program)h(Startup)f(Prop)r(osal{MPI)p 1370 583 21 2 v 27 w(INIT)75 684 y Fs(One)13 b(goal)g(of)f(MPI)h(is)g (to)f(ac)o(hiev)o(e)h Fh(sour)n(c)n(e)h(c)n(o)n(de)g(p)n(ortability)p Fs(.)19 b(By)13 b(this)g(w)o(e)f(mean)h(that)f(a)g(program)g(writ-)75 740 y(ten)j(using)i(MPI)e(and)g(complying)i(with)f(the)f(relev)m(an)o (t)h(language)g(standards)f(is)g(p)q(ortable)h(as)f(written,)75 797 y(and)k(m)o(ust)e(not)h(require)i(an)o(y)e(source)g(co)q(de)h(c)o (hanges)f(when)h(mo)o(v)o(ed)f(from)g(one)g(system)g(to)g(another.)75 853 y(This)h(explicitly)i(do)q(es)e Fh(not)f Fs(sa)o(y)g(an)o(ything)g (ab)q(out)g(ho)o(w)g(an)h(MPI)f(program)f(is)i(started)f(or)f(launc)o (hed)75 910 y(from)h(the)i(command)f(line,)i(nor)e(what)f(the)i(user)f (m)o(ust)f(do)i(to)e(set)h(up)h(the)f(en)o(vironmen)o(t)g(in)h(whic)o (h)75 966 y(an)h(MPI)g(program)f(will)j(run.)38 b(Ho)o(w)o(ev)o(er,)21 b(an)h(implemen)o(tation)g(ma)o(y)e(require)i(some)f(setup)h(to)e(b)q (e)75 1023 y(p)q(erformed)c(b)q(efore)h(other)f(MPI)g(routines)g(ma)o (y)g(b)q(e)h(called.)24 b(T)l(o)16 b(pro)o(vide)g(for)g(this,)g(MPI)h (includes)h(an)75 1079 y(initialization)g(routine)e Fl(MPI)p 572 1079 15 2 v 17 w(init)p Fs(.)166 1136 y(All)g(MPI)f(programs)f(m)o (ust)h(con)o(tain)g(a)g(call)h(to)f Fl(MPI)p 1069 1136 V 16 w(init)p Fs(;)g(this)g(routine)h(m)o(ust)e(b)q(e)i(called)h(b)q (efore)75 1192 y(an)o(y)e(other)g(MPI)g(routines)g(are)g(called.)22 b(The)15 b(format)f(is:)75 1281 y Fl(start_ctx)23 b(=)g(MPI_init\()g (...)g(args)h(...)f(\);)75 1370 y Fs(The)13 b Fl(start)p 289 1370 V 16 w(ctx)f Fs(can)h(then)g(b)q(e)g(used)g(in)h(place)f(of)f (the)h Fl(all)p 1092 1370 V 17 w(ctx)f Fs(\(con)o(text)f(of)i(all)g (pro)q(cesses\).)19 b(Multiple)75 1426 y(calls)h(to)e Fl(MPI)p 314 1426 V 17 w(init)g Fs(\(sa)o(y)f(\014rst)i(b)o(y)f(a)h (user-co)q(de)g(and)g(then)g(b)o(y)g(a)f(soft)o(w)o(are)f(pac)o(k)m (age)i(called)h(b)o(y)f(the)75 1483 y(user's)d(co)q(de\))h(ha)o(v)o(e)g (meaning;)h(they)e(set)h(up)g(the)g(con)o(text)f(that)g(will)j(b)q(e)e (used)g(as)g(the)g(con)o(text)f(of)g(all)75 1539 y(pro)q(cesses.)166 1672 y Fk(Discussion:)g Fj(Sev)o(eral)e(other)h(approac)o(hes)g(w)o (ere)h(discussed,)f(including)e(a)h(replacemen)o(t)g(main)f(program)75 1722 y(and)f(a)g(subroutine-call)g(in)o(terface.)18 b(I'v)o(e)12 b(included)h(the)g Ff(MPI)p 1033 1722 14 2 v 15 w(init)e Fj(v)o(ersion)i(b)q(ecause)h(it)e(seemed)h(to)f(b)q(e)h(the)g(least)75 1771 y(ob)r(jectionable)h(and)g(since)g(w)o(ork)n(able)f(v)o(ersions)i (of)e(the)h(other)h(t)o(w)o(o)e(ma)o(y)f(b)q(e)j(written)f(using)g Ff(MPI)p 1618 1771 V 15 w(init)p Fj(.)166 1821 y(I'm)8 b(not)i(sure)h(what)f(argumen)o(ts)f Ff(MPI)p 758 1821 V 15 w(init)g Fj(should)h(tak)o(e.)17 b(One)11 b(argumen)o(t)e(should)g (b)q(e)i(the)g(initial)d(con)o(text;)75 1871 y(if)j(n)o(ull,)g(then)h (the)h(\\all)d(con)o(text")j(is)e(assumed.)17 b(There)c(is)f(some)f (reason)i(to)e(pass)i(the)f(command)d(line)j(argumen)o(ts)75 1921 y Ff(argc,argv)f Fj(to)i Ff(MPI)p 403 1921 V 15 w(init)f Fj(in)h(order)g(to)g(allo)o(w)e(it)i(to)g(lo)q(ok)f(for)g (MPI-sp)q(eci\014c)j(argumen)o(ts)d(\(suc)o(h)i(as)f(the)h(n)o(um)o(b)q (er)75 1971 y(and)g(name)e(of)i(pro)q(cessors)i(to)e(use)g(in)g(a)f (heterogeneous)j(en)o(vironmen)o(t\),)d(but)h(this)g(ma)o(y)e(b)q(e)i (to)q(o)g(con)o(tro)o(v)o(ersial.)166 2027 y(W)m(e)g(should)g(also)g (discuss)h(the)g(other)g(p)q(ossibilities.)k(One,)c Ff(MPI)p 1184 2027 V 15 w(main)p Fj(,)e(is)h(a)g(main)e(program)h(replacemen)o (t)75 2084 y(\(users)21 b(use)f Ff(MPI)p 347 2084 V 15 w(main)e Fj(instead)i(of)e Ff(main)h Fj(in)f(C)i(and)f Ff(PROGRAM)h(name)e Fj(in)h(F)m(ortran\).)34 b(Another)20 b(approac)o(h)f(is)75 2140 y Ff(MPI)p 144 2140 V 15 w(call)13 b Fj(that)h(calls)f(a)h(sp)q(eci\014ed)i(routine.)75 2365 y Fm(7.3)70 b(MPI-Sp)r(eci\014c)21 b(Enquiry)i(F)-6 b(unctions)75 2466 y Fs(These)16 b(routines)f(are)g(sp)q(eci\014c)i(to) e(the)g(op)q(eration)g(of)g(MPI.)166 2523 y(The)g(\014rst)g(t)o(w)o(o)f (routines)i(are)f(absolutely)h(essen)o(tial;)f(they)h(are:)75 2612 y Ft(MPI)p 184 2612 16 2 v 18 w(rank)22 b Fs(My)15 b(pro)q(cess)h(id)g(relativ)o(e)f(to)g(a)g(group)75 2704 y Ft(MPI)p 184 2704 V 18 w(size)23 b Fs(The)15 b(n)o(um)o(b)q(er)h(of)f (pro)q(cess)g(id's)h(in)g(a)f(group)p eop %%Page: 87 92 87 91 bop 75 -100 a Fq(7.3.)34 b(MPI-SPECIFIC)16 b(ENQUIR)l(Y)g (FUNCTIONS)811 b Fs(87)166 45 y(MPI)14 b(also)g(con)o(tain)h(routines)g (for)e(managing)i(implemen)o(tation)g(limits)h(in)f(MPI.)f(These)h (routines)75 102 y(will)i(tak)o(e)d(a)h(requested)h(v)m(alue)g(and)f (return)h(the)f(actual)g(v)m(alue,)h(p)q(ossibly)h(including)h(a)d(v)m (alue)h(for)f(\\not)75 158 y(applicable.")22 b(Suc)o(h)16 b(routines)g(ma)o(y)e(include)75 246 y Ft(MPI)p 184 246 16 2 v 18 w(V)l(alidT)l(ags)24 b Fs(The)15 b(range)g(\(or)f(other)h (description\))h(of)f(v)m(alid)i(tags)75 337 y Ft(MPI)p 184 337 V 18 w(NumGroups)22 b Fs(Maxim)o(um)15 b(n)o(um)o(b)q(er)g(of)g (groups)75 428 y Ft(MPI)p 184 428 V 18 w(NumCn)o(txs)21 b Fs(Maxim)o(um)15 b(n)o(um)o(b)q(er)h(of)e(con)o(texts)75 520 y Ft(MPI)p 184 520 V 18 w(Bu\013erSize)23 b Fs(Bu\013er)15 b(space)g(for)g(unsen)o(t/unreceiv)o(ed)i(messages)75 611 y Ft(MPI)p 184 611 V 18 w(Bu\013erManagemen)o(t)22 b Fs(Bu\013er)15 b(managemen)o(t)f(t)o(yp)q(e)166 699 y(Note)h(that)f(all)i(of)f(these)h(could)g(return)f(\\not)f (applicable.")166 838 y Fk(Discussion:)36 b Fj(These)16 b(need)f(to)f(refer)i(to)e(a)g(diagram)e(of)i(p)q(ossible)h(bu\013er)g (usages.)20 b(W)m(e)14 b(m)o(ust)g(b)q(e)h(careful)75 894 y(to)f(handle)f(the)i(case)g(of)e(\(a\))h(no)g(bu\013ering)g(and)g (\(b\))g(bu\013ers)h(on)f(the)g(sender.)166 1034 y Fs(More)21 b(detailed)i(routines)f(for)g(bu\013er)f(managemen)o(t)g(ma)o(y)g(also) h(b)q(e)g(pro)o(vided.)41 b(These)22 b(could)75 1090 y(include:)75 1178 y Ft(MPI)p 184 1178 V 18 w(Bu\013erSizeP)o(airs)g Fs(Bet)o(w)o(een)15 b(pairs)h(of)e(pro)q(cessors)75 1269 y Ft(MPI)p 184 1269 V 18 w(Bu\013erSizeAll)23 b Fs(Bet)o(w)o(een)15 b(\\me")g(and)g(all)i(others)75 1360 y Ft(MPI)p 184 1360 V 18 w(Exclusiv)o(eBu\013er)22 b Fs(Dedicate)g(a)f(bu\013er)g(of)g(a)g (sp)q(eci\014ed)j(size)e(b)q(et)o(w)o(een)g(a)f(selected)i(pair)e(of) 629 1417 y(pro)q(cessors)166 1581 y Fk(Discussion:)h Fj(Another)c(approac)o(h)f(is)g(to)g(use)h(a)f(single)g(routine)g(that) g(tak)o(es)h(a)f(query)g(parameter)g(and)75 1630 y(returns)e(the)g (requested)h(data)d(\(or)h(a)g(\\not)g(a)o(v)n(ailable"\).)166 1680 y(The)h(routines)h(returning)f(the)g(maxim)o(um)10 b(n)o(um)o(b)q(er)k(of)g(things)h(\(lik)o(e)f(groups\))i(could)e (instead)h(return)h(the)75 1730 y(maxim)n(um)g(n)o(um)o(b)q(er)j(of)g Fd(new)24 b Fj(things)c(that)g(could)f(b)q(e)i(created.)37 b(All)19 b(of)g(these)i(routines)g(are)f(mean)o(t)f(to)g(b)q(e)75 1780 y(advisory)m(,)12 b(particularly)i(in)f(a)h(m)o(ultithreaded)e(en) o(vironmen)o(t,)h(since)h(they)h(ma)o(y)d(b)q(e)i(allo)q(cated)g (dynamically)l(.)166 1919 y Fs(There)h(is)h(also)f(some)g(desire)h(for) 75 2007 y Ft(MPI)p 184 2007 V 18 w(IOmo)q(de)23 b Fs(Whether)15 b(IO)h(is)g(p)q(er)g(no)q(de)f(or)g(p)q(er)h(group/con)o(text)75 2098 y Ft(MPI)p 184 2098 V 18 w(ErrorMo)q(de)22 b Fs(V)l(arious)16 b(c)o(hoices)g(of)e(error)h(handling)166 2186 y(Both)j(of)f(these,)h (since)h(they)f(ha)o(v)o(e)f(global)i(e\013ect,)f(are)f(somewhat)g (undesirable.)30 b(They)18 b(should)75 2242 y(return)j(the)g(previous)h (mo)q(de;)j(m)o(ultithreaded)d(applications)h(will)g(ha)o(v)o(e)d(to)h (b)q(e)h(v)o(ery)f(careful)h(with)75 2299 y(these.)166 2355 y(It)16 b(is)h(imp)q(ortan)o(t)f(to)g(b)q(e)g(able)h(to)f (determine,)h(b)q(oth)g(at)e(run)i(time)g(and)f(compile)i(time,)e(whic) o(h)h(of)75 2412 y(the)e(\\optional")h(capabilities)h(the)e(MPI)h (implemen)o(tation)g(supp)q(orts.)k(These)c(include)75 2499 y Ft(MPI)p 184 2499 V 18 w(Has)p 289 2499 V 19 w(Ready)p 453 2499 V 19 w(Receiv)o(er)22 b Fs(Whether)10 b(ready-receiv)o(er)h (comm)o(unication)g(is)g(sim)o(ulated)g(with)f(reg-)629 2556 y(ular)16 b(sends)g(and)f(receiv)o(es.)75 2647 y Ft(MPI)p 184 2647 V 18 w(Has)p 289 2647 V 19 w(Non)o(blo)q(c)o(king)23 b Fs(Whether)h(non)o(blo)q(c)o(king)h(op)q(erations)e(are)g(sim)o (ulated)i(with)f(blo)q(c)o(king)629 2704 y(sends)16 b(and)f(receiv)o (es)p eop %%Page: 88 93 88 92 bop 75 -100 a Fs(88)326 b Fq(CHAPTER)16 b(7.)34 b(ENVIR)o(ONMENT)l(AL)17 b(MANA)o(GEMENT)d(AND)h(INQUIR)l(Y)75 45 y Ft(MPI)p 184 45 16 2 v 18 w(Has)p 289 45 V 19 w(Heterogeneous)22 b Fs(Whether)16 b(heterogeneous)g(pro)q(cessing)g(is)h(supp)q(orted)f (\(that)f(is,)h(this)629 102 y(MPI)f(w)o(orks)g(with)g(heterogeneous)g (systems\))166 198 y(In)f(addition,)h(it)f(is)g(useful)h(to)e(b)q(e)h (able)h(to)e(unpac)o(k)h(the)g(v)m(arious)g(opaque)g(ob)s(jects)f(in)o (to)h(individual)75 254 y(v)m(ariables,)25 b(suc)o(h)d(as)f(bu\013er)h (descriptors)h(and)f(con)o(texts,)g(and)h(the)f(message)f(queue.)41 b(Op)q(erations)75 310 y(should)16 b(include)75 406 y Ft(MPI)p 184 406 V 18 w(Unpac)o(kBu\013erDescriptor)22 b Fs(T)l(ak)o(es)d(a)f(bu\013er)h(and)h(returns)f(a)f(con)o(text)h(and) g(\(p)q(ossibly\))h(a)629 463 y(n)o(um)o(b)q(er)c(of)f(mem)o(b)q(ers.)k (Individual)g(routines)c(can)629 572 y Ft(Bu\013erDescT)o(yp)q(e)22 b Fs(Giv)o(en)d(a)g(con)o(text,)f(return)h(the)g(kind)h(of)e(bu\013er) 729 628 y(descriptor)e(elemen)o(t)g(\(con)o(tiguous)f(etc\).)629 702 y Ft(MPI)p 738 702 V 18 w(Unpac)o(kCon)o(tig)23 b Fs(Giv)o(en)18 b(a)f(con)o(text,)g(return)g(the)g(data.)26 b(Ad-)729 759 y(v)m(ance)16 b(the)g(con)o(text.)629 833 y Ft(MPI)p 738 833 V 18 w(Unpac)o(kIndex)22 b Fs(Ditto)15 b(of)g(indexed)629 907 y Ft(MPI)p 738 907 V 18 w(Unpac)o(kBlo)q(c)o(k) 24 b Fs(Ditto)14 b(for)h(blo)q(c)o(ks.)75 1016 y Ft(MPI)p 184 1016 V 18 w(DumpMessageQueue)22 b Fs(T)l(ak)o(es)14 b(a)g(length)h(and)g(a)f(user-allo)q(cated)i(arra)o(y)l(,)d(it)h (return)h(n)o(um)o(b)q(er)629 1073 y(of)j(elemen)o(ts)i(a)o(v)m (ailable)g(\(ma)o(y)d(b)q(e)j(greater)d(than)i(length\))g(and)g(infor-) 629 1129 y(mation)g(on)g(the)g(messages)g(in)h(the)f(format)f(includes) j(\(tag,)e(con)o(text,)629 1186 y(source,)c(size\))75 1282 y Ft(MPI)p 184 1282 V 18 w(Unpac)o(kCon)o(text)22 b Fs(T)l(ak)o(es)15 b(a)g(con)o(text)g(and)g(returns)g(mem)o(b)q(ers)h (of)e(group/con)o(text)75 1378 y Ft(MPI)p 184 1378 V 18 w(StatusOfHandles)24 b Fs(I'v)o(e)15 b(forgotten)f(what)h(this)g(is) h(for.)166 1474 y(Note)f(that)f(some)h(of)g(these)g(are)g(tough)g(to)g (implemen)o(t)h(directly)h(or)d(e\016cien)o(tly)j(in)f(F)l(ortran.)166 1613 y Fk(Discussion:)i Fj(An)e(alternate)f(approac)o(h)h(for)f(the)g (queue)i(and)e(con)o(text)h(inquiry)e(functions)i(\(at)f(least)g(for)75 1670 y(C\))f(is)g(for)f(the)i(routine)f(to)g(allo)q(cate)f(the)h(space) h(rather)g(than)f(ha)o(ving)f(it)g(passed)i(in.)75 1877 y Fi(7.3.1)55 b(P)n(arallel)18 b(programming)75 1964 y Fs(This)c(section)f(includes)i(routines)f(that)e(relate)h(to)f (parallel)j(programming)d(but)h(are)g(not)f(MPI-sp)q(eci\014c.)75 2020 y(These)k(ma)o(y)e(include)k(suc)o(h)d(things)h(as:)75 2116 y Ft(MPI)p 184 2116 V 18 w(GetNbrs)22 b Fs(Return)16 b(the)f(\\neigh)o(b)q(ors")h(in)g(the)f(underlying)i(ph)o(ysical)g(top) q(ology)75 2212 y Ft(MPI)p 184 2212 V 18 w(GetPh)o(ysNo)q(de)22 b Fs(Return)i(a)f(unique)h(sp)q(eci\014er)h(for)e(the)g(actual)g(\(as)f (opp)q(osed)i(to)f(virtual\))629 2269 y(no)q(de.)h(This)17 b(should)h(b)q(e)f(a)f(c)o(haracter)g(string)g(for)g(maxim)o(um)g (\015exibil-)629 2325 y(it)o(y)l(.)26 b(>F)l(rom)17 b(this)g(v)m(alue)h (it)g(m)o(ust)e(b)q(e)i(p)q(ossible)h(to)d(iden)o(tify)i(a)f(sp)q (eci\014c)629 2382 y(piece)h(of)e(hardw)o(are;)g(p)q(ossible)i(v)m (alues)f(include)i(\\pro)q(cessor)c(9)h(in)i(rac)o(k)629 2438 y(4)e(of)g(mpp.cs.org")f(and)h(\\231")f(\(where)h(231)f(is)i(the)f (actual)g(pro)q(cessor)629 2495 y(n)o(um)o(b)q(er)i(in)h(the)f(running) g(homogeneous)g(system\).)26 b(A)18 b(return)g(v)m(alue)629 2551 y(of)d(\\not)g(applicable")i(is)f(allo)o(w)o(ed)f(but)h(strongly)e (discouraged.)75 2647 y Ft(GetT)l(op)q(ology)24 b Fs(Return)18 b(ph)o(ysical)g(net)o(w)o(ork)f(top)q(ology)l(.)25 b(Prede\014ned)19 b(v)m(alues)f(include)h Fl(MPI)p 1705 2647 15 2 v 17 w(mesh2d)p Fs(,)629 2704 y Fl(MPI)p 704 2704 V 17 w(hypercube)p Fs(,)14 b(and)h Fl(MPI)p 1124 2704 V 17 w(omeganetwork)p Fs(.)p eop %%Page: 89 94 89 93 bop 75 -100 a Fq(7.3.)34 b(MPI-SPECIFIC)16 b(ENQUIR)l(Y)g (FUNCTIONS)811 b Fs(89)75 45 y Ft(MPI)p 184 45 16 2 v 18 w(GetDistance)24 b Fs(Return)16 b(the)f(\\distance")h(from)e(one)i (pro)q(cessor)f(to)f(another)75 139 y Ft(MPI)p 184 139 V 18 w(GetDiameter)23 b Fs(Return)16 b(the)g(maxim)o(um)f(\\distance")g (from)g(one)g(pro)q(cessor)g(to)f(another.)166 233 y(Eac)o(h)f(of)g (these)h(routines)g(ma)o(y)e(return)i(\\not)e(applicable;")k(it)e(is)g (hop)q(ed)g(that)f(qualit)o(y)h(implemen-)75 289 y(tations)g(will)j (try)d(and)g(pro)o(vide)i(as)e(m)o(uc)o(h)h(information)f(as)h(p)q (ossible)h(to)e(the)h(user.)20 b(In)15 b(addition,)g(some)75 346 y(of)g(the)g(\\get")g(routines)g(m)o(ust)g(b)q(e)h(in)o(terpreted)g (as)f(pro)o(viding)h(appro)o(ximate)f(answ)o(ers.)20 b(F)l(or)14 b(example,)75 402 y(if)k(groups)f(and)h(con)o(texts)f(are)h (allo)q(cated)g(from)f(the)h(user's)f(address)h(space,)g(then)g(an)o(y) f(return)h(v)m(alue)75 459 y(will)f(b)q(e)f(appro)o(ximate.)166 515 y(The)g(exact)f(meaning)h(of)f(\\distance")h(is)g(left)g(to)f(the)g (implemen)o(tation.)22 b(In)16 b(a)g(system)f(with)g(only)75 571 y(lo)q(cal)23 b(connections,)g(suc)o(h)f(as)g(a)f(mesh)h(or)f(h)o (yp)q(ercub)q(e,)j(the)e(distance)g(should)h(b)q(e)f(the)f(n)o(um)o(b)q (er)h(of)75 628 y(edges)17 b(in)h(the)e(connection)i(graph)f(that)f(a)g (message)h(w)o(ould)g(tra)o(v)o(el)f(along.)25 b(In)17 b(a)f(system)h(with)g(a)f(\015at)75 684 y(in)o(terconnect,)d(suc)o(h)g (as)g(an)f(omega)g(or)g(butter\015y)g(net)o(w)o(ork,)g(the)h(distance)g (could)h(b)q(e)f(zero)f(from)g(a)g(no)q(de)75 741 y(to)j(itself)h(and)f (either)h(one)f(or)g(the)g(n)o(um)o(b)q(er)h(of)f(stages)f(in)i(the)f (net)o(w)o(ork)f(for)h(all)h(other)f(no)q(des.)75 863 y Fi(7.3.2)55 b(non-MPI)75 948 y Fs(This)21 b(section)h(con)o(tains)e (routines)h(that)f(are)h Fh(not)k Fs(part)20 b(of)g(MPI)h(but)f(are)h (included)i(for)d(complete-)75 1005 y(ness.)k(These)17 b(will)h(include)h(v)m(arious)e(routines)g(for)f(resource)g(usage,)h (time-of-da)o(y)l(,)f(etc.)24 b(The)17 b(POSIX)75 1061 y(in)o(terfaces)e(will)i(b)q(e)f(summarized)g(here.)75 1181 y Ft(In)o(teraction)i(with)g(signals)75 1267 y Fs(The)d(in)o(ten)o (t)g(of)f(MPI)h(is)g(to)f(b)q(e)h(thread)g(and)f(signal)i(safe.)j(This) d(has)e(a)g(n)o(um)o(b)q(er)h(of)g(subtle)g(e\013ects.)20 b(F)l(or)75 1324 y(example,)g(on)e(Unix)h(systems,)g(a)f(catc)o(hable)h (signal)g(suc)o(h)g(as)f(SIGALRM)h(\(an)f(alarm)h(signal\))g(m)o(ust)75 1380 y(not)f(cause)g(an)h(MPI)f(routine)g(to)g(b)q(eha)o(v)o(e)h (di\013eren)o(tly)g(than)f(it)g(w)o(ould)h(ha)o(v)o(e)f(in)h(the)f (absence)h(of)f(the)75 1437 y(signal.)37 b(Of)21 b(course,)h(if)f(the)g (signal)h(handler)f(issues)h(MPI)f(calls)g(or)g(c)o(hanges)f(the)h(en)o (vironmen)o(t)g(in)75 1493 y(whic)o(h)c(the)g(MPI)g(routine)g(is)g(op)q (erating)f(\(for)g(example,)h(consuming)h(all)f(a)o(v)m(ailable)h (memory)e(space\),)75 1549 y(the)e(MPI)f(routine)h(should)h(b)q(eha)o (v)o(e)f(as)f(appropriate)h(for)f(that)g(situation)h(\(in)g (particular,)g(in)g(this)g(case,)75 1606 y(the)h(b)q(eha)o(vior)h (should)g(b)q(e)g(the)f(same)g(as)g(for)g(a)g(m)o(ultithreaded)h(MPI)f (implemen)o(tation\).)166 1662 y(A)j(second)i(e\013ect)e(is)h(that)f(a) g(signal)h(handler)h(that)e(do)q(es)h(MPI)f(calls)i(m)o(ust)e(not)g(in) o(terfere)h(with)75 1719 y(the)c(op)q(eration)f(of)g(MPI.)g(F)l(or)g (example,)h(an)g(MPI)f(receiv)o(e)i(of)e(an)o(y)g(t)o(yp)q(e)h(that)e (o)q(ccurs)i(within)h(a)e(signal)75 1775 y(handler)22 b(m)o(ust)f(not)g(cause)h(erroneous)f(b)q(eha)o(vior)h(b)o(y)f(the)h (MPI)f(implemen)o(tation.)39 b(Note)21 b(that)g(an)75 1832 y(implemen)o(tation)16 b(is)g(p)q(ermitted)g(to)e(prohibit)i(the)f (use)h(of)e(MPI)h(calls)h(from)e(within)j(a)d(signal)i(handler.)p eop %%Page: 90 95 90 94 bop 75 359 a Fn(Chapter)34 b(8)75 570 y Fu(Pro\014ling)75 814 y Fm(8.1)70 b(Requiremen)n(ts)75 921 y Fs(T)l(o)15 b(meet)g(the)g(MPI)g(pro\014ling)i(in)o(terface,)e(an)g(implemen)o (tation)i(of)d(the)i(MPI)f(functions)h Fh(must)131 1027 y Fs(1.)22 b(pro)o(vide)14 b(a)f(mec)o(hanism)h(through)f(whic)o(h)h (all)g(of)f(the)g(MPI)h(de\014ned)h(functions)f(ma)o(y)e(b)q(e)i (accessed)189 1083 y(with)e(a)g(name)f(shift.)20 b(Th)o(us)11 b(all)i(of)f(the)g(MPI)g(functions)h(\(whic)o(h)f(normally)g(start)f (with)h(the)g(pre\014x)189 1140 y(\\)p Fl(MPI)p 287 1140 15 2 v 16 w Fs("\))j(should)h(also)f(b)q(e)h(accessable)g(with)g(the)f (pre\014x)h(\\)p Fl(PMPI)p 1300 1140 V 16 w Fs(".)131 1246 y(2.)22 b(ensure)c(that)f(those)h(MPI)g(functions)h(whic)o(h)f (are)g(not)f(replaced)i(ma)o(y)f(still)h(b)q(e)g(link)o(ed)g(in)o(to)f (an)189 1302 y(executable)e(image)f(without)h(causing)f(name)h (clashes.)131 1408 y(3.)22 b(do)q(cumen)o(t)c(the)f(implemen)o(tation)i (of)e(di\013eren)o(t)h(language)g(bindings)h(of)e(the)h(MPI)g(in)o (terface)f(if)189 1464 y(they)g(are)h(la)o(y)o(ered)g(on)f(top)g(of)h (eac)o(h)f(other,)h(so)f(that)g(the)h(pro\014ler)g(dev)o(elop)q(er)h (kno)o(ws)e(whether)189 1521 y(she)d(m)o(ust)f(implemen)o(t)i(the)e (pro\014le)i(in)o(terface)f(for)f(eac)o(h)h(binding,)h(or)e(can)h (economise)h(b)o(y)e(imple-)189 1577 y(men)o(ting)i(it)h(only)f(for)g (the)g(lo)o(w)o(est)g(lev)o(el)h(routines.)75 1737 y Fm(8.2)70 b(Discussion)75 1845 y Fs(The)13 b(ob)s(jectiv)o(e)g(of)g (the)g(MPI)g(pro\014ling)h(in)o(terface)g(is)f(to)f(ensure)i(that)e(it) i(is)f(relativ)o(ely)h(easy)f(for)g(authors)75 1901 y(of)18 b(pro\014ling)i(\(and)f(other)f(similar\))h(to)q(ols)g(to)f(in)o (terface)h(their)g(co)q(des)g(to)f(MPI)h(implemen)o(tations)h(on)75 1958 y(di\013eren)o(t)15 b(mac)o(hines.)166 2017 y(Since)20 b(MPI)e(is)h(a)f(mac)o(hine)h(indep)q(enden)o(t)h(standard)e(with)h (man)o(y)f(di\013eren)o(t)g(implemen)o(tations,)75 2074 y(it)e(is)h(unreasonable)g(to)e(exp)q(ect)i(that)f(the)g(authors)f(of)h (pro\014ling)h(to)q(ols)f(for)g(MPI)g(will)i(ha)o(v)o(e)d(access)i(to) 75 2130 y(the)c(source)h(co)q(de)f(whic)o(h)h(implemen)o(ts)h(MPI)e(on) g(an)o(y)g(particular)h(mac)o(hine.)20 b(It)13 b(is)h(therefore)f (necessary)75 2187 y(to)21 b(pro)o(vide)i(a)f(mec)o(hanism)g(b)o(y)g (whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o(h)g(to)q(ols)g(can)g (collect)i(whatev)o(er)75 2243 y(p)q(erformance)15 b(information)h (they)f(wish)h Fh(without)g Fs(access)f(to)g(the)g(underlying)i (implemen)o(tation.)166 2302 y(W)l(e)d(b)q(eliev)o(e)i(that)e(ha)o (ving)g(suc)o(h)g(an)h(in)o(terface)f(is)h(imp)q(ortan)o(t)e(if)i(MPI)f (is)g(to)g(b)q(e)h(attractiv)o(e)e(to)g(end)75 2359 y(users,)i(since)i (the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g(di\013eren)o(t)h(to)q (ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h(factor)f(in)h(attracting) 75 2415 y(users)f(to)g(the)g(MPI)g(standard.)166 2475 y(The)e(pro\014ling)h(in)o(terface)e(is)h(just)g(that,)f(an)g(in)o (terface.)20 b(It)12 b(sa)o(ys)g Fh(nothing)g Fs(ab)q(out)g(the)h(w)o (a)o(y)f(in)h(whic)o(h)75 2531 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f(to)g(la)o(y)h(do)o(wn)g(what)g (information)g(is)h(collected)g(through)75 2588 y(the)g(in)o(terface,)g (or)g(ho)o(w)g(the)g(collected)i(information)e(is)h(sa)o(v)o(ed,)e (\014ltered,)i(or)f(displa)o(y)o(ed.)166 2647 y(While)g(the)g(initial)h (imp)q(etus)f(for)f(the)g(dev)o(elopmen)o(t)h(of)e(this)i(in)o(terface) f(arose)g(from)f(the)h(desire)i(to)75 2704 y(p)q(ermit)g(the)f (implemen)o(tation)i(of)d(pro\014ling)j(to)q(ols,)d(it)i(is)f(clear)h (that)f(an)g(in)o(terface)g(lik)o(e)h(that)f(sp)q(eci\014ed)952 2828 y(90)p eop %%Page: 91 96 91 95 bop 75 -100 a Fq(8.3.)34 b(LOGIC)16 b(OF)f(THE)g(DESIGN)1115 b Fs(91)75 45 y(ma)o(y)17 b(also)h(pro)o(v)o(e)f(useful)i(for)e(other)h (purp)q(oses,)g(suc)o(h)g(as)g(\\in)o(ternet)o(w)o(orking")f(m)o (ultiple)i(MPI)f(imple-)75 102 y(men)o(tations.)j(Since)c(all)f(that)f (is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there)h(is)g(no)f(ob)s (jection)h(to)f(its)h(b)q(eing)g(used)75 158 y(wherev)o(er)f(it)h(is)f (useful.)166 214 y(As)j(the)g(issues)h(b)q(eing)g(addressed)f(here)h (are)f(in)o(timately)h(tied)f(up)h(with)f(the)g(w)o(a)o(y)f(in)i(whic)o (h)g(ex-)75 271 y(ecutable)j(images)e(are)g(built,)j(whic)o(h)e(ma)o(y) f(di\013er)h(greatly)f(on)g(di\013eren)o(t)h(mac)o(hines,)h(the)e (examples)75 327 y(giv)o(en)c(b)q(elo)o(w)g(should)h(b)q(e)f(treated)g (solely)g(as)f(one)h(w)o(a)o(y)f(of)g(implemen)o(ting)j(the)d(ob)s (jectiv)o(e)h(of)f(the)h(MPI)75 384 y(pro\014ling)h(in)o(terface.)k (The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o (tation)h(are)e(those)g(detailed)75 440 y(in)i(the)f(Requiremen)o(ts)i (section)e(ab)q(o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this) i(c)o(hapter)f(is)g(only)h(presen)o(t)f(as)75 497 y(justi\014cation)g (and)f(discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.) 166 553 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f (in)h(whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed) 75 610 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 666 y(equally)d(v)m(alid\).)75 806 y Fm(8.3)70 b(Logic)22 b(of)i(the)e(design)75 907 y Fs(Pro)o(vided)16 b(that)f(an)g(MPI)g (implemen)o(tation)i(meets)e(the)h(requiremen)o(ts)g(ab)q(o)o(v)o(e,)e (it)i(is)g(p)q(ossible)h(for)e(the)75 964 y(implemen)o(tor)20 b(of)e(the)h(pro\014ling)h(system)f(to)f(in)o(tercept)i(all)f(of)g(the) g(MPI)g(calls)h(whic)o(h)f(are)g(made)g(b)o(y)75 1020 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h(collect)g(whatev)o (er)f(information)g(she)h(requires)g(b)q(efore)f(calling)75 1076 y(the)c(underlying)i(MPI)d(implemen)o(tation)i(\(through)f(its)g (name)f(shifted)i(en)o(try)e(p)q(oin)o(ts\))h(to)f(ac)o(hiev)o(e)i(the) 75 1133 y(desired)g(e\013ects.)75 1272 y Fm(8.4)70 b(Examples)75 1376 y Fi(8.4.1)55 b(Pro\014ler)18 b(implem)o(en)n(tation)75 1461 y Fs(Supp)q(ose)25 b(that)e(the)h(pro\014ler)h(wishes)g(to)e (accum)o(ulate)i(the)f(total)f(amoun)o(t)g(of)h(data)f(sen)o(t)h(b)o(y) g(the)75 1518 y(MPI)p 167 1518 14 2 v 16 w(SEND)18 b(function,)g(along) f(with)h(the)f(total)g(elapsed)h(time)f(sp)q(en)o(t)h(in)g(the)f (function.)27 b(This)18 b(could)75 1574 y(trivially)f(b)q(e)f(ac)o (hiev)o(ed)g(th)o(us)f(:{)166 1631 y Fi(W)-6 b(ARNING)18 b(:)d Fs(syn)o(tax)f(correction)h(needed)i(in)f(examples)g(when)g(MPI)f (syn)o(tax)f(is)i(de\014ned.)75 1709 y Fl(static)23 b(int)g (totalBytes;)75 1766 y(static)g(int)g(totalTime;)75 1822 y(extern)g(int)g(now\(void\);)71 b(/*)23 b(Some)h(suitable)e(timer)i (function)e(!)i(*/)75 1935 y(int)f(MPI_SEND\(const)f(int)i(tag,)f (const)g(int)h(bytes,)f(void)g(*)h(buffer,)f(const)g(int)g(who\))75 1992 y({)147 2048 y(int)g(tstart)g(=)h(now\(\);)147 2105 y(int)f(result)g(=)h(PMPI_SEND\(tag,bytes,buffe)o(r,who\);)68 b(/*)24 b(Pass)f(on)h(all)f(the)g(arguments)g(*/)147 2217 y(totalBytes)f(+=)i(bytes;)405 b(/*)23 b(Accumulate)g(byte)g (count)g(*/)147 2274 y(totalTime)46 b(+=)24 b(now\(\))f(-)h(tstart;)190 b(/*)23 b(and)h(time)333 b(*/)147 2387 y(return)23 b(result;)75 2443 y(})75 2561 y Fi(8.4.2)55 b(MPI)19 b(library)f(implem)o(en)n (tation)75 2647 y Fs(On)g(a)f(Unix)h(system,)f(in)h(whic)o(h)g(the)g (MPI)f(library)h(is)g(implemen)o(ted)h(in)f(C,)f(then)g(there)h(are)e (v)m(arious)75 2704 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i(t)o (w)o(o)e(of)g(the)h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.)35 b(Whic)o(h)20 b(is)h(b)q(etter)p eop %%Page: 92 97 92 96 bop 75 -100 a Fs(92)1180 b Fq(CHAPTER)15 b(8.)35 b(PR)o(OFILING)75 45 y Fs(dep)q(ends)17 b(on)e(whether)g(the)h(link)o (er)g(and)f(compiler)i(supp)q(ort)e(w)o(eak)g(sym)o(b)q(ols.)75 165 y Ft(Systems)h(with)i(w)o(eak)f(sym)o(b)q(ols)75 250 y Fs(If)i(the)f(compiler)i(and)f(link)o(er)h(supp)q(ort)f(w)o(eak)f (external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g (system)75 307 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h (library)g(is)f(required)h(through)f(the)g(use)g(of)g Fl(#pragma)23 b(weak)13 b Fs(th)o(us)75 363 y(:{)75 465 y Fl(#pragma)23 b(weak)g(MPI_EXAMPLE)g(=)g(PMPI_EXAMPLE)75 578 y(int)g(PMPI_EXAMPLE\(/*)f(appropriate)h(args)g(*/\))75 634 y({)170 691 y(/*)h(Useful)f(content)g(*/)75 747 y(})166 849 y Fs(The)d(e\013ect)g(of)g(this)g Fl(#pragma)f Fs(is)i(to)e (de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Fl(MPI)p 1472 849 15 2 v 17 w(EXAMPLE)f Fs(as)h(a)g(w)o(eak)75 906 y(de\014nition,)d(this)f(means)g(that)f(the)g(link)o(er)i(will)g (not)e(complain)i(if)f(there)g(is)g(another)f(de\014nition)j(of)d(the) 75 962 y(sym)o(b)q(ol)g(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1019 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1138 y Ft(Systems)h(without)j(w)o(eak)d(sym)o(b)q (ols)75 1224 y Fs(In)j(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h (then)g(one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h (the)f(C)h(macro)75 1280 y(pre-pro)q(cessor)c(th)o(us)g(:{)75 1382 y Fl(#ifdef)23 b(PROFILELIB)75 1438 y(#)95 b(ifdef)23 b(__STDC__)75 1495 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1551 y(#)95 b(else)75 1608 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 1664 y(#)95 b(endif)75 1721 y(#else)75 1777 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 1834 y(#endif)166 1935 y Fs(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2037 y Fl(int)23 b(FUNCTION\(MPI_EXAMPLE\)\(/*)e (appropriate)h(args)i(*/\))75 2094 y({)170 2150 y(/*)g(Useful)f (content)g(*/)75 2207 y(})166 2308 y Fs(The)d(same)g(source)g(\014le)h (can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q(duce)i(b)q(oth)f(v)o (ersions)g(of)g(the)g(library)l(,)75 2365 y(dep)q(ending)d(on)f(the)f (state)f(of)h(the)g Fl(PROFILELIB)f Fs(macro)h(sym)o(b)q(ol.)166 2421 y(It)e(is)g(required)h(that)e(the)h(standard)f(MPI)h(library)g(b)q (e)h(built)g(in)f(suc)o(h)g(a)g(w)o(a)o(y)e(that)h(the)h(inclusion)i (of)75 2478 y(MPI)g(functions)g(can)g(b)q(e)h(ac)o(hiev)o(ed)f(one)g (at)g(a)f(time.)20 b(This)c(is)f(a)f(somewhat)g(unpleasan)o(t)i (requiremen)o(t,)75 2534 y(since)22 b(it)f(ma)o(y)f(mean)h(that)f(eac)o (h)g(external)i(function)f(has)g(to)f(b)q(e)h(compiled)i(from)d(a)g (separate)g(\014le.)75 2591 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g (necessary)g(so)f(that)g(the)g(author)g(of)g(the)h(pro\014ling)g (library)g(need)h(only)f(de\014ne)g(those)75 2647 y(MPI)f(functions)g (whic)o(h)g(she)g(wishes)h(to)e(in)o(tercept,)h(references)g(to)f(an)o (y)g(others)g(b)q(eing)i(ful\014lled)h(b)o(y)e(the)75 2704 y(normal)g(MPI)g(library)l(.)21 b(Therefore)15 b(the)h(link)g (step)f(can)h(lo)q(ok)f(something)h(lik)o(e)g(this)g(:{)p eop %%Page: 93 98 93 97 bop 75 -100 a Fq(8.5.)34 b(MUL)l(TIPLE)16 b(LEVELS)g(OF)f(INTER)o (CEPTION)762 b Fs(93)75 45 y Fl(\045)24 b(cc)f(...)h(-lmyprof)f(-lpmpi) g(-lmpi)166 151 y Fs(Here)e Fl(libmyprof.a)e Fs(con)o(tains)i(the)g (pro\014ler)g(functions)h(whic)o(h)f(in)o(tercept)g(some)g(of)f(the)h (MPI)75 208 y(functions.)j Fl(libpmpi.a)15 b Fs(con)o(tains)i(the)f (\\name)h(shifted")g(MPI)f(functions,)h(and)g Fl(libmpi.a)e Fs(con)o(tains)75 264 y(the)g(normal)g(de\014nitions)i(of)e(the)g(MPI)h (functions.)75 407 y Fm(8.5)70 b(Multiple)20 b(lev)n(els)g(of)k(in)n (terception)75 509 y Fs(The)15 b(sc)o(heme)h(giv)o(en)g(here)f(do)q(es) h(not)f(directly)h(supp)q(ort)f(the)h(nesting)g(of)e(pro\014ling)j (functions,)e(since)i(it)75 565 y(pro)o(vides)f(only)f(a)g(single)h (alternativ)o(e)g(name)f(for)f(eac)o(h)h(MPI)g(function.)21 b(Consideration)16 b(w)o(as)e(giv)o(en)i(to)75 622 y(an)f(implemen)o (tation)i(whic)o(h)f(w)o(ould)g(allo)o(w)g(m)o(ultiple)h(lev)o(els)g (of)e(call)h(in)o(terception,)h(ho)o(w)o(ev)o(er)d(w)o(e)h(w)o(ere)75 678 y(unable)j(to)e(construct)g(an)g(implemen)o(tation)i(of)e(this)h (whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o(wing)i(disadv)m(an-) 75 735 y(tages)143 829 y Fo(\017)23 b Fs(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 922 y Fo(\017)23 b Fs(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o(en) g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 1016 y(Since)i(one)e(of)g(the)h(ob)s(jectiv)o(es)f(of)g(MPI)h(is)f(to)g (p)q(ermit)h(e\016cien)o(t,)g(lo)o(w)f(latency)h(implemen)o(tations,)h (and)75 1073 y(it)j(is)f(not)g(the)h(business)g(of)f(a)g(standard)g(to) g(require)h(a)f(particular)h(implemen)o(tation)g(language,)g(w)o(e)75 1129 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h(outlined)h(ab)q(o)o(v) o(e.)166 1186 y(Note,)f(ho)o(w)o(ev)o(er,)f(that)h(it)h(is)f(p)q (ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o(v)o(e)g(to)g(implemen) o(t)i(a)e(m)o(ulti-lev)o(el)75 1242 y(system,)k(since)g(the)g(function) g(called)i(b)o(y)d(the)h(user)f(ma)o(y)g(call)i(man)o(y)e(di\013eren)o (t)g(pro\014ling)i(functions)75 1299 y(b)q(efore)15 b(calling)i(the)f (underlying)h(MPI)e(function.)166 1355 y(Unfortunately)22 b(suc)o(h)g(an)g(implemen)o(tation)h(ma)o(y)e(require)h(more)g(co-op)q (eration)g(b)q(et)o(w)o(een)g(the)75 1411 y(di\013eren)o(t)g (pro\014ling)i(libraries)g(than)e(is)h(required)g(for)e(the)i(single)g (lev)o(el)h(implemen)o(tation)f(detailed)75 1468 y(ab)q(o)o(v)o(e.)p eop %%Page: 94 99 94 98 bop 75 359 a Fn(Chapter)34 b(9)75 569 y Fu(Maps,)39 b(Groups)h(and)f(Con)m(texts)431 794 y Fs(Lyndon)16 b(Clark)o(e,)e (Mark)h(Sears,)f(T)l(on)o(y)h(Skjellum,)i(Marc)d(Snir)75 952 y Fm(9.1)70 b(In)n(tro)r(duction)75 1059 y Fs(MPI)20 b(pro)o(vides)h(supp)q(ort)g(for)f(the)g(execution)i(of)e Ft(parallel)25 b(pro)q(cedures)p Fs(.)36 b(A)20 b(parallel)i(pro)q (cedure)75 1115 y(is)f(executed)g(collectiv)o(ely)h(b)o(y)e(a)g(set)g (of)g(comm)o(unicating)h(pro)q(cesses.)35 b(T)l(ransfer)20 b(of)f(con)o(trol)h(to)g(the)75 1172 y(parallel)d(pro)q(cedure,)e(and)h (bac)o(k,)e(is)i(ac)o(hiev)o(ed)g(b)o(y)f(ha)o(ving)g(eac)o(h)g (executing)i(pro)q(cess)e(transfer)f(con)o(trol)75 1228 y(to)h(the)i(lo)q(cal)g(pro)q(cedure)g(co)q(de,)f(and)h(return)f(from)f (it.)23 b(Not)15 b(all)i(pro)q(cesses)g(need)g(transfer)e(con)o(trol)h (to)75 1284 y(the)f(same)g(parallel)h(pro)q(cedure;)g(the)f(parallel)h (pro)q(cedure)g(ma)o(y)f(in)o(v)o(olv)o(e)g(only)h(a)e(subset)h(of)g (pro)q(cesses,)75 1341 y(or)g(di\013eren)o(t)i(parallel)g(calls)g(ma)o (y)f(b)q(e)g(executed)h(b)o(y)f(di\013eren)o(t)g(subsets)h(of)e(pro)q (cesses.)23 b(The)16 b(collectiv)o(e)75 1397 y(comm)o(unication)h (calls)h(pro)o(vided)f(b)o(y)g(MPI)f(are)g(examples)i(of)e(suc)o(h)h (parallel)h(pro)q(cedures.)25 b(Libraries)75 1454 y(suc)o(h)16 b(as)e(parallel)j(scien)o(ti\014c)g(libraries)g(will)g(b)q(e)e(another) g(example.)166 1513 y(It)d(is)h(highly)h(desirable)g(to)e(allo)o(w)g (the)h(pro)q(cesses)g(that)e(execute)j(a)e(parallel)i(pro)q(cedure)f (use)g(a)f(\\vir-)75 1569 y(tual)k(pro)q(cess)g(name)f(space")h(lo)q (cal)g(to)f(the)h(in)o(v)o(ok)m(ation.)21 b(Th)o(us,)15 b(the)h(co)q(de)g(of)f(the)h(parallel)h(pro)q(cedure)75 1626 y(will)j(lo)q(ok)e(iden)o(tical,)i(irresp)q(ectiv)o(e)f(of)f(the)g (absolute)g(addresses)g(of)f(the)h(executing)h(pro)q(cesses.)29 b(It)17 b(is)75 1682 y(often)e(the)f(case)h(that)f(parallel)j (application)f(co)q(de)g(is)f(built)h(b)o(y)f(comp)q(osing)g(sev)o (eral)g(parallel)i(mo)q(dules)75 1739 y(\(e.g.,)e(a)h(n)o(umerical)i (solv)o(er,)e(and)h(a)f(graphic)h(displa)o(y)g(mo)q(dule\).)24 b(Supp)q(ort)17 b(of)f(a)g(virtual)h(name)g(space)75 1795 y(for)j(eac)o(h)g(mo)q(dule)i(will)g(allo)o(w)f(to)e(comp)q(ose)i (mo)q(dules)g(that)f(w)o(ere)g(dev)o(elop)q(ed)i(separately)f(without) 75 1852 y(c)o(hanging)d(all)g(message)f(passing)g(calls)h(within)h(eac) o(h)e(mo)q(dule.)27 b(The)17 b(set)g(of)g(pro)q(cesses)h(that)e (execute)75 1908 y(a)i(parallel)i(pro)q(cedure)f(ma)o(y)f(b)q(e)h (\014xed,)g(or)f(ma)o(y)g(b)q(e)h(determined)g(dynamically)i(b)q(efore) d(the)h(in)o(v)o(ok)m(a-)75 1965 y(tion.)g(Th)o(us,)13 b(MPI)h(has)e(to)h(pro)o(vide)g(a)g(mec)o(hanism)h(for)e(creating)i (dynamically)g(sets)f(of)g(lo)q(cally)i(named)75 2021 y(pro)q(cesses.)35 b(W)l(e)20 b(alw)o(a)o(ys)f(n)o(um)o(b)q(er)h(pro)q (cesses)h(that)e(execute)h(a)g(parallel)i(pro)q(cedure)f(consecutiv)o (ely)l(,)75 2077 y(starting)15 b(form)g(zero.)21 b(Th)o(us,)16 b(a)f Ft(group)h Fs(is)g(an)f(ordered)h(set)g(of)f(pro)q(cesses.)22 b(Eac)o(h)15 b(pro)q(cess)h(in)h(a)e(group)75 2134 y(is)h(asso)q (ciated)f(with)h(a)e Ft(rank)p Fs(,)h(starting)f(from)h(zero.)20 b(Pro)q(cesses)15 b(are)g(iden)o(ti\014ed)i(b)o(y)e(their)h(ranks)e (when)75 2190 y(comm)o(unication)i(o)q(ccurs)f(within)i(the)e(group.) 166 2249 y(Another)g(imp)q(ortan)o(t)g(goal)g(is)g(the)h(supp)q(ort)f (of)g(a)f Ft(lo)q(osely)19 b(sync)o(hronous)13 b Fs(transfer)i(of)g (con)o(trol:)75 2306 y(no)f(sync)o(hronization)g(o)q(ccurs)h(either)f (b)q(efore)g(or)f(after)h(the)f(call.)21 b(Th)o(us,)14 b(a)f(pro)q(cess)h(ma)o(y)f(start)g(sending)75 2362 y(messages)g(that)g (p)q(ertain)h(to)f(the)g(execution)i(of)e(a)g(parallel)i(pro)q(cedure)g (b)q(efore)e(all)i(other)e(participating)75 2419 y(pro)q(cesses)k(ha)o (v)o(e)f(joined)i(the)e(execution;)i(a)e(pro)q(cess)h(ma)o(y)f(receiv)o (e)i(a)e(message)g(that)g(p)q(ertains)h(to)f(the)75 2475 y(execution)e(of)e(a)g(parallel)i(pro)q(cedure,)g(while)g (participating)f(in)h(another)e(parallel)i(execution.)20 b(A)13 b Ft(com-)75 2532 y(m)o(unication)20 b(con)o(text)c Fs(mec)o(hanism)h(is)f(needed)i(to)d(distinguish)j(comm)o(unication)f (that)f(b)q(elongs)h(to)75 2588 y(distinct)f(parallel)h(pro)q(cedure)f (executions.)21 b(\(Ev)o(en)15 b(if)g(parallel)i(transfer)d(of)h(con)o (trol)g(is)h(executed)g(syn-)75 2645 y(c)o(hroneously)l(,)f(one)g (still)i(needs)e(a)g(con)o(text)f(mec)o(hanism)h(for)g(the)f(sync)o (hronization)i(calls)g(themselv)o(es.\))166 2704 y(Normally)l(,)24 b(a)e(parallel)h(pro)q(cedure)g(is)g(written)f(so)g(that)f(all)i (messages)f(pro)q(duced)h(during)g(its)952 2828 y(94)p eop %%Page: 95 100 95 99 bop 75 -100 a Fq(9.2.)34 b(MAPS)1517 b Fs(95)75 45 y(execution)17 b(are)e(also)h(consumed)g(b)o(y)g(the)g(pro)q(cesses) g(that)f(execute)h(the)g(pro)q(cedure.)22 b(Ho)o(w)o(ev)o(er,)15 b(if)h(one)75 102 y(parallel)k(pro)q(cedure)g(calls)f(another,)g(then)g (it)g(migh)o(t)f(b)q(e)i(desirable)g(to)e(allo)o(w)g(suc)o(h)h(call)h (to)e(pro)q(ceed)75 158 y(while)k(messages)e(are)h(p)q(ending)h(\(the)e (messages)g(will)j(b)q(e)e(consumed)g(b)o(y)g(the)f(pro)q(cedure)i (after)e(the)75 214 y(call)e(returns\).)24 b(In)18 b(suc)o(h)f(case,)f (a)h(new)g(comm)o(unication)g(con)o(text)g(is)g(needed)h(for)e(the)h (called)h(parallel)75 271 y(pro)q(cedure,)e(ev)o(en)f(if)h(the)f (transfer)g(of)f(con)o(trol)h(is)h(sync)o(hronized.)166 327 y(A)d Ft(con)o(text)h Fs(is)g(the)f(MPI)g(mec)o(hanism)h(for)f (partitioning)h(comm)o(unication)g(space.)20 b(A)13 b(send)h(made)75 384 y(in)k(a)f(con)o(text)g(cannot)g(b)q(e)i(receiv)o(ed)f(in)g (another)g(con)o(text.)26 b(Con)o(texts)16 b(are)h(iden)o(ti\014ed)j (in)e(MPI)f(using)75 440 y(in)o(teger-v)m(alued)g Ft(con)o(text)p 540 440 16 2 v 19 w(id)p Fs('s.)166 497 y(The)d(comm)o(unication)i (domain)e(used)h(b)o(y)f(a)g(parallel)i(pro)q(cedure)g(is)e(iden)o (ti\014ed)j(b)o(y)d(a)g Ft(comm)o(uni-)75 553 y(cator)p Fs(.)29 b(Comm)o(unicators)17 b(bring)h(together)f(the)h(concepts)h(of) e(pro)q(cess)h(group)g(and)g(comm)o(unication)75 610 y(con)o(text.)29 b(A)18 b(comm)o(unicator)g(is)h(an)f(explicit)j (parameter)c(in)j(eac)o(h)e(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)75 666 y(op)q(eration.)31 b(The)19 b(comm)o(unicator)f(iden) o(ti\014es)j(the)d(comm)o(unication)i(con)o(text)e(of)g(that)g(op)q (eration;)j(it)75 723 y(iden)o(ti\014es)f(the)f(group)f(of)g(pro)q (cesses)g(that)g(can)g(b)q(e)h(in)o(v)o(olv)o(ed)h(in)f(this)g(comm)o (unication;)h(and)e(it)h(pro-)75 779 y(vides)g(the)g(translation)f (from)g(virtual)h(pro)q(cess)g(names,)g(whic)o(h)g(are)f(ranks)g (within)i(the)e(group,)h(in)o(to)75 835 y(absolute)13 b(addresses.)20 b(Collectiv)o(e)14 b(comm)o(unication)g(calls)g(also)f (tak)o(e)f(a)h(comm)o(unicator)f(as)h(parameter;)75 892 y(it)k(is)h(exp)q(ected)g(that)e(parallel)j(libraries)g(will)f(b)q(e)g (built)g(to)f(accept)g(a)g(comm)o(unicator)f(as)h(parameter.)75 948 y(Comm)o(unicators)d(are)h(represen)o(ted)h(b)o(y)f(opaque)g(MPI)h (ob)s(jects.)166 1005 y(MPI)g(do)q(es)h(not)e(pro)o(vide)i(for)f (absolute)g(pro)q(cess)h(names.)22 b(Rather,)17 b(pro)q(cesses)f(are)g (alw)o(a)o(ys)g(iden-)75 1061 y(ti\014ed)h(b)o(y)g(their)g(rank)f (inside)i(a)e(group.)23 b(A)16 b(univ)o(ersal)i(group)e(that)g (includes)i(all)g(pro)q(cesses)e(a)o(v)m(ailable)75 1118 y(when)k(computation)f(starts)f(is)i(the)g(nearest)f(equiv)m(alen)o(t)i (to)e(an)g(absolute)h(pro)q(cess)g(name)f(space)h(in)75 1174 y(MPI.)i(Relativ)o(e)i(naming)e(is)h(consisten)o(t)g(with)f(MPI)h (implemen)o(tations)g(where)g(pro)q(cesses)g(can)f(b)q(e)75 1231 y(added)16 b(dynamically)l(.)166 1287 y(New)j(pro)q(cess)g(groups) f(are)h(built)h(b)o(y)f(subsetting)g(and)g(reordering)g(pro)q(cesses)g (within)h(existing)75 1344 y(groups)i(\(as)f(de\014ned)j(b)o(y)e(comm)o (unicators\).)40 b(A)23 b(new)f(group)g(is)h(de\014ned)h(from)d(an)h (old)h(group)f(b)o(y)75 1400 y(sp)q(ecifying)d(the)f(corresp)q(ondence) h(b)q(et)o(w)o(een)e(the)h(rank)f(of)g(eac)o(h)h(pro)q(cess)f(in)i(the) e(new)h(group)f(and)h(its)75 1456 y(rank)f(in)h(the)f(old)h(group.)26 b(Suc)o(h)18 b(corresp)q(ondence)g(is)g(called)h(a)e Ft(map)p Fs(;)h(it)f(is)h(represen)o(ted)g(in)g(MPI)f(b)o(y)75 1513 y(an)e(opaque)g(ob)s(ject.)75 1656 y Fm(9.2)70 b(Maps)75 1757 y Fs(A)19 b Ft(map)g Fs(is)g(a)g(one-to)f(one-mapping)i(from)e(0)p Fr(:::m)11 b Fo(\000)i Fs(1)19 b(in)o(to)g(the)g(set)f(of)h(non)g (negativ)o(e)g(in)o(tegers;)h Fr(m)75 1814 y Fs(is)e(the)g Ft(size)g Fs(of)f(map)g Fr(f)5 b Fs(.)28 b(A)17 b(p)q(ossible)j (represen)o(tation)d(for)g(suc)o(h)h(map)g(is)g(a)f(list)h(of)g Fr(m)f Fs(in)o(tegers;)i(it)e(is)75 1870 y(often)d(con)o(v)o(enien)o(t) h(to)e(think)j(of)d(a)h(map)g(as)g(b)q(eing)i(suc)o(h)f(list.)20 b(Another)14 b(p)q(ossible)i(represen)o(tation)f(is)f(as)75 1927 y(an)j(algorithmic)g(sp)q(eci\014cation)i(of)d(the)h(map)f (\(e.g.,)g(hash)h(function\).)25 b(A)16 b(map)h(is)g(represen)o(ted)g (b)o(y)g(an)75 1983 y(opaque)e(map)g(ob)s(ject.)166 2040 y(Maps)e(are)g(used)h(in)h(order)e(to)g(represen)o(t)h(groups.)19 b(If)14 b(pro)q(cesses)g(are)f(n)o(um)o(b)q(ered)h(from)f(0)g(to)g Fr(n)7 b Fo(\000)g Fs(1,)75 2096 y(then)15 b(a)f(map)g Fr(f)k Fs(:)12 b(0)p Fr(:::m)7 b Fo(\000)i Fs(1)j Fo(!)h Fs(0)p Fr(:::n)7 b Fo(\000)i Fs(1)14 b(represen)o(ts)h(a)f(subset)g (group)g(of)g Fr(m)g Fs(pro)q(cesses:)20 b(pro)q(cess)15 b Fr(f)5 b Fs(\()p Fr(i)p Fs(\))75 2153 y(has)16 b(rank)f Fr(i)h Fs(in)h(the)f(new)g(group.)22 b(A)16 b(map,)f(b)o(y)h(itself,)h (do)q(es)f(not)f(represen)o(t)h(a)g(group;)f(it)i(do)q(es)f(so)f(only) 75 2209 y(relativ)o(e)g(to)e(a)h(pre-de\014ned)i(con)o(taining)f (group,)f(i.e.,)g(relativ)o(e)h(to)e(another)h(ranking)h(of)f(the)g (pro)q(cesses.)166 2342 y Fk(Discussion:)51 b Fj(Should)17 b(decide)i(if)f(opaque)g(ob)r(jects)h(are)g(justi\014ed,)g(or)f (whether)h(an)f(arra)o(y)g(of)g(indices)75 2391 y(is)f(OK.)h(On)g(the)g (\\opaque)f(ob)r(ject")h(side:)26 b(Ma)o(y)17 b(ha)o(v)o(e)g (algorithmic)e(map)h(de\014nitions)h(and)g(ma)o(y)f(spread)i(the)75 2441 y(de\014nition)f(o)o(v)o(er)g(more)g(than)g(one)h(no)q(de,)g(on)f (a)g(v)o(ery)g(large)h(system.)28 b(On)17 b(the)h(explicit)f(represen)o (tation)i(side:)75 2491 y(The)g(represen)o(tation)h(is)e(simple,)g(w)o (e)h(a)o(v)o(oid)e(an)h(additional)f(ob)r(ject,)i(and)g(there)h(is)e (no)g(m)o(uc)o(h)f(w)o(aste.)33 b(Also,)75 2541 y(with)13 b(an)g(explicit)f(represen)o(tation,)j(the)f(user)g(can)f(easily)g(add) g(his/her)g(o)o(wn)g(map)f(constructors,)i(and)f(building)75 2591 y(comm)o(unicators)e(is)j(faster.)19 b(Need)c(to)f(understand)h (what)e(an)h(opaque)g(map)e(w)o(ould)h(b)q(e)i(in)e(F)m(ortran.)166 2647 y(If)j(maps)f(are)i(explictly)e(represen)o(ted)20 b(as)c(arra)o(ys,)h(then)g(one)f(needs)i(map)d(size)i(to)f(b)q(e)h(an)f (additional)e(pa-)75 2704 y(rameter.)p eop %%Page: 96 101 96 100 bop 75 -100 a Fs(96)694 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)166 45 y Fs(F)l(or)h(eac)o(h)g Fl(m)p Fs(,)h(an)f(iden)o(tit)o(y)i(map)e Fl(MPI)p 817 45 15 2 v 17 w(IDENT\(m\))f Fs(of)h(size)i Fl(m)e Fs(is)h (prede\014ned:)24 b Fl(MPI)p 1581 45 V 17 w(IDENT)p Fs(\()p Fl(m)p Fs(\))o(\()p Fr(i)p Fs(\))14 b(=)75 102 y Fr(i)h(;)8 b(i)j Fs(=)i(0)p Fr(;)8 b(:::;)g(m)f Fo(\000)k Fs(1.)166 241 y Fk(Discussion:)43 b Fj(If)16 b(maps)f(are)h(non)g(opaque)g(then)h (the)g(iden)o(t)f(map)e(is)i(just)h(a)f(long)f(arra)o(y)h(with)g Fc(i)g Fj(stored)75 297 y(at)h(the)g Fc(i)p Fj(-th)g(en)o(try)m(,)g (and)f(it)h(is)f(just)h Ff(MPI)p 733 297 14 2 v 15 w(IDENT)p Fj(.)e(If)h(maps)g(are)h(opaque,)g(w)o(e)g(need)g(a)f(di\013eren)o(t)i (map)d(for)h(eac)o(h)75 354 y(domain)c(size)i(\(a)g(function)g(from)e (in)o(tegers)i(to)g(maps?\))j(Need)e(to)f(think)f(of)h(F)m(ortran)f (implemen)o(tation.)75 555 y Fi(9.2.1)55 b(Op)r(erations)18 b(on)h(maps)75 641 y Fs(The)12 b(follo)o(wing)h(op)q(erations)f(are)g (de\014ned)h(on)f(maps.)19 b(Eac)o(h)12 b(function)h(is)f(in)o(v)o(ok)o (ed)h(lo)q(cally)g(b)o(y)f(a)g(pro)q(cess.)166 733 y Ft(MPI)p 275 733 16 2 v 18 w(MAP)p 419 733 V 18 w(SIZE\(map,)17 b(size\))75 856 y(IN)g(map)23 b Fs(handle)16 b(to)f(map)g(ob)s(ject)75 945 y Ft(OUT)j(size)23 b Fs(size)16 b(of)f(map)g(\(in)o(teger\))166 1068 y Ft(MPI)p 275 1068 V 18 w(MAP)p 419 1068 V 18 w(APPL)l(Y\()h (map,)h(argumen)o(t,)f(v)m(alue\))75 1191 y(IN)h(map)23 b Fs(handle)16 b(to)f(map)g(ob)s(ject)75 1279 y Ft(IN)i(argumen)o(t)22 b Fs(map)15 b(argumen)o(t)g(\(in)o(teger\))75 1368 y Ft(OUT)j(v)m(alue)24 b Fs(image)15 b(of)g Fl(argument)f Fs(under)i Fl(map)e Fs(\(in)o(teger\))166 1491 y Ft(MPI)p 275 1491 V 18 w(MAP)p 419 1491 V 18 w(INVERSE\()k(map,)e(argumen)o(t,)h (v)m(alue\))75 1614 y(IN)g(map)23 b Fs(handle)16 b(to)f(map)g(ob)s (ject)75 1702 y Ft(OUT)j(argumen)o(t)23 b Fs(in)o(v)o(erse)14 b(image)g(of)g Fl(value)f Fs(under)i Fl(map)p Fs(;)e(-1)h(if)h Fl(value)e Fs(is)i(not)e(in)i(the)f(range)g(of)g Fl(map)p Fs(.)75 1791 y Ft(IN)j(v)m(alue)23 b Fs(map)16 b(v)m(alue)g(\(in)o (teger\))166 1914 y Ft(MPI)p 275 1914 V 18 w(MAP)p 419 1914 V 18 w(LIST\(map,)i(len,)f(arra)o(y)p 924 1914 V 18 w(of)p 984 1914 V 19 w(in)o(teger,)g(size\))166 2006 y Fs(Creates)d(an)i(explicit)h(list)f(represen)o(tation)f(of)g(a)g (map.)75 2086 y Ft(IN)i(map)23 b Fs(handle)16 b(to)f(map)g(ob)s(ject)75 2175 y Ft(IN)i(LEN)24 b Fs(length)15 b(of)g(arra)o(y)f(\(in)o(teger\)) 75 2263 y Ft(OUT)k(arra)o(y)p 332 2263 V 18 w(of)p 392 2263 V 19 w(in)o(teger)23 b Fs(list)16 b(of)f(v)m(alues)h(in)g(the)f (range)g(of)g Fl(map)75 2351 y Ft(OUT)j(size)23 b Fs(length)16 b(of)f(returned)h(list)g({)e(size)j(of)d(map)h(\(in)o(teger\).)166 2508 y Fk(Discussion:)166 2564 y Fj(I)i(c)o(hanged)h(terminology)c (from)i(\\elemen)o(t,)h(rank")f(to)h(\\argumen)o(t,)g(v)n(alue")f (since)i(b)q(oth)f(argumen)o(t)f(and)75 2621 y(v)n(alue)d(are)i(ranks.) p eop %%Page: 97 102 97 101 bop 75 -100 a Fq(9.2.)34 b(MAPS)1517 b Fs(97)75 45 y Fi(9.2.2)55 b(Map)19 b(constructors)75 132 y Fs(The)c(map)g (constructors)g(ma)o(y)f(b)q(e)i(either)g(lo)q(cal)g(or)f(ma)o(y)g (require)h(comm)o(unication.)166 224 y Ft(MPI)p 275 224 16 2 v 18 w(MAP)p 419 224 V 18 w(BUILD\(map,)i(arra)o(y)p 877 224 V 17 w(of)p 936 224 V 19 w(in)o(teger,)f(size\))166 317 y Fs(Build)g(a)d(map)g(from)g(an)h(explicit)i(list)f(represen)o (tation.)j(This)d(function)f(is)g(called)i(lo)q(cally)f(b)o(y)f(one)75 373 y(pro)q(cess.)75 469 y Ft(OUT)j(map)23 b Fs(handle)16 b(to)f(map)g(ob)s(ject)75 565 y Ft(IN)i(arra)o(y)p 277 565 V 18 w(of)p 337 565 V 19 w(in)o(teger)23 b Fs(list)16 b(of)e(v)m(alues)j(in)f(the)f(range)g(of)g Fl(map)75 660 y Ft(IN)i(size)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h (arra)o(y)e({)h(map)g(size)h(\(in)o(teger\))166 792 y Ft(MPI)p 275 792 V 18 w(MAP)p 419 792 V 18 w(SPLIT\(comm,)h(map,)g(k)o (ey)l(,)e(index,)i(newmap\))166 884 y Fs(This)22 b(is)g(a)f(collectiv)o (e)j(function)e(that)f(is)h(called)h(b)o(y)f(all)g(pro)q(cesses)g(in)h (the)e(group)h(asso)q(ciated)75 941 y(with)17 b Fl(\(comm,)23 b(map\))p Fs(.)g(All)18 b(calling)g(pro)q(cesses)f(pro)o(vide)g(the)g (same)f(v)m(alues)h(for)f(the)h(parameters)f Fl(comm)75 997 y Fs(and)h Fl(map)p Fs(.)25 b(A)17 b(separate)f(group)g(of)h(pro)q (cesses)g(is)h(formed)e(for)h(eac)o(h)f(distinct)j(v)m(alue)f(of)e Fl(key)p Fs(,)h(with)g(the)75 1053 y(pro)q(cesses)f(ordered)g (according)g(to)f(the)h(v)m(alue)h(of)e Fl(index)p Fs(.)21 b(Eac)o(h)15 b(pro)q(cess)h(in)h(suc)o(h)f(group)f(is)i(returned)75 1110 y(in)f Fl(newmap)e Fs(the)i(map)f(of)f(this)i(group)f(within)h Fl(comm)p Fs(.)75 1219 y Ft(IN)h(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1314 y Ft(IN)i(map)23 b Fs(map)15 b(ob)s(ject)g(handle)75 1410 y Ft(IN)i(k)o(ey)22 b Fs(\(in)o(teger\))75 1506 y Ft(IN)17 b(index)23 b Fs(\(in)o(teger\))75 1602 y Ft(OUT)18 b(newmap)23 b Fs(map)15 b(ob)s(ject)f(handle)166 1711 y(Additional)k(map)f(creation)g(functions)g(ma)o(y)f(b)q(e)h(pro)o (vided.)25 b(F)l(ollo)o(ws)17 b(a)f(p)q(ossible)i(list)g(construc-)75 1767 y(tors.)166 1907 y Fk(Discussion:)d Fj(Do)f(w)o(e)g(w)o(an)o(t)f (to)h(mandate)f(them)g(in)g(MPI?)166 2082 y Ft(MPI)p 275 2082 V 18 w(MAP)p 419 2082 V 18 w(UNION\()k(map1,)h(map2,)f (newmap\))166 2174 y Fl(map1)g Fs(and)h Fl(map2)f Fs(are)h(maps)f(with) i(disjoin)o(t)f(ranges.)27 b(The)18 b(resulting)h(concatenation)f(is)h (a)e(map)75 2231 y(with)d(a)g(range)g(whic)o(h)h(is)f(the)g(union)h(of) f(the)g(ranges)g(of)f Fl(map1)h Fs(and)g Fl(map2)p Fs(,)f(with)i (elemen)o(ts)f(in)h(the)f(range)75 2287 y(of)h Fl(map1)f Fs(listed)i(\014rst,)e(and)h(original)h(order)f(preserv)o(ed)g(within)i (eac)o(h)e(range.)k(The)c(union)h(op)q(eration)f(is)75 2343 y(asso)q(ciativ)o(e)h(but)f(not)g(comm)o(utativ)o(e.)415 2469 y Fl(newmap)o Fs(\()p Fr(i)p Fs(\))d(=)670 2397 y Fb(\()724 2441 y Fl(map1)o Fs(\()p Fr(i)p Fs(\))308 b(if)16 b Fr(i)c(<)h(siz)r(e)p Fs(\()p Fl(map1)p Fs(\))724 2497 y Fl(map2)o Fs(\()p Fr(i)d Fo(\000)g Fr(siz)r(e)p Fs(\()p Fl(map1)p Fs(\)\))40 b(otherwise)75 2608 y Ft(IN)17 b(map1)23 b Fs(map)15 b(ob)s(ject)g(handle)75 2704 y Ft(IN)i(map2)23 b Fs(map)15 b(ob)s(ject)g(handle)p eop %%Page: 98 103 98 102 bop 75 -100 a Fs(98)694 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y Ft(OUT)j(newmap)23 b Fs(map)15 b(ob)s(ject)f(handle)166 194 y Ft(MPI)p 275 194 16 2 v 18 w(MAP)p 419 194 V 18 w(PR)o(ODUCT\()k(map1,)f(map2,)g (range,)g(newmap\))166 287 y Fl(map1)12 b Fs(is)h(a)g(map)f(with)i(v)m (alues)g(in)f(the)g(range)g(0)p Fr(;)8 b(:::;)g Fl(r)o(ange)r Fo(\000)d Fs(1.)20 b Fl(newmap)11 b Fs(is)j(the)f(cartesian)g(pro)q (duct)75 343 y(of)j Fl(map1)f Fs(and)h Fl(map2)p Fs(,)g(whic)o(h)g (maps)g(the)g(pair)h(\()p Fr(i;)8 b(j)s Fs(\))13 b(in)o(to)j(the)g (pair)h(\()p Fl(map1)o Fs(\()p Fr(i)p Fs(\))p Fr(;)8 b Fl(map2)n Fs(\()p Fr(j)s Fs(\)\).)20 b(P)o(airs)c(in)h(the)75 400 y(domain)f(and)g(in)g(the)f(range)g(of)g(this)h(mapping)g(are)g(n)o (um)o(b)q(ered)g(in)g(ro)o(w)f(ma)s(jor)f(order,)g(i.e.,)i(pair)f(\()p Fr(i;)8 b(j)s Fs(\))75 456 y(has)15 b(n)o(um)o(b)q(er)h Fr(i)9 b Fo(\001)h Fl(range)f Fs(+)i Fr(j)s Fs(.)19 b(The)d(pro)q(duct) f(op)q(eration)h(is)f(asso)q(ciativ)o(e)h(but)f(not)g(comm)o(utativ)o (e.)480 574 y Fl(newmap)o Fs(\()p Fr(i)10 b Fo(\001)f Fl(range)h Fs(+)g Fr(j)s Fs(\))i(=)h Fl(map1)o Fs(\()p Fr(i)p Fs(\))c Fo(\001)h Fl(range)f Fs(+)i Fl(map2)o Fs(\()p Fr(j)s Fs(\))75 687 y Ft(IN)17 b(map1)23 b Fs(map)15 b(ob)s(ject)g(handle)75 786 y Ft(IN)i(map2)23 b Fs(map)15 b(ob)s(ject)g(handle)75 885 y Ft(OUT)j(newmap)23 b Fs(map)15 b(ob)s(ject)f(handle)75 1036 y Fm(9.3)70 b(Con)n(text)p 478 1036 21 2 v 24 w(id)75 1140 y Fs(A)18 b Ft(con)o(text)p 299 1140 16 2 v 19 w(id)g Fs(is)h(an)f(in)o(teger.)28 b(The)18 b(range)g(of)f(v)m(alid)j(v)m(alues)f(for)e Fl(context)p 1438 1140 15 2 v 17 w(id)g Fs(is)i(implemen)o(tation)75 1197 y(dep)q(enden)o(t,)d(and)g(can)f(b)q(e)h(found)f(b)o(y)h(calling)h (a)d(suitable)j(query)e(function,)h(as)f(describ)q(ed)i(in)f(7.)75 1326 y Fi(9.3.1)55 b(Op)r(erations)18 b(on)h(con)n(text)p 797 1326 17 2 v 19 w(id's)75 1450 y Ft(MPI)p 184 1450 16 2 v 18 w(CONTEXT)p 466 1450 V 19 w(ALLOCA)l(TE\(comm,)e(map,)g(arra) o(y)p 1210 1450 V 18 w(of)p 1270 1450 V 19 w(con)o(textids,)g(len\))166 1543 y Fs(Allo)q(cates)i(an)f(arra)o(y)f(of)h(con)o(text)p 759 1543 14 2 v 16 w(id's.)29 b(This)19 b(is)g(a)f(collectiv)o(e)i(op)q (eration)f(that)e(is)i(executed)g(b)o(y)75 1600 y(all)e(pro)q(cesses)g (in)g(the)g(group)f(de\014ned)i(b)o(y)e Fl(\(comm,)23 b(map\))p Fs(.)g(The)17 b(con)o(text)p 1367 1600 V 16 w(ids)g(that)e(are)h(returned)h(are)75 1656 y(unique)g(within)g(the)f (group)f(asso)q(ciated)h(with)g Fl(\(comm,)23 b(map\))p Fs(.)d(The)c(arra)o(y)e(returned)i(is)g(the)g(same)f(on)75 1712 y(all)j(pro)q(cesses)f(that)f(call)i(the)f(function)h(\(same)e (order,)g(same)h(n)o(um)o(b)q(er)g(of)g(elemen)o(ts\).)25 b(The)17 b(call)h(ma)o(y)75 1769 y(blo)q(c)o(k)e(un)o(til)g(all)g(pro)q (cesses)g(within)g(the)g(executing)g(group)f(ha)o(v)o(e)g(in)o(v)o(ok)o (ed)g(the)g(call.)75 1882 y Ft(IN)i(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1981 y Ft(IN)i(map)23 b Fs(map)15 b(ob)s(ject)g(handle)75 2080 y Ft(OUT)j(arra)o(y)p 332 2080 16 2 v 18 w(of)p 392 2080 V 19 w(con)o(text)p 580 2080 V 19 w(ids)23 b Fs(\(arra)o(y)14 b(of)g(in)o(tegers\))75 2180 y Ft(IN)j(len)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(con)o(text)p 611 2180 14 2 v 16 w(id's)g(to)g(allo)q(cate)h(\(in)o(teger\))166 2328 y Ft(MPI)p 275 2328 16 2 v 18 w(CONTEXT)p 557 2328 V 19 w(RESER)-6 b(VE\(con)o(text)p 1008 2328 V 21 w(id\))166 2421 y Fs(Reserv)o(e)13 b(a)f(con)o(text)p 516 2421 14 2 v 16 w(id)h(v)m(alue.)21 b(This)13 b(is)g(a)f(lo)q(cal)i(call)g(that) d(is)j(in)o(v)o(ok)o(ed)e(b)o(y)h(a)f(single)i(pro)q(cess.)19 b(A)13 b(re-)75 2478 y(serv)o(ed)g(con)o(text)p 362 2478 V 16 w(id)g(v)m(alue)h(will)g(not)e(b)q(e)i(allo)q(cated)f(b)o(y)g(a)f (subsequen)o(t)h(call)h(to)e Fl(MPI)p 1486 2478 15 2 v 17 w(CONTEXT)p 1671 2478 V 16 w(ALLOCATE)75 2534 y Fs(b)o(y)j(the)h(same)f(pro)q(cess.)20 b(It)15 b(is)h(erroneous)f(to)g (reserv)o(e)g(a)g(con)o(text)p 1203 2534 14 2 v 16 w(id)h(that)f(has)g (b)q(een)h(already)g(b)q(een)g(al-)75 2591 y(lo)q(cated)g(b)o(y)f Fl(MPI)p 371 2591 15 2 v 17 w(CONTEXT)p 556 2591 V 16 w(ALLOCATE)p Fs(.)75 2704 y Ft(IN)i(con)o(text)p 325 2704 16 2 v 19 w(id)23 b Fs(\(in)o(teger\))p eop %%Page: 99 104 99 103 bop 75 -100 a Fq(9.4.)34 b(COMMUNICA)l(TORS)1224 b Fs(99)166 45 y Fk(Discussion:)166 95 y Fj(Ma)o(y)13 b(w)o(an)o(t)h(to)g(reserv)o(e)h(an)f(arra)o(y)g(of)f(v)n(alues,)g (rather)i(than)f(one)g(v)n(alue.)166 152 y(Implemen)o(tations)6 b(ma)o(y)i(ha)o(v)o(e)h(there)h(o)o(wn)f(reserv)o(ed)j(v)n(alues.)k (The)10 b(e\013ect)g(is)g(as)f(if)f(calls)h(to)g Ff(MPI)p 1638 152 14 2 v 15 w(CONTEXT)p 1807 152 V 15 w(RESERVE)75 209 y Fj(where)15 b(executed)h(in)d(a)h(pream)o(ble.)166 384 y Ft(MPI)p 275 384 16 2 v 18 w(CONTEXT)p 557 384 V 19 w(FREE\(con)o(text)p 905 384 V 20 w(id\))166 476 y Fs(F)l(ree)i(a)g(reserv)o(ed)h(con)o(text)p 634 476 14 2 v 16 w(id)g(v)m(alue.)24 b(The)17 b(v)m(alue)g(b)q(ecomes)g(a)o(v) m(ailable)h(for)e(a)g(subsequen)o(t)h(reser-)75 532 y(v)m(ation)g(b)o (y)g Fl(MPI)p 354 532 15 2 v 16 w(CONTEXT)p 538 532 V 17 w(RESERVE)e Fs(or)h(subsequen)o(t)h(allo)q(cation)h(b)o(y)f Fl(MPI)p 1376 532 V 16 w(CONTEXT)p 1560 532 V 17 w(ALLOCATE)p Fs(.)e(It)h(is)75 589 y(erroneous)f(to)g(free)g(a)g(v)m(alue)h(that)f (is)g(asso)q(ciated)h(with)f(an)g(activ)o(e)h(comm)o(unicator.)75 697 y Ft(IN)h(con)o(text)p 325 697 16 2 v 19 w(id)23 b Fs(\(in)o(teger\))166 881 y Fk(Discussion:)166 931 y Fj(Here,)15 b(to)q(o,)e(w)o(e)h(migh)o(t)e(prefer)j(to)e(free)i(an)f (arra)o(y)f(of)h(v)n(alues,)f(rather)h(than)g(one.)166 981 y(Comm)o(uni)o(cators,)c(lik)o(e)h(all)g(opaque)h(ob)r(jects,)h (are)g(freed)g(b)o(y)e(a)h(call)g(to)f Ff(MPI)p 1346 981 14 2 v 16 w(FREE)p Fj(.)f(W)m(e)i(ma)o(y)e(prefer)j(to)f(ha)o(v)o (e)75 1031 y(that)i(same)g(call)f(free)i(also)f(the)h(asso)q(ciated)g (con)o(text)p 915 1031 13 2 v 16 w(id.)j(With)c(the)h(curren)o(t)h (design,)e(if)f(w)o(e)i(w)o(an)o(t)f(to)g(free)h(b)q(oth)75 1081 y(comm)o(unicator)f(and)j(con)o(text)p 568 1081 V 15 w(id)g(w)o(e)g(need)h(to)e(store)i(the)g(con)o(text)p 1165 1081 V 15 w(id)f(v)n(alue,)f(free)i(the)f(comm)o(unicator,)d(next) 75 1131 y(free)h(the)f(con)o(text)p 361 1131 V 16 w(id.)75 1359 y Fm(9.4)70 b(Comm)n(unicators)75 1461 y Fs(A)21 b Ft(comm)o(unicator)g Fs(is)h(an)f(opaque)g(ob)s(ject)g(that)f(iden)o (ti\014es)j(a)e(group)g(of)f(pro)q(cesses)i(and)f(a)g(com-)75 1518 y(m)o(unication)e(con)o(text)e(for)g(that)h(group.)27 b(Lik)o(e)19 b(other)f(opaque)g(ob)s(jects,)f(comm)o(unicators)h (cannot)f(b)q(e)75 1574 y(transfered)e(b)q(et)o(w)o(een)g(pro)q (cesses;)h(con)o(text)p 824 1574 14 2 v 15 w(id's)g(are)f(used)h(to)e (transfer)h(information)g(on)g(con)o(text.)166 1631 y(As)g(a)g (short-hand,)g(w)o(e)g(shall)i(iden)o(ti\014y)g(the)e(group)g(of)g(pro) q(cesses)h(asso)q(ciated)g(with)f(a)g(comm)o(uni-)75 1687 y(cator)h Fl(comm)g Fs(with)h(the)g(comm)o(unicator)f(itself.)26 b(Th)o(us,)16 b(\\the)h(pro)q(cess)g(with)g(rank)f Fr(i)h Fs(in)g Fl(comm)p Fs(")f(should)75 1744 y(b)q(e)e(understo)q(o)q(d)g (as)g(meaning)g(\\)f(the)h(pro)q(cess)g(with)g(rank)f Fr(i)g Fs(in)i(the)f(group)f(asso)q(ciated)h(with)g Fl(comm)p Fs(".)k(In)75 1800 y(the)e(same)g(manner,)g(\\the)g(comm)o(unication)h (con)o(text)f Fl(comm)p Fs(")f(should)i(b)q(e)g(understo)q(o)q(d)f(to)g (mean)g(\\the)75 1857 y(comm)o(unication)g(con)o(text)f(asso)q(ciated)g (with)g Fl(comm)p Fs(".)166 1913 y(An)h(initial)h(comm)o(unicator)f Fl(MPI)p 745 1913 15 2 v 16 w(COMM)p 857 1913 V 17 w(INIT)f Fs(is)h(de\014ned)h(when)f(the)g(program)e(starts.)20 b(Its)c(asso-)75 1970 y(ciated)j(group)e(con)o(tains)h(all)h(pro)q (cesses)g(that)e(start)g(the)h(computation.)28 b(Applications)20 b(that)d(do)h(not)75 2026 y(need)e(m)o(ultiple)h(pro)q(cess)f(groups)e (or)h(m)o(ultiple)i(con)o(texts,)d(will)j(only)f(use)f(this)h(comm)o (unicator.)166 2083 y(Let)i Fl(comm)f Fs(b)q(e)h(a)g(comm)o(unicator)f (that)g(is)i(asso)q(ciated)f(with)g(a)f(group)h(of)f(size)i Fr(n)p Fs(,)f(and)g(let)g Fl(map)e Fs(:)75 2140 y(0)p Fr(::m)8 b Fo(\000)h Fs(1)k Fo(!)g Fs(0)p Fr(::n)8 b Fo(\000)h Fs(1)15 b(b)q(e)g(a)f(map.)20 b(Then)15 b(the)g(pair)g Fl(\(comm,)23 b(map\))14 b Fs(de\014nes)i(a)e(subgroup,)h(namely)g(the) 75 2196 y(subgroup)g(of)g(pro)q(cesses)h(with)f(ranks)g Fl(map)p Fs(\(0\))p Fr(;)8 b(:::;)f Fl(map)m Fs(\()p Fr(m)i Fo(\000)i Fs(1\))j(in)i Fl(comm)p Fs(.)75 2320 y Fi(9.4.1)55 b(Op)r(erations)18 b(on)h(comm)n(unicators)75 2442 y Ft(MPI)p 184 2442 16 2 v 18 w(COMM)p 379 2442 V 18 w(MAP\(comm,)d(sub)q(comm,)g(map\))166 2534 y Fs(Returns)g(a)e (map)h(suc)o(h)h(that)e Fl(\(comm,)23 b(map\))15 b Fs(is)h(the)f(group) g(asso)q(ciated)g(with)h Fl(subcomm)p Fs(;)e(i.e.,)g(if)i Fl(i)75 2591 y Fs(is)h(the)f(rank)g(of)g(a)h(pro)q(cess)f(in)h(the)g (group)f(asso)q(ciated)h(with)f Fl(subcomm)p Fs(,)g(then)g Fl(map\(i\))g Fs(is)h(the)f(rank)g(of)75 2647 y(that)f(same)g(pro)q (cess)g(in)i(the)e(group)g(asso)q(ciated)h(with)g Fl(comm)p Fs(.)j(The)d(call)h(is)f(erroneous)f(if)h Fl(subcomm)e Fs(has)75 2704 y(a)h(pro)q(cess)g(that)g(is)h(not)e(mem)o(b)q(er)i(of)e Fl(comm)p Fs(.)p eop %%Page: 100 105 100 104 bop 75 -100 a Fs(100)671 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)75 45 y Ft(IN)i(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 141 y Ft(IN)i(sub)q(comm)23 b Fs(comm)o(unicator)14 b(ob)s(ject)h(handle)75 237 y Ft(OUT)j(map)23 b Fs(map)15 b(ob)s(ject)g(handle)166 346 y(The)i Fl(MPI)p 336 346 15 2 v 17 w(COMM)p 449 346 V 17 w(MAP)g Fs(function)h(retriev)o(es)f(the)g(group)g(of)g(pro)q (cesses)h(that)e(is)i(asso)q(ciated)g(with)f(a)75 403 y(comm)o(unicator.)h(Ho)o(w)o(ev)o(er,)12 b(since)g(absolute)h(pro)q (cess)f(names)f(are)h(not)f(visible)j(in)f(MPI,)e(the)h(group)g(can)75 459 y(only)j(b)q(e)g(de\014ned)h(relativ)o(e)f(to)f(another)g (encompassing)h(group.)20 b(The)14 b(\\absolute")h(pro)q(cess)g(n)o(um) o(b)q(er)f(is)75 516 y(obtained)i(b)o(y)f(using)h Fl(MPI)p 519 516 V 17 w(COMM)p 632 516 V 16 w(INIT)f Fs(as)g(a)g(reference)h (comm)o(unicator.)166 608 y Ft(MPI)p 275 608 16 2 v 18 w(COMM)p 470 608 V 18 w(SIZE\(comm,)h(size\))166 701 y Fs(Returns)f(the)f(size)h(of)f(the)g(group)g(asso)q(ciated)g(with)h Fl(comm)p Fs(.)75 810 y Ft(IN)h(comm)22 b Fs(handle)17 b(to)d(comm)o(unicator)75 906 y Ft(OUT)k(size)23 b Fs(group)15 b(size)h(\(in)o(teger\))166 1051 y Ft(MPI)p 275 1051 V 18 w(COMM)p 470 1051 V 18 w(CONTEXTID\(comm,)h(con)o(text)p 1170 1051 V 19 w(id\))166 1143 y Fs(Returns)f(the)f(con)o(text)p 567 1143 14 2 v 16 w(id)h(asso)q(ciated)f(with)h(the)f(comm)o(unicator) g Fl(comm)p Fs(.)75 1252 y Ft(IN)i(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 1348 y Ft(OUT)j(con)o(text)p 380 1348 16 2 v 19 w(id)24 b Fs(con)o(text)p 613 1348 14 2 v 15 w(id)75 1473 y Fi(9.4.2)55 b(Comm)n(unicator)17 b(constructors)75 1596 y Ft(MPI)p 184 1596 16 2 v 18 w(COMM)p 379 1596 V 18 w(MAKE\(comm,)f(map,)h(con)o(text)p 1049 1596 V 19 w(id,)h(new)o(comm\))166 1688 y Fs(Creates)h(a)h(new)g (comm)o(unicator)g(ob)s(ject,)g(whic)o(h)h(is)f(asso)q(ciated)h(with)f (the)g(group)g(de\014ned)h(b)o(y)75 1745 y Fl(\(comm,)i(map\))p Fs(,)14 b(and)h(the)g(con)o(text)p 681 1745 14 2 v 15 w(id)h Fl(context)p 917 1745 15 2 v 16 w(id)p Fs(.)j(This)d(is)f(a)f (lo)q(cal)i(call)g(executed)g(b)o(y)e(one)h(pro)q(cess.)75 1801 y(Ho)o(w)o(ev)o(er,)d(the)h(new)h(comm)o(unicator)e(ob)s(ject)h (should)h(not)e(b)q(e)i(used)g(for)e(comm)o(unication)i(b)q(et)o(w)o (een)f(t)o(w)o(o)75 1858 y(pro)q(cesses)j(unless)g(they)f(b)q(oth)h(ha) o(v)o(e)e(called)j(the)e(function.)166 1915 y(It)d(is)h(erroneous)f(to) g(create)g(on)h(the)f(same)g(pro)q(cess)h(t)o(w)o(o)e(distinct)i(comm)o (unicators)f(with)h(the)f(same)75 1971 y(con)o(text)p 225 1971 14 2 v 16 w(id.)75 2080 y Ft(IN)17 b(comm)22 b Fs(comm)o(unicator)15 b(ob)s(ject)g(handle)75 2176 y Ft(IN)i(map)23 b Fs(map)15 b(ob)s(ject)g(handle)75 2272 y Ft(IN)i(con)o(text)p 325 2272 16 2 v 19 w(id)23 b Fs(con)o(text)p 557 2272 14 2 v 16 w(id)75 2369 y Ft(OUT)18 b(new)o(comm)k Fs(comm)o(unicator)15 b(ob)s(ject)f(handle)166 2478 y(The)23 b(con)o(tect)p 413 2478 V 16 w(id)g(used)g(in)h(this)f (call)g(ma)o(y)f(b)q(e)i(an)e(in)o(teger)h(returned)g(b)o(y)f(a)h (previous)g(call)h(to)75 2534 y Fl(MPI)p 150 2534 15 2 v 17 w(ALLOC)p 287 2534 V 16 w(CONTEXT)p Fs(,)17 b(executed)h(within) h(the)f(same)f(group.)27 b(In)18 b(suc)o(h)g(case,)g(it)g(is)g(guaran)o (teed)g(that)75 2591 y(the)f(con)o(text)p 305 2591 14 2 v 16 w(id)h(is)g(unique,)h(and)e(has)g(not)g(b)q(een)h(used)g (already)g(to)e(create)h(a)g(comm)o(unicator.)25 b(Ho)o(w-)75 2647 y(ev)o(er,)17 b(the)g(user)g(is)h(free)f(to)f(manage)g(on)h(it)g (o)o(wn)g(the)g(con)o(text)p 1165 2647 V 16 w(id's,)g(and)g(use)g (other)g(mec)o(hanisms)h(for)75 2704 y(their)e(allo)q(cation.)22 b(F)l(or)14 b(example,)i(one)g(could)g(ha)o(v)o(e)f(a)g(single)i Ft(con)o(text)p 1330 2704 16 2 v 19 w(id)h(serv)o(er)13 b Fs(generate)i(unique)p eop %%Page: 101 106 101 105 bop 75 -100 a Fq(9.5.)34 b(W)o(ORKING)16 b(WITHOUT)f(CONTEXT)p 915 -100 14 2 v 17 w(ID'S)789 b Fs(101)75 45 y(con)o(text)p 225 45 V 16 w(id's)14 b(for)g(the)g(en)o(tire)h(system;)f(one)g(can)g (preallo)q(cate)i(statically)f(some)f(con)o(text)p 1609 45 V 15 w(id)h(v)m(alues,)h(for)75 102 y(the)f(use)h(of)f(libraries;)h (etc.)166 234 y Fk(Discussion:)166 284 y Fj(Do)d(w)o(e)h(w)o(an)o(t)f (to)h(prohibit)f(implem)o(en)o(tations)e(that)j(sync)o(hronize)h(the)f (creation)g(of)f(comm)o(unicators)e(\(i.e.,)75 334 y(where)k(a)f(call)f (to)g Ff(MPI)p 425 334 V 16 w(MAKE)p 529 334 V 14 w(COMM)g Fj(blo)q(c)o(ks)h(un)o(til)f(all)g(mem)o(b)q(ers)g(of)g(the)h(group)g (ha)o(v)o(e)g(made)f(the)h(call\)?)166 383 y(Is)g(it)g(preferable)g(to) g(ha)o(v)o(e)g(a)f Ff(array)p 739 383 V 15 w(of)p 798 383 V 15 w(comm)g Fj(as)h(parameter,)f(rather)i(than)f(one)g(comm)o (unicator?)166 440 y(W)m(e)i(don't)g(ha)o(v)o(e)g(no)o(w)g(the)h (abilit)o(y)e(to)h(ha)o(v)o(e)g(t)o(w)o(o)g(di\013eren)o(t)i(groups)e (with)g(the)h(same)f(con)o(text;)i(e.g.,)d(t)o(w)o(o)75 496 y(di\013eren)o(t)d(n)o(um)o(b)q(erings)e(of)g(the)i(pro)q(cesses,)i (used)d(with)g(the)h(same)e(comm)o(unicati)o(on)e(con)o(text.)18 b(The)11 b(alternativ)o(e)g(is)75 553 y(to)h(remo)o(v)o(e)f(the)i (restriction)g(that)f(t)o(w)o(o)g(distinct)g(comm)o(unicators)e(cannot) i(use)h(the)g(same)e(con)o(text)p 1641 553 13 2 v 16 w(id.)17 b(It)12 b(is)g(no)o(w)75 609 y(the)18 b(user)h(resp)q (onsibilit)o(y)e(to)g(disam)o(biguate)e(these)k(t)o(w)o(o)e(comm)o (unicators;)g(the)h(system)f(do)q(es)h(not)f(guaran)o(tee)75 666 y(that)e(a)g(message)g(sen)o(t)h(with)e(one)i(comm)o(unicator)c (can)j(b)q(e)h(receiv)o(ed)g(only)f(with)f(this)h(same)g(comm)o (unicator,)d(if)75 722 y(they)h(use)g(the)g(same)f(con)o(text)p 545 722 V 16 w(id.)17 b(W)m(e)12 b(sa)o(v)o(e)h(on)f(con)o(text)p 972 722 V 16 w(id's)f(but)i(lo)q(ose)g(safet)o(y)f(\(the)h(sender)i (and)d(receiv)o(er)i(ma)o(y)75 779 y(b)q(e)i(using)g(di\013eren)o(t)g (pro)q(cess)i(n)o(um)o(b)q(erings,)c(if)h(t)o(w)o(o)g(di\013eren)o(t)i (comm)o(unicators)c(are)j(used)h({)e(the)i(receiv)o(er)g(ma)o(y)75 835 y(not)d(iden)o(tify)f(correctly)i(the)f(sender\).)75 1061 y Fm(9.5)70 b(W)-6 b(orking)23 b(without)f(con)n(text)p 993 1061 21 2 v 25 w(id's)75 1162 y Fs(It)12 b(is)h(p)q(ossible)g(to)f (create)g(comm)o(unicators)f(directly)l(,)j(without)e(using)h(con)o (text)p 1430 1162 14 2 v 16 w(id's.)19 b(A)12 b(single)h(function)75 1219 y(com)o(bines)f(the)f(allo)q(cation)h(of)e(a)h(new)g(con)o(text)p 865 1219 V 16 w(id)h(and)f(the)g(generation)g(of)f(a)h(new)g(comm)o (unicator.)18 b(This)75 1275 y(allo)o(ws)f(a)g(\\naiv)o(e")f(user)h (that)g(do)q(es)g(not)f(need)i(to)e(customize)i(con)o(text)p 1329 1275 V 15 w(id)g(allo)q(cation)g(to)e(ignore)i(this)75 1332 y(MPI)j(feature.)38 b(Direct)21 b(comm)o(unicator)g(creation)h(is) g(safer,)g(since)g(uniqueness)h(of)e(con)o(text)p 1742 1332 V 16 w(id's)g(is)75 1388 y(guaran)o(teed,)14 b(b)o(y)h (construction.)166 1480 y Ft(MPI)p 275 1480 16 2 v 18 w(COMM)p 470 1480 V 18 w(SAFEMAKE\(comm,)h(map,)h(new)o(comm\))166 1572 y Fs(Creates)c(a)h(new)g(comm)o(unicator)g(\(with)g(an)g(attac)o (hed)f(con)o(text)p 1270 1572 14 2 v 16 w(id\))i(that)e(is)h(asso)q (ciated)h(with)f(the)75 1628 y(group)19 b(de\014ned)i(b)o(y)e(the)g (pair)h Fl(\(comm,)j(map\))p Fs(.)31 b(This)20 b(is)g(a)f(collectiv)o (e)i(call)f(that)f(has)g(to)g(b)q(e)h(in)o(v)o(ok)o(ed)75 1685 y(b)o(y)d(all)h(mem)o(b)q(ers)f(in)g(this)h(group.)24 b(All)18 b(of)f(them)g(are)f(pro)o(viding)i(the)f(same)g(input)h (parameters)e Fl(comm)75 1741 y Fs(and)j Fl(map)p Fs(.)32 b(The)19 b(call)h(ma)o(y)f(blo)q(c)o(k)h(un)o(til)g(all)g(pro)q(cesses) g(in)g(the)f(group)g(ha)o(v)o(e)g(in)o(v)o(ok)o(ed)g(it.)32 b(The)20 b(new)75 1798 y(comm)o(unicator)c(ma)o(y)f(b)q(e)i(safely)g (used)f(for)g(comm)o(unication)h(with)f(an)o(y)g(mem)o(b)q(er)g(of)g (the)g(new)h(group,)75 1854 y(once)f(the)f(call)h(returns.)75 1958 y Ft(IN)h(comm)22 b Fs(comm)o(unicator)15 b(handle)75 2052 y Ft(IN)i(map)23 b Fs(map)15 b(handle)75 2145 y Ft(OUT)j(new)o(comm)k Fs(comm)o(unicator)15 b(handle)166 2249 y Fl(MPI)p 241 2249 15 2 v 17 w(COMM)p 354 2249 V 16 w(SAFEMAKE)f Fs(is)75 2354 y Fl(MPI_CONTEXT_ALLOC\(comm,)21 b(map,)i(context_id,)f(1\);)75 2410 y(MPI_COMM_MAKE\(comm,)f(map,)j (context_id,)e(newcomm\);)75 2467 y(``MPI_SYNCH\(comm,map\)'')f(/*)i (by)h(which)f(we)h(mean)f(a)h(synchronization)576 2523 y(operation)f(in)g(the)h(subgroup)f(defined)f(by)i(\(comm,map\))f(*/) 166 2704 y Fk(Discussion:)p eop %%Page: 102 107 102 106 bop 75 -100 a Fs(102)671 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)166 45 y Fj(Do)d(w)o(e)g (prefer)i(to)e(create)i(an)e(arra)o(y)g(of)f(comm)o(unicators,)f (rather)j(than)g(one?)k(\(T)m(o)12 b(b)q(e)h(consisten)o(t)g(with)f (the)75 95 y Ff(MPI)p 144 95 14 2 v 15 w(CONTEXT)p 313 95 V 14 w(ALLOC)h Fj(function\).)166 145 y Ff(MPI)p 235 145 V 15 w(SYNCH\(comm,map\))f Fj(is)i(not)h(curren)o(tly)h(de\014ned)g (in)e(the)i(collectiv)o(e)f(comm)o(unicati)o(on)d(library;)i(it)h(can) 75 195 y(b)q(e)g(co)q(ded)f(using)g(p)q(oin)o(t-to-p)q(oin)o(t)e (calls,)h(but)i(seems)f(imp)q(ortan)o(t)e(enough)i(to)f(b)q(e)i (included)f(there.)166 251 y(General)i(though)o(t)h({)f(if)g(w)o(e)g (supp)q(ort)i(collectiv)o(e)e(comm)o(unication)d(calls)k(with)f(a)g (map)f(as)i(an)f(additional)75 308 y(parameter,)g(then)h(w)o(e)g(ha)o (v)o(e)f(disasso)q(ciated)h(the)g(\\con)o(text")g(of)f(a)g(collectiv)o (e)g(comm)o(unication,)e(from)g(its)j(sup-)75 364 y(p)q(orting)c (\\group".)k(W)m(e)12 b(need)i(to)f(do)f(it)h(when)g(w)o(e)g(build)f (new)i(comm)o(unicators,)c(in)i(order)i(to)f(b)q(o)q(otstrap)g({)g(if)f (w)o(e)75 421 y(w)o(an)o(t)h(the)h(call)f(that)h(generates)h(a)f(new)g (comm)o(unicator)d(to)i(in)o(v)o(olv)o(e)f(only)h(the)h(pro)q(cesses)j (in)c(the)h(new)g(group.)75 647 y Fm(9.6)70 b(W)-6 b(orking)23 b(without)f(maps)h(and)h(con)n(text)p 1305 647 21 2 v 24 w(id's)75 748 y Fs(It)16 b(is)g(p)q(ossible)h(to)e(a)o(v)o(oid)g(b)q (oth)h(the)g(use)f(of)h(map)f(ob)s(jects)g(and)h(of)f(con)o(text)p 1384 748 14 2 v 16 w(id's)h(altogether.)k(A)c(single)75 805 y(function)d(com)o(bines)g(the)f(generation)g(of)f(a)h(map,)g(the)g (allo)q(cation)i(of)d(a)h(con)o(text)p 1445 805 V 16 w(id)g(and)h(the)f(generation)75 861 y(of)17 b(a)h(new)g(comm)o (unicator.)28 b(This)18 b(allo)o(ws)g(\\naiv)o(e")g(users)g(to)f(use)h (MPI)g(without)g(b)q(eing)h(exp)q(osed)g(to)75 918 y(maps)c(or)g(con)o (text)p 400 918 V 15 w(id's.)166 1010 y Ft(MPI)p 275 1010 16 2 v 18 w(COMM)p 470 1010 V 18 w(BUILD\(comm,)i(arra)o(y)p 966 1010 V 18 w(of)p 1026 1010 V 19 w(ranks,)f(size,)i(new)o(comm\))166 1102 y Fs(Build)k(a)f(map)f(from)g(an)g(explicit)j(list)e(represen)o (tation.)36 b(This)21 b(is)g(a)f(collectiv)o(e)i(function)g(that)75 1158 y(is)f(called)i(b)o(y)d(all)i(mem)o(b)q(ers)f(of)g(the)f(new)h (group.)37 b(All)22 b(pro)o(vide)f(the)g(same)g(parameters)f(for)g Fl(comm)p Fs(,)75 1215 y Fl(array)p 198 1215 15 2 v 16 w(of)p 262 1215 V 17 w(ranks)15 b Fs(and)h Fl(size)p Fs(.)j(The)d(call)g(ma)o(y)f(blo)q(c)o(k)h(un)o(til)g(it)g(w)o(as)e(in) o(v)o(ok)o(ed)i(on)f(all)h(the)g(group)f(mem-)75 1271 y(b)q(ers.)40 b(When)22 b(it)f(returns,)i(the)f(new)g(comm)o(unicator)f (ma)o(y)g(b)q(e)h(used)g(to)f(comm)o(unicate)h(with)g(an)o(y)75 1328 y(mem)o(b)q(er)15 b(of)g(the)g(new)h(group.)75 1422 y Ft(IN)h(comm)22 b Fs(handle)17 b(to)d(comm)o(unicator)75 1516 y Ft(IN)j(arra)o(y)p 277 1516 16 2 v 18 w(of)p 337 1516 V 19 w(ranks)22 b Fs(list)16 b(of)f(v)m(alues)h(in)g(the)f(range)g (of)g Fl(map)75 1610 y Ft(IN)i(size)23 b Fs(n)o(um)o(b)q(er)16 b(of)f(en)o(tries)g(in)h(arra)o(y)e({)h(map)g(size)h(\(in)o(teger\))75 1704 y Ft(OUT)i(new)o(comm)k Fs(handle)16 b(to)f(new)g(comm)o(unicator) 166 1798 y Fl(MPI)p 241 1798 15 2 v 17 w(COMM)p 354 1798 V 16 w(BUILD)g Fs(is)75 1905 y Fl(MPI_MAP_BUILD\(comm,)21 b(array_of_ranks,)h(size,)h(newmap\);)75 1961 y (MPI_COMM_SAFEMAKE\(comm,)e(newmap,)i(newcomm\);)166 2103 y Ft(MPI)p 275 2103 16 2 v 18 w(COMM)p 470 2103 V 18 w(COPY\(comm,)16 b(new)o(comm\))166 2195 y Fs(Creates)h(a)g(new)g (comm)o(unicator)g(with)h(the)g(same)f(group)g(as)g(the)g(old)h(comm)o (unicator.)26 b(This)18 b(is)75 2252 y(a)i(collectiv)o(e)i(function)g (that)e(is)h(called)h(b)o(y)e(all)i(mem)o(b)q(ers)e(of)g(new)h(group.) 36 b(All)22 b(pro)o(vide)f(the)f(same)75 2308 y(parameters)d(for)g Fl(comm)p Fs(.)27 b(The)18 b(call)h(ma)o(y)e(blo)q(c)o(k)h(un)o(til)h (it)f(w)o(as)f(in)o(v)o(ok)o(ed)h(on)f(all)i(the)f(group)f(mem)o(b)q (ers.)75 2365 y(When)e(it)f(returns,)g(the)g(new)h(comm)o(unicator)f (ma)o(y)f(b)q(e)i(used)g(to)f(comm)o(unicate)g(with)h(an)o(y)f(mem)o(b) q(er)g(of)75 2421 y(the)h(group.)75 2515 y Ft(IN)i(comm)22 b Fs(handle)17 b(to)d(comm)o(unicator)75 2609 y Ft(OUT)k(new)o(comm)k Fs(handle)16 b(to)f(new)g(comm)o(unicator)166 2704 y Fl(MPI)p 241 2704 15 2 v 17 w(COMM)p 354 2704 V 16 w(COPY)g Fs(is)p eop %%Page: 103 108 103 107 bop 75 -100 a Fq(9.7.)34 b(EXAMPLES)1370 b Fs(103)75 45 y Fl(MPI_COMM_SIZE\(comm,)21 b(size\);)75 102 y (MPI_COMM_SAFEMAKE\(comm,)g(MPI_IDENT\(size\),)g(newcomm\);)166 248 y Ft(MPI)p 275 248 16 2 v 18 w(COMM)p 470 248 V 18 w(SPLIT\(comm,)c(k)o(ey)l(,)e(index,)j(new)o(comm\))166 340 y Fs(Split)c(the)e(group)h(asso)q(ciated)f(with)h Fl(comm)p Fs(;)g(creates)f(a)g(new)h(group)f(for)g(eac)o(h)h(distinct)h (v)m(alue)f(of)f Fl(key)75 397 y Fs(that)g(con)o(tains)h(the)g(pro)q (cesses)g(that)f(supplied)k(that)c(k)o(ey)g(v)m(alue;)j(the)e(pro)q (cesses)g(are)g(rank)o(ed)f(according)75 453 y(to)g(the)h Fl(index)f Fs(v)m(alues)h(they)g(supplied.)22 b(A)12 b(new)h(comm)o(unicator)f(is)i(created)e(for)g(eac)o(h)h(subgroup.)19 b(Eac)o(h)75 510 y(pro)q(cess)c(is)f(returned)h(a)f(handle)i(to)d(the)i (comm)o(unicator)f(for)f(the)i(new)f(subgroup)h(it)f(b)q(elongs)h(to.)k (This)75 566 y(is)f(a)e(collectiv)o(e)j(call)f(executed)g(b)o(y)f(all)h (pro)q(cesses)f(in)h(the)f(group)g(asso)q(ciated)g(with)h Fl(comm)p Fs(.)25 b(They)17 b(call)75 623 y(ma)o(y)h(blo)q(c)o(k)i(un)o (til)g(all)h(pro)q(cesses)e(ha)o(v)o(e)g(in)o(v)o(ok)o(ed)g(the)g (function.)33 b(When)19 b(the)g(call)i(returns)e(the)g(new)75 679 y(comm)o(unicator)c(ma)o(y)f(b)q(e)i(safely)g(used)g(for)e(comm)o (unication)i(in)g(the)f(new)h(group.)75 790 y Ft(IN)h(comm)22 b Fs(handle)17 b(to)d(comm)o(unicator)75 887 y Ft(IN)j(k)o(ey)22 b Fs(\(in)o(teger\))75 984 y Ft(IN)17 b(index)23 b Fs(\(in)o(teger\))75 1082 y Ft(OUT)18 b(new)o(comm)k Fs(handle)16 b(to)f(new)g(comm)o (unicator)166 1192 y Fl(MPI)p 241 1192 15 2 v 17 w(COMM)p 354 1192 V 16 w(SPLIT\(comm,)23 b(key,)g(index,)g(newcomm\))14 b Fs(is)75 1304 y Fl(MPI_COMM_SIZE\(comm,)21 b(size\);)75 1360 y(MPI_MAP_SPLIT\(comm,)g(MPI_IDENT\(size\),)h(key,)h(index,)g (newmap\);)75 1417 y(MPI_COMM_SAFEMAKE\(comm,)e(newmap,)i(newcomm\);) 166 1603 y Fk(Discussion:)166 1654 y Fj(Ma)o(y)13 b(w)o(an)o(t)g(a)h (sp)q(ecial)f(DONTCARE)h(k)o(ey)g(v)n(alue)f(that)g(indicates)h(that)g (the)g(caller)g(need)g(not)g(ha)o(v)o(e)f(a)h(new)75 1704 y(comm)o(unicator)75 1935 y Fm(9.7)70 b(Examples)75 2038 y Fs(W)l(e)19 b(sa)o(y)f(that)f(a)i(parallel)h(pro)q(cedure)f(is)g Fh(active)g Fs(at)f(a)g(pro)q(cess)h(if)g(the)g(pro)q(cess)g(b)q (elongs)g(to)f(a)g(group)75 2094 y(that)e(ma)o(y)h(collectiv)o(ely)i (execute)f(the)f(pro)q(cedure,)h(and)f(some)f(mem)o(b)q(er)h(of)g(that) f(group)h(is)g(curren)o(tly)75 2151 y(executing)c(the)f(pro)q(cedure)g (co)q(de.)20 b(If)12 b(a)f(parallel)j(pro)q(cedure)e(is)h(activ)o(e)e (at)h(a)f(pro)q(cess,)h(then)g(this)h(pro)q(cess)75 2207 y(ma)o(y)e(b)q(e)i(receiving)g(messages)e(p)q(ertaining)i(to)f(this)g (pro)q(cedure,)h(ev)o(en)f(if)g(it)g(do)q(es)h(not)e(curren)o(tly)h (execute)75 2264 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 2390 y Fi(9.7.1)55 b(Nonreen)n(tran)n(t)19 b(parallel)e(pro)r(cedures) 75 2478 y Fs(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o (y)g(p)q(oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(ok)m(ation)g (of)f(a)g(parallel)75 2534 y(pro)q(cedure)e(can)g(b)q(e)g(activ)o(e)g (at)f(an)o(y)g(pro)q(cess.)30 b(I.e.,)19 b(concurren)o(t)g(in)o(v)o(ok) m(ations)f(of)h(the)f(same)g(parallel)75 2591 y(pro)q(cedure)e(ma)o(y)e (o)q(ccur)h(only)g(within)h(disjoin)o(t)f(groups)f(of)h(pro)q(cesses.) 20 b(F)l(or)14 b(example,)h(all)h(in)o(v)o(ok)m(ations)75 2647 y(of)f(parallel)i(pro)q(cedures)f(in)o(v)o(olv)o(e)g(all)h(pro)q (cesses,)e(pro)q(cesses)h(are)f(single-threaded,)i(and)e(there)h(are)f (no)75 2704 y(recursiv)o(e)h(in)o(v)o(ok)m(ations.)p eop %%Page: 104 109 104 108 bop 75 -100 a Fs(104)671 b Fq(CHAPTER)16 b(9.)34 b(MAPS,)15 b(GR)o(OUPS)h(AND)f(CONTEXTS)166 45 y Fs(In)i(suc)o(h)g(a)g (case,)g(a)f(con)o(text)p 668 45 14 2 v 16 w(id)i(can)f(b)q(e)g (statically)h(allo)q(cated)f(to)g(eac)o(h)g(pro)q(cedure.)25 b(The)17 b(static)75 102 y(allo)q(cation)h(can)g(b)q(e)g(done)g(in)g(a) f(pream)o(ble,)h(as)f(part)g(of)g(initialization)j(co)q(de.)27 b(Or,)17 b(it)h(can)g(b)q(e)g(done)f(a)75 158 y(compile/link)j(time,)d (if)h(the)f(implemen)o(tation)i(has)e(additional)h(mec)o(hanisms)g(to)e (reserv)o(e)h(con)o(text)p 1824 158 V 16 w(id)75 214 y(v)m(alues.)22 b(Comm)o(unicators)14 b(to)h(b)q(e)h(used)g(b)o(y)f (the)h(di\013eren)o(t)f(pro)q(cedures)i(can)e(b)q(e)h(build)h(in)g(a)e (pream)o(ble,)75 271 y(if)i(the)g(executing)h(groups)e(are)h (statically)g(de\014ned;)i(if)e(the)g(executing)h(groups)e(c)o(hange)h (dynamically)l(,)75 327 y(then)f(a)g(new)h(comm)o(unicator)e(has)h(to)g (b)q(e)h(built)g(whenev)o(er)g(the)f(executing)h(group)f(c)o(hanges,)g (but)h(this)75 384 y(new)k(comm)o(unicator)g(can)h(b)q(e)g(built)g (using)g(the)g(same)e(preallo)q(cated)j(con)o(text)p 1491 384 V 16 w(id.)39 b(If)21 b(the)g(parallel)75 440 y(pro)q(cedures)d(can)f(b)q(e)g(organized)g(in)o(to)g(libraries,)h(so)f (that)f(only)h(one)g(pro)q(cedure)h(of)e(eac)o(h)h(library)h(can)75 497 y(b)q(e)i(concurren)o(tly)g(activ)o(e)f(at)f(eac)o(h)i(pro)q (cessor,)f(then)g(it)h(is)g(su\016cien)o(t)f(to)g(allo)q(cate)h(one)f (con)o(text)f(p)q(er)75 553 y(library)l(.)75 687 y Fi(9.7.2)55 b(P)n(arallel)26 b(pro)r(cedures)f(that)i(are)f(nonreen)n(tran)n(t)g (within)g(eac)n(h)h(executing)246 746 y(group)75 836 y Fs(This)21 b(co)o(v)o(ers)g(the)g(case)f(where,)j(at)d(an)o(y)g(p)q (oin)o(t)i(in)f(time,)h(for)f(eac)o(h)g(pro)q(cess)g(group,)g(there)g (can)g(b)q(e)75 892 y(at)d(most)g(one)h(activ)o(e)g(in)o(v)o(ok)m (ation)g(of)g(a)f(parallel)i(pro)q(cedure)g(b)o(y)f(a)f(pro)q(cess)h (mem)o(b)q(er.)31 b(Ho)o(w)o(ev)o(er,)19 b(it)75 948 y(migh)o(t)i(b)q(e)h(p)q(ossible)i(that)c(the)i(same)f(pro)q(cedure)h (is)g(concurren)o(tly)g(in)o(v)o(ok)o(ed)g(in)g(t)o(w)o(o)e(partially)j (\(or)75 1005 y(completely\))17 b(o)o(v)o(erlapping)f(groups.)k(F)l(or) 15 b(example,)i(the)e(same)g(collectiv)o(e)j(comm)o(unication)e (function)75 1061 y(ma)o(y)e(b)q(e)i(concurren)o(tly)g(in)o(v)o(ok)o (ed)g(on)f(t)o(w)o(o)f(partially)i(o)o(v)o(erlapping)g(groups.)166 1120 y(In)i(suc)o(h)h(a)e(case,)h(a)g(con)o(text)p 674 1120 V 16 w(id)h(is)f(asso)q(ciated)g(with)g(eac)o(h)g(parallel)i(pro)q (cedure)e(and)g(eac)o(h)g(exe-)75 1176 y(cuting)f(group,)e(so)h(that)f (o)o(v)o(erlapping)i(execution)g(groups)e(ha)o(v)o(e)h(distinct)h(comm) o(unication)g(con)o(texts.)75 1233 y(\(One)c(do)q(es)g(not)g(need)h(a)e (di\013eren)o(t)h(con)o(text)p 831 1233 V 16 w(id)h(from)e(eac)o(h)h (group;)g(one)g(merely)g(needs)h(a)e(\\coloring")h(of)75 1289 y(the)j(groups,)f(so)g(that)g(One)h(can)g(generate)f(the)h(comm)o (unicators)f(for)g(eac)o(h)h(parallel)h(pro)q(cedure)f(when)75 1346 y(the)g(execution)i(groups)e(are)g(de\014ned.)25 b(Here,)17 b(again,)f(one)g(only)h(need)h(one)e(con)o(text)g(for)g(eac) o(h)g(library)l(,)75 1402 y(if)g(no)f(t)o(w)o(o)f(pro)q(cedures)i(from) e(the)h(same)g(library)h(can)g(b)q(e)g(concurren)o(tly)f(activ)o(e)h (in)g(the)f(same)g(group.)166 1461 y(Note)j(that,)h(for)f(collectiv)o (e)j(comm)o(unication)e(libraries,)i(w)o(e)d(do)h(allo)o(w)g(sev)o (eral)g(concurren)o(t)g(in-)75 1517 y(v)o(ok)m(ations)13 b(within)h(the)f(same)g(group:)18 b(a)13 b(broadcast)f(in)i(a)f(group)f (ma)o(y)h(b)q(e)g(started)f(at)h(a)f(pro)q(cess)i(b)q(efore)75 1574 y(the)h(previous)h(broadcast)f(in)h(that)e(group)h(ended)h(at)f (another)g(pro)q(cess.)20 b(In)c(suc)o(h)f(a)g(case,)g(one)g(cannot)75 1630 y(rely)e(on)f(con)o(text)g(mec)o(hanisms)h(to)f(disam)o(biguate)h (successiv)o(e)h(in)o(v)o(ok)m(ations)e(of)g(the)h(same)f(parallel)i (pro-)75 1687 y(cedure)f(within)h(the)f(same)f(group:)18 b(the)13 b(pro)q(cedure)g(need)h(b)q(e)f(implemen)o(ted)h(so)e(as)g(to) g(a)o(v)o(oid)h(confusion.)75 1743 y(E.g.,)h(for)h(broadcast,)f(one)h (ma)o(y)g(need)h(to)f(carry)f(additional)j(information)f(in)g (messages,)e(suc)o(h)i(as)f(the)75 1800 y(broadcast)h(ro)q(ot,)g(to)g (help)i(in)f(suc)o(h)g(disam)o(biguation;)i(one)d(also)h(relies)h(on)f (preserv)m(ation)g(of)f(message)75 1856 y(order)e(b)o(y)g(MPI.)g(With)g (suc)o(h)h(an)f(approac)o(h,)f(w)o(e)h(ma)o(y)g(b)q(e)h(gaining)g(p)q (erformance,)f(but)g(w)o(e)g(lo)q(ose)h(mo)q(d-)75 1913 y(ularit)o(y)l(.)28 b(It)18 b(is)g(not)f(su\016cien)o(t)i(to)e (implemen)o(t)i(the)f(parallel)h(pro)q(cedure)g(so)e(that)g(it)h(w)o (orks)f(correctly)75 1969 y(in)i(isolation,)h(when)g(in)o(v)o(ok)o(ed)e (only)i(once;)g(it)f(needs)g(to)f(b)q(e)i(implemen)o(ted)g(so)e(that)g (an)o(y)g(n)o(um)o(b)q(er)h(of)75 2026 y(successiv)o(e)d(in)o(v)o(ok)m (ations)f(will)h(execute)g(correctly)l(.)k(Of)14 b(course,)h(the)g (same)f(approac)o(h)g(can)h(b)q(e)g(used)h(for)75 2082 y(other)f(parallel)i(libraries.)75 2216 y Fi(9.7.3)55 b(W)-5 b(ell)18 b(nested)g(parallel)f(pro)r(cedures)75 2306 y Fs(Calls)e(of)g(parallel)h(pro)q(cedures)g(are)e(w)o(ell)i (nested)f(if)g(a)g(new)g(parallel)h(pro)q(cedure)g(is)f(alw)o(a)o(ys)f (in)o(v)o(ok)o(ed)h(in)75 2363 y(a)j(subset)h(of)e(a)h(group)g (executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 2419 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 2478 y(In)e(suc)o(h)f(a)g(case,)g(a)g(new)g (con)o(text)f(need)i(to)f(b)q(e)h(dynamically)g(allo)q(cated)h(for)d (eac)o(h)h(new)g(in)o(v)o(ok)m(ation)75 2534 y(of)d(a)g(parallel)i(pro) q(cedure.)20 b(Ho)o(w)o(ev)o(er,)10 b(a)g(stac)o(k)g(mec)o(hanism)h (can)f(b)q(e)i(used)f(for)f(allo)q(cating)h(new)g(con)o(texts.)75 2591 y(Th)o(us,)19 b(a)f(p)q(ossible)j(mec)o(hanism)e(is)g(to)f(allo)q (cate)h(\014rst)f(a)h(large)f(n)o(um)o(b)q(er)h(of)f(con)o(text)p 1566 2591 V 16 w(id's)h(\(up)g(to)f(the)75 2647 y(upp)q(er)h(b)q(ound)g (on)f(the)g(depth)h(of)f(nested)g(parallel)i(pro)q(cedure)f(calls\),)g (and)g(then)f(use)h(a)e(lo)q(cal)j(stac)o(k)75 2704 y(managemen)o(t)f (of)h(these)g(con)o(text)p 674 2704 V 15 w(id's)h(on)e(eac)o(h)h(pro)q (cess)h(to)e(create)g(a)h(new)g(comm)o(unicator)g(\(using)p eop %%Page: 105 110 105 109 bop 75 -100 a Fq(9.8.)34 b(LEFT)1504 b Fs(105)75 45 y Fl(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(MAKE)p Fs(\))15 b(for)f(eac)o(h)i(new)f(in)o(v)o(ok)m(ation.)166 184 y Fk(Discussion:)g Fj(General)f(case)166 323 y Fs(In)h(the)g (general)g(case,)g(there)f(ma)o(y)g(b)q(e)i(m)o(ultiple)g(concurren)o (tly)f(activ)o(e)g(in)o(v)o(ok)m(ations)g(of)g(the)f(same)75 380 y(parallel)23 b(pro)q(cedure)f(within)g(the)g(same)f(group;)i(in)o (v)o(ok)m(ations)f(ma)o(y)f(not)g(b)q(e)h(w)o(ell)g(nested.)38 b(A)22 b(new)75 436 y(con)o(text)17 b(need)i(to)e(b)q(e)h(created)g (for)f(eac)o(h)g(in)o(v)o(ok)m(ation.)28 b(It)18 b(is)g(the)g(user)g (resp)q(onsibilit)o(y)i(to)d(mak)o(e)g(sure)75 493 y(that,)k(if)h(t)o (w)o(o)d(distinct)j(parallel)h(pro)q(cedures)e(are)g(in)o(v)o(ok)o(ed)g (concurren)o(tly)h(on)e(o)o(v)o(erlapping)i(sets)e(of)75 549 y(pro)q(cesses,)15 b(then)h(con)o(text)p 538 549 14 2 v 16 w(id)g(allo)q(cation)g(or)f(comm)o(unicator)f(creation)i(is)g (prop)q(erly)g(co)q(ordinated.)75 692 y Fm(9.8)70 b(Left)75 794 y Fs(Remote)15 b(pro)q(cedure)h(call)h(and)e(C2)g(ob)s(jects.)166 850 y(Ho)o(w)f(to)h(handle)h(gro)o(wing)f(pro)q(cess)h(set.)p eop %%Page: 106 111 106 110 bop 75 356 a Fn(Chapter)34 b(10)75 564 y Fu(Initial)43 b(Implemen)m(tation)f(Subset)748 786 y Fs(Stev)o(en)15 b(Huss-Lederman)75 927 y Fm(10.1)70 b(In)n(tro)r(duction)75 1028 y Fs(This)11 b(c)o(hapter)f(de\014nes)h(a)f(minimal)i(subset)f(of) f(MPI)g(for)g(initial)i(implemen)o(tation.)19 b(This)11 b(subset)g(is)g(b)q(eing)75 1085 y(de\014ned)i(so)f(that)f(consisten)o (t)h(implemen)o(tations)h(can)f(app)q(ear)g(more)f(rapidly)l(.)21 b(It)12 b(w)o(as)f(recognized)i(early)75 1141 y(in)h(the)e(pro)q(cess)i (that)e(MPI)g(needed)i(to)f(app)q(ear)f(as)h(quic)o(kly)h(as)e(p)q (ossible)j(and)e(practical.)20 b(The)13 b(creation)75 1198 y(of)k(a)g(subset)g(will)i(hop)q(efully)g(allo)o(w)e(users)g (earlier)h(access)g(to)e(the)h(standard)g(and)g(still)i(allo)o(w)e(for) g(the)75 1254 y(writing)f(of)e(p)q(ortable)i(message)f(passing)g(co)q (des.)166 1311 y(This)20 b(subset)g(should)h(not)e(in)i(an)o(y)e(w)o(a) o(y)g(b)q(e)h(construed)g(as)f(a)h(limitation)h(on)e(MPI)h(implemen-)75 1367 y(tations.)35 b(It)20 b(is)h(strictly)g(the)g(minim)o(um)g (necessary)g(to)e(ha)o(v)o(e)h(an)h(initial)h(MPI)e(subset)h (conforming)75 1423 y(implemen)o(tation.)28 b(It)18 b(is)g(hop)q(ed)g (that)f(an)g(o\016cially)i(sanctioned)g(subset)e(will)i(encourage)f (and)g(allo)o(w)75 1480 y(implemen)o(tors)c(to)e(in)o(tro)q(duce)j(MPI) e(in)h(a)e(more)h(timely)h(fashion.)20 b(It)13 b(should)h(b)q(e)g (noted,)f(ho)o(w)o(ev)o(er,)g(that)75 1536 y(implemen)o(tation)20 b(of)e(the)h(subset)g(do)q(es)g(not)f(mak)o(e)h(an)f(implemen)o(tation) i(conform)e(with)h(MPI.)g(The)75 1593 y(subset)g(is)g(only)h(a)e(p)q (oten)o(tial)i(\014rst)e(step)h(in)h(the)f(pro)q(cess.)31 b(All)20 b(implemen)o(tations)g(m)o(ust)f(ultimately)75 1649 y(conform)14 b(to)h(the)g(en)o(tire)g(standard;)g(implemen)o (tations)h(are)f(encouraged)g(to)f(do)h(the)g(full)i(standard)d(as)75 1706 y(rapidly)i(as)f(p)q(ossible.)166 1762 y(The)c(subset)g(presen)o (ted)g(is)g(consisten)o(t)f(with)h(the)g(complete)g(MPI)g(standard.)18 b(This)11 b(w)o(as)f(an)g(imp)q(or-)75 1819 y(tan)o(t)i(goal)g(so)h (the)g(additional)h(MPI)f(features)f(could)i(b)q(e)g(added)f(without)g (c)o(hanging)g(an)o(y)g(functionalit)o(y)75 1875 y(from)19 b(the)i(user's)f(p)q(ersp)q(ectiv)o(e.)36 b(Th)o(us,)21 b(users)f(can)g(b)q(e)h(assured)f(that)g(programs)f(written)h(no)o(w)g (for)75 1932 y(the)c(subset)f(will)i(run)f(without)g(mo)q(di\014cation) g(under)g(the)g(full)h(MPI)e(standard.)21 b(F)l(urthermore,)14 b(using)75 1988 y(additional)j(features)f(of)g(the)g(full)h(MPI)f (standard)f(in)i(the)f(future)g(will)i(not)d(require)i(c)o(hanges)f(to) f(co)q(de)75 2044 y(already)f(written.)19 b(Users)13 b(ma)o(y)g(use)h(MPI)f(features)g(outside)i(this)e(subset)h(that)f(are) g(o\013ered)g(b)o(y)g(v)m(arious)75 2101 y(implemen)o(tors.)33 b(Ho)o(w)o(ev)o(er,)19 b(p)q(eople)i(who)e(require)i(p)q(ortabilit)o(y) f(during)g(the)g(early)f(dev)o(elopmen)o(t)h(of)75 2157 y(MPI)15 b(ma)o(y)g(exp)q(erience)i(some)e(di\016culties)i(un)o(til)g (later)e(in)h(the)f(dev)o(elopmen)o(t)h(pro)q(cess.)75 2298 y Fm(10.2)70 b(Criteria)21 b(and)j(Rational)75 2399 y Fs(Ha)o(ving)19 b(the)g(subset)g(b)q(e)h(consisten)o(t)f(with)g(the)g (en)o(tire)g(MPI)g(standard)g(w)o(as)f(considered)i(critical)h(to)75 2456 y(the)16 b(e\013ort.)22 b(In)16 b(addition)i(to)d(this,)h(there)h (w)o(ere)e(man)o(y)h(p)q(ossible)i(criteria)f(to)e(use)h(in)h (determining)h(the)75 2512 y(elemen)o(ts)d(of)e(MPI)h(to)f(include)j (in)f(the)f(subset.)20 b(The)14 b(main)g(criteria)h(used)f(w)o(ere)g (that)f(the)h(MPI)g(subset)75 2569 y(should)131 2647 y(1.)22 b(con)o(tain)13 b(routines)h(that)e(are)h(as)g(close)h(as)f(p)q (ossible)i(to)e(curren)o(t)g(standard)g(practice)h(to)f(minimize)189 2704 y(the)i(e\013ort)f(to)h(p)q(ort)f(co)q(des.)941 2828 y(106)p eop %%Page: 107 112 107 111 bop 75 -100 a Fq(10.2.)34 b(CRITERIA)16 b(AND)f(RA)l(TIONAL)997 b Fs(107)131 45 y(2.)22 b(con)o(tain)15 b(as)g(man)o(y)g(new)g(and)g (imp)q(ortan)o(t)g(MPI)g(features)g(as)g(p)q(ossible.)131 218 y(3.)22 b(allo)o(w)17 b(dev)o(elop)q(ers)i(to)e(b)q(e)i(able)f(to)f (ha)o(v)o(e)g(the)h(subset)g(sho)o(w)f(up)h(as)f(rapidly)i(as)e(p)q (ossible)i(while)189 274 y(still)d(meeting)g(the)f(other)g(criteria.) 166 447 y(There)i(w)o(ere)g(man)o(y)f(rationales)h(for)g(these)g (criteria.)26 b(It)17 b(w)o(as)f(recognized)i(that)e(it)h(is)h(imp)q (ossible)75 503 y(to)h(come)h(up)g(with)g(an)g(ideal)h(subset)f(and)g (compromises)g(w)o(ere)g(necessary)l(.)34 b(It)19 b(w)o(as)g(felt)i (that)e(cur-)75 560 y(ren)o(t)h(users)g(should)i(b)q(e)f(comfortable)f (in)h(migrating)f(to)g(MPI.)g(Th)o(us,)h(the)g(subset)f(should)i(con)o (tain)75 616 y(the)17 b(message)f(passing)h(elemen)o(ts)g(that)f (represen)o(t)g(common)g(practice)i(to)q(da)o(y)l(.)23 b(F)l(urthermore,)16 b(it)h(w)o(as)75 673 y(felt)f(that)g(the)g (initial)i(\(and)e(later\))g(v)o(ersion)g(of)g(MPI)g(should)h(main)o (tain)g(e\016ciency)l(.)24 b(Otherwise)17 b(users)75 729 y(could)k(su\013er)e(from)g(disenc)o(han)o(tmen)o(t)h(with)g(MPI)f (on)h(their)g(\014rst)f(usage)h(and)f(ma)o(y)g(nev)o(er)h(giv)o(e)g (the)75 785 y(standard)e(a)h(second)g(try)l(.)29 b(Ha)o(ving)19 b(the)g(standard)f(routines)h(in)h(the)e(subset)h(should)h(encourage)e (de-)75 842 y(v)o(elop)q(ers)g(to)f(optimized)i(these)f(routines)g(in)h (the)f(initial)h(v)o(ersion)f(of)f(MPI.)h(Also,)g(though)f(some)h(felt) 75 898 y(that)c(the)g(p)q(ortabilit)o(y)h(o\013ered)f(b)o(y)g(the)h (MPI)f(subset)h(w)o(as)e(su\016cien)o(t)i(to)f(en)o(tice)h(users,)f (man)o(y)g(felt)h(that)75 955 y(the)20 b(subset)g(should)g(con)o(tain)g (some)f(new)h(feature\(s\))f(o)o(v)o(er)g(and)g(ab)q(o)o(v)o(e)h (curren)o(t)f(common)g(practice)75 1011 y(as)d(an)g(added)h(incen)o (tiv)o(e.)24 b(The)17 b(additional)h(items)e(selected)i(w)o(ere)e(mean) o(t)f(to)h(represen)o(t)g(some)g(of)g(the)75 1068 y(signi\014can)o(t)i (new)f(features)f(of)h(MPI.)f(Balanced)i(against)f(these)g(\014rst)f(t) o(w)o(o)g(goals)h(is)g(the)g(need)h(for)e(the)75 1124 y(subset)e(to)f(sho)o(w)g(up)i(in)f(a)g(timely)h(fashion.)k(Th)o(us,)14 b(eac)o(h)g(feature)g(c)o(hosen)g(for)f(inclusion)j(in)f(the)f(subset) 75 1181 y(w)o(as)g(deemed)h(of)f(su\016cien)o(t)h(imp)q(ortance)g(to)f (out)o(w)o(eigh)g(its)h(added)g(complexit)o(y)g(in)h(implemen)o(ting)g (the)75 1237 y(subset.)k(Though)14 b(some)h(functions)g(seem)g(easy)f (to)g(implemen)o(t)i(there)e(are)h(often)f(o)o(v)o(erlo)q(ok)o(ed)g (costs)g(in)75 1294 y(testing,)f(do)q(cumen)o(tation)h(and)g(dev)o (elopmen)o(t)g(that)e(w)o(as)h(considered)h(b)q(efore)g(a)f(feature)g (w)o(as)g(added)h(to)75 1350 y(the)j(MPI)g(subset.)25 b(Inclusion)20 b(of)c(to)q(o)g(man)o(y)h(routines)g(migh)o(t)g(lead)h (to)e(mo)q(derately)i(e\016cien)o(t)f(imple-)75 1406 y(men)o(tation)g(of)f(them)h(all)h(instead)f(of)g(a)f(v)o(ery)h (e\016cien)o(t)h(and)f(p)q(ossibly)h(more)f(useful)h(implemen)o(tation) 75 1463 y(of)d(a)g(smaller)h(subset.)166 1622 y Fk(Discussion:)32 b Fj(Some)11 b(felt)h(that)g(w)o(e)g(should)g(aim)e(the)j(subset)g(at)f (t)o(ypical)f(users.)19 b(Should)12 b(this)g(b)q(e)h(another)75 1678 y(criteria?)18 b(Is)d(the)f(compatibilit)o(y)d(with)i(curren)o(t)j (practice)f(enough?)166 1920 y Fk(Discussion:)36 b Fj(Should)14 b(it)g(b)q(e)h(a)f(criteria)h(that)f(the)h(subset)h(and)e(common)e (practice)j(features)h(b)q(e)f(imple-)75 1976 y(men)o(ted)g(e\016cien)o (tly)m(.)21 b(The)16 b(text)g(touc)o(hes)g(on)f(this)h(but)f(it)g(is)g (not)g(a)g(sp)q(eci\014c)i(item.)k(I)15 b(did)g(not)g(giv)o(e)f(it)h(a) g(bullet)75 2033 y(since)f(it)f(seems)g(lik)o(e)f(a)h(qualit)o(y)f(of)g (implemen)o(tatio)o(n)f(issue.)18 b(P)o(o)q(orly)12 b(implemen)o(ted)f (subsets)k(will)d(b)q(e)h(forced)h(out)75 2089 y(b)o(y)g(the)g(mark)o (et.)166 2331 y Fk(Discussion:)49 b Fj(Should)17 b(it)h(b)q(e)g(a)f (criteria)h(that)g(the)g(rest)h(of)e(MPI)h(b)q(e)h(able)e(to)h(b)q(e)g (la)o(y)o(ered)g(on)f(top)h(so)75 2387 y(users)f(migh)o(t)c(get)j (early)f(\(though)g(lo)o(w)o(er)g(p)q(erformance\))g(access)j(to)d(the) h(complete)e(standard?)23 b(This)15 b(is)h(aimed)75 2443 y(at)g(allo)o(wing)f(easy)i(and)f(e\016cien)o(t)h(la)o(y)o(ering)e(of)h (the)h(rest)h(so)f(that)f(a)g(\\mac)o(hine)f(indep)q(enden)o(t")j(pac)o (k)n(age)e(could)75 2500 y(b)q(e)i(created)g(without)f(ha)o(ving)f(the) h(dev)o(elop)q(er)h(jump)d(through)i(lots)g(of)f(hops.)28 b(This)17 b(migh)o(t)e(happ)q(en)i(if)f(a)h(few)75 2556 y(functions)e(w)o(ere)i(left)e(out)g(that)g(didn't)g(meet)g(the)h (other)g(criteria.)22 b(Ho)o(w)o(ev)o(er,)16 b(some)e(felt)h(it)g(w)o (as)g(not)h(a)f(strong)75 2613 y(enough)f(reason)g(to)g(lea)o(v)o(e)g (it)f(in)h(and)f(m)o(y)g(lead)g(to)h(p)q(o)q(or)g(MPI)g(implemen)o (tatio)o(ns)e(of)h(these)i(la)o(y)o(ered)f(routines.)p eop %%Page: 108 113 108 112 bop 75 -100 a Fs(108)571 b Fq(CHAPTER)15 b(10.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)75 45 y Fm(10.3)70 b(Subset)23 b(F)-6 b(unctionalit)n(y)75 240 y Fk(Discussion:)45 b Fj(This)17 b(section)g(only)f(discuss)i(the)g (pt-to-pt)e(and)h(collcomm)c(c)o(hapters.)28 b(The)17 b(other)h(c)o(hapters)75 296 y(will)12 b(b)q(e)j(added)f(after)g(they)h (are)f(read)g(at)g(meetings)f(and)h(\014nalized.)75 532 y Fi(10.3.1)55 b(Itemized)16 b(F)-5 b(unctionalit)n(y)75 628 y Fs(The)22 b(T)l(able)h(b)q(elo)o(w)g(summarizes)f(the)g(sp)q (eci\014c)i(functionalit)o(y)g(included)g(and)f(excluded)h(from)d(the) 75 684 y(subset.)75 780 y(Section)206 b(Op)q(eration)337 b(Included)404 b(Excluded)p 75 799 1882 2 v 709 871 a(P)o(oin)o(t)15 b(to)f(P)o(oin)o(t)h(Comm)o(unication)p 709 878 614 2 v 100 945 a(1.5.2)151 b(Bu\013er)16 b(Op)q(erations)194 b(\\All")16 b(functions)112 b(Multiple)17 b(app)q(ends,)f(h)f(v)o (ersions\(?\))118 1001 y(1.7)210 b(Comm.)19 b(Mo)q(de)293 b(Standard)287 b(Ready)l(,)16 b(Sync)o(hronous)100 1057 y(1.8.1)86 b(Comm.)19 b(Ob)s(ject)d(Creation)220 b(All\(?\))100 1114 y(1.8.2)197 b(Comm.)19 b(Start)172 b(ST)l(AR)l(T\(if)16 b(ha)o(v)o(e)f(INIT\))100 1170 y(1.8.3)132 b(Comm.)20 b(Completion)295 b(All)100 1227 y(1.8.4)123 b(Multiple)17 b(Completion)286 b(All)118 1283 y(1.9)182 b(Blo)q(c)o(king)16 b(Comm.)235 b(Std.)20 b(v)o(ersion)238 b(Ready)l(,)16 b(Sync)o(h.)k(v)o(ersion)107 1340 y(1.10)133 b(Non)o(blo)q(c)o(king)16 b(Comm.)198 b(Std.)20 b(v)o(ersion)238 b(Ready)l(,)16 b(Sync)o(h.)k(v)o(ersion)107 1396 y(1.11)75 b(Con)o(tiguous)15 b(Bu\013er)g(Comm.)141 b(Std.)20 b(v)o(ersion)238 b(Ready)l(,)16 b(Sync)o(h.)k(v)o(ersion)107 1453 y(1.12)776 b(PR)o(OBE)291 b(CANCEL,GET)p 1726 1453 14 2 v 16 w(LEN)754 1551 y(Collectiv)o(e)16 b(Comm)o(unication)p 754 1558 524 2 v 118 1624 a(2.4)188 b(Sync)o(hronization)334 b(All)118 1680 y(2.5)238 b(Data)14 b(Mo)o(v)o(e)353 b(All\(?\))118 1737 y(2.6)183 b(Global)16 b(Compute)266 b(MPI)15 b(Ops)396 b(User)15 b(Ops)75 1858 y(All)h(other)e(general)h(functionalit)o(y)h(is)f(included)i(in)f(the)e (subset.)20 b(Th)o(us,)14 b(all)i(matc)o(hing)f(criteria,)g(order,)75 1915 y(etc.)20 b(are)15 b(part)f(of)h(the)g(subset.)75 2068 y Fi(10.3.2)55 b(P)n(oin)n(t)20 b(to)e(P)n(oin)n(t)h(F)-5 b(unctionalit)n(y)75 2164 y Fs(The)12 b(op)q(erations)f(in)o(v)o (olving)i(con)o(tiguous)e(bu\013ers)g(are)h(included)i(since)e(they)g (represen)o(t)f(standard)g(prac-)75 2220 y(tice)k(to)q(da)o(y)l(.)20 b(In)15 b(addition,)g(the)g(comm)o(unications)g(bu\013ers)g(are)f(also) h(included)i(b)q(ecause)e(they)g(o\013er)f(an)75 2277 y(imp)q(ortan)o(t)20 b(new)g(feature)g(in)h(MPI.)e(Ho)o(w)o(ev)o(er,)h (to)g(simplify)i(the)e(implemen)o(tation,)i(the)e(comm)o(uni-)75 2333 y(cation)h(bu\013er)g(will)i(b)q(e)e(limited)i(to)d(one)h(comp)q (onen)o(t,)i(e.g.,)e(only)g(one)g(app)q(end)h(is)g(allo)o(w)o(ed.)37 b(This)75 2390 y(limitation)16 b(will)h(still)g(allo)o(w)e(the)g(user)g (access)h(to)e(the)h(strided)h(and)f(indexed)i(cases)e(but)g(require)h (more)75 2446 y(complex)g(cases)f(to)g(b)q(e)h(done)f(b)o(y)g(the)h (user.)166 2591 y Fk(Discussion:)40 b Fj(If)15 b(w)o(e)h(eliminated)d (comm)o(unication)f(bu\013ers)17 b(then)f(the)g(n)o(um)o(b)q(er)f(of)g (routines)h(w)o(ould)e(b)q(e)75 2647 y(greatly)f(reduced.)19 b(W)m(e)12 b(w)o(ould)g(only)g(ha)o(v)o(e)h(the)g(\\C")f(routines)i(in) e(collectiv)o(e)h(comm)o(unicatio)o(ns)e(and)h(con)o(tiguous)75 2704 y(in)h(pt-2-pt.)18 b(Righ)o(t)13 b(no)o(w)g(I)h(think)g(w)o(e)g (should)g(lea)o(v)o(e)f(it)h(in)f(but)h(I)g(p)q(oin)o(t)g(this)f(out)p eop %%Page: 109 114 109 113 bop 75 -100 a Fq(10.3.)34 b(SUBSET)15 b(FUNCTIONALITY)1014 b Fs(109)166 45 y Fk(Discussion:)48 b Fj(The)18 b(heterogeneous)i (cases)e(\(h)o(v)o(ector,)h(hindexed\))f(seem)g(v)o(ery)f(easy)h(to)g (co)q(de)g(but)g(are)75 102 y(another)d(set)g(of)f(functions.)19 b(They)c(allo)o(w)d(elemen)o(ts)j(of)e(a)h(C)h(t)o(yp)q(e)f(structure)j (to)d(b)q(e)h(pulled)f(out)g(in)g(the)h(subset.)75 158 y(It)g(has)g(b)q(een)h(p)q(oin)o(ted)f(out)g(that)g(ev)o(en)h(simple)e (routines)h(ha)o(v)o(e)g(a)g(cost)g(of)g(testing,)g(do)q(cumen)o (tation,)f(etc.)22 b(that)75 214 y(can)14 b(easily)f(out)o(w)o(eigh)h (the)g(co)q(ding)g(cost.)166 354 y Fs(Of)j(the)g(three)h(t)o(yp)q(es)f (of)f(comm)o(unication)i(mo)q(des,)f(the)h(ST)l(AND)o(ARD)f(mo)q(de)g (represen)o(ts)g(stan-)75 410 y(dard)12 b(practice)g(and)g(is)g (included.)22 b(The)12 b(other)f(t)o(w)o(o)g(mo)q(des)h(are)f(not)g (used)i(as)e(commonly)h(and)g(therefore)75 467 y(excluded.)22 b(This)16 b(applies)g(to)f(all)h(of)f(the)g(p)q(oin)o(t-to-p)q(oin)o(t) h(comm)o(unications)g(routines.)166 523 y(The)h(common)g(v)o(ersions)h (of)f(blo)q(c)o(king)i(and)e(non)o(blo)q(c)o(king)i(send)f(and)f (receiv)o(e)h(are)f(included.)29 b(In)75 579 y(addition,)16 b(the)f(lo)o(w)g(lev)o(el)i(routines)e(of)g(INIT)h(and)f(ST)l(AR)l(T)h (are)f(included.)23 b(These)15 b(should)h(help)h(users)75 636 y(who)12 b(wish)i(greater)d(con)o(trol)i(o)o(v)o(er)e(comm)o (unications)j(for)e(optimization)h(purp)q(oses.)20 b(F)l(or)12 b(example,)i(this)75 692 y(w)o(ould)i(allo)o(w)f(for)g(the)g(reuse)g (of)g(a)g(p)q(ersisten)o(t)h(handle.)166 831 y Fk(Discussion:)44 b Fj(Using)16 b(INIT,)h(ST)m(AR)m(T,)d(COMPLETE,)j(FREE)g(is)f(not)h (what)f(most)f(p)q(eople)i(do)f(to)q(da)o(y)75 888 y(but)c(do)q(es)h (allo)o(w)d(for)h(greater)i(sp)q(eed.)19 b(It)12 b(seems)g(as)g(if)f(y) o(ou)h(w)o(ould)f(ha)o(v)o(e)g(this)h(in)g(a)f(system)h(an)o(yw)o(a)o (y)f(and)g(it)h(w)o(ould)75 944 y(b)q(e)j(useful)e(to)h(the)h(user.)k (Should)13 b(w)o(e)h(include)g(it?)75 1149 y Fi(10.3.3)55 b(Collectiv)n(e)17 b(Comm)n(unication)f(F)-5 b(unctionalit)n(y)75 1235 y Fs(The)18 b(collectiv)o(e)i(comm)o(unication)f(routines)g(ha)o (v)o(e)f(the)g(same)g(general)g(restrictions)h(as)f(the)g(p)q(oin)o(t)h (to)75 1291 y(p)q(oin)o(t)d(routines.)k(Therefore,)15 b(the)g(bu\013er)g(descriptors)h(will)h(b)q(e)e(limited)i(to)e(one)g (en)o(try)l(.)166 1347 y(A)h(collectiv)o(e)h(comm)o(unication)g(op)q (erates)e(on)h(a)f(group.)21 b(This)c(de\014nes)g(the)e(scop)q(e)i(of)e (the)h(op)q(era-)75 1404 y(tion.)k(Man)o(y)15 b(curren)o(t)g(users)g (only)h(op)q(erate)f(on)h(all)g(the)f(pro)q(cesses.)21 b(This)15 b(is)h(the)g(MPI)p 1589 1404 14 2 v 16 w(ALL)g(group)f(in)75 1460 y(MPI.)g(T)l(o)g(simplify)i(the)e(subset,)g(groups)g(will)i(b)q(e) f(limited)h(to)d(b)q(e)i(MPI)p 1319 1460 V 16 w(ALL.)166 1600 y Fk(Discussion:)32 b Fj(There)13 b(are)g(a)f(lot)g(of)f(data)h (mo)o(v)o(e)f(functions.)17 b(I)12 b(could)g(not)h(think)e(of)h(a)g(w)o (a)o(y)f(to)h(dra)o(w)h(a)e(line)75 1656 y(to)j(exclude)h(some.)i(Do)d (w)o(e)g(w)o(an)o(t)g(all)f(of)g(these)i(in)f(the)h(subset.)20 b(Ho)o(w)13 b(often)h(are)h(ALLCAST,)e(ALLSCA)m(TTER)75 1712 y(used?)166 1934 y Fk(Discussion:)34 b Fj(F)m(or)13 b(the)i(subset)g(do)f(w)o(e)g(w)o(an)o(t)f(b)q(oth)h(the)h(REDUCE)f (and)f(ALLREDUCE)h(v)o(ersions?)p eop %%Page: 110 115 110 114 bop 75 379 a Fu(Biblio)q(graph)m(y)98 604 y Fs([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 661 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 717 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 820 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 876 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 933 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1035 y([3])j(R.)14 b(Butler)h(and)f(E.)f(Lusk.)18 b(User's)13 b(guide)i(to)e(the)h(P4)g(programming)f(system.)k(T)l(ec)o(hnical)f (Rep)q(ort)168 1092 y(TM-ANL{92/17,)e(Argonne)h(National)h(Lab)q (oratory)l(,)e(1992.)98 1194 y([4])21 b(S.)c(Chittor)g(and)h(R.)f(J.)g (En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d (mesh{connected)i(w)o(ormhole{)168 1251 y(routed)f(net)o(w)o(orks)e (for)i(in)o(terpro)q(cessor)f(comm)o(unication)i(in)f(m)o (ulticomputers.)25 b(In)17 b Fh(Pr)n(o)n(c)n(e)n(e)n(dings)168 1307 y(of)g(the)f(1990)i(Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p Fs(,)c(pages)j(647{656,)e(1990.)98 1410 y([5])21 b(S.)f(Chittor)g(and)g (R.)g(J.)f(En)o(b)q(o)q(dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g (of)f(mapping)i(on)e(the)h(comm)o(unica-)168 1466 y(tion)g(p)q (erformance)g(of)f(large)g(m)o(ulticomputers.)34 b(In)20 b Fh(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f(1991)h(International)168 1523 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n (essing,)f(vol.)i(II)f(\(Softwar)n(e\))p Fs(,)f(pages)h(I)q(I{1)h({)f (I)q(I{4,)g(1991.)98 1626 y([6])21 b(J.)13 b(Dongarra,)f(R.)h(Hemp)q (el,)i(A.)e(Hey)l(,)g(and)h(D.)e(W)l(alk)o(er.)k(A)e(prop)q(osal)f(for) f(a)h(user{lev)o(el,)i(message{)168 1682 y(passing)i(in)o(terface)g(in) g(a)f(distributed)i(memory)e(en)o(vironmen)o(t.)23 b(T)l(ec)o(hnical)18 b(Rep)q(ort)f(TM-12231,)168 1739 y(ORNL,)g(Octob)q(er)e(1992.)98 1841 y([7])21 b(Edin)o(burgh)g(P)o(arallel)f(Computing)g(Cen)o(tre,)g (Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fh(CHIMP)19 b(Conc)n(epts)p Fs(,)168 1898 y(June)e(1991.)98 2000 y([8])k(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f(Univ)o (ersit)o(y)h(of)g(Edin)o(burgh.)22 b Fh(CHIMP)15 b(V)m(ersion)h(1.0)168 2057 y(Interfac)n(e)p Fs(,)e(Ma)o(y)g(1992.)98 2159 y([9])21 b(G.)13 b(Geist)f(and)h(V.)g(Sunderam.)k(Net)o(w)o(ork)11 b(based)i(concurren)o(t)g(computing)h(on)e(the)h(PVM)g(system.)168 2216 y(T)l(ec)o(hnical)k(Rep)q(ort)f(TM-11760,)d(Oak)i(Ridge)i (National)e(Lab)q(oratory)l(,)f(June)j(1991.)75 2319 y([10])k(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o(eyton,) f(and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f (PICL:)g(a)168 2375 y(p)q(ortable)f(instrumen)o(ted)f(comm)o(unication) h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e(TM-11616,)f(Oak)h(Ridge) 168 2432 y(National)k(Lab)q(oratory)l(,)e(Octob)q(er)i(1990.)75 2534 y([11])21 b(R.)15 b(Hemp)q(el.)20 b(The)14 b(ANL/GMD)g(macros)f (\(P)l(ARMA)o(CS\))h(in)i(fortran)d(for)g(p)q(ortable)i(parallel)h (pro-)168 2591 y(gramming)f(using)g(the)g(message)f(passing)h (programming)f(mo)q(del)i({)e(users')h(guide)g(and)g(reference)168 2647 y(man)o(ual.)23 b(T)l(ec)o(hnical)18 b(rep)q(ort,)d(GMD,)g(P)o (ostfac)o(h)f(1316,)h(D-5205)g(Sankt)h(Augustin)g(1,)g(German)o(y)l(,) 168 2704 y(No)o(v)o(em)o(b)q(er)f(1991.)941 2828 y(110)p eop %%Page: 111 116 111 115 bop 75 -100 a Fq(BIBLIOGRAPHY)1371 b Fs(111)75 45 y([12])21 b(R.)15 b(Hemp)q(el,)h(H.-C.)e(Hopp)q(e,)h(and)f(A.)h (Supalo)o(v.)k(P)o(armacs{6.0)12 b(library)k(in)o(terface)e(sp)q (eci\014cation.)168 102 y(T)l(ec)o(hnical)h(rep)q(ort,)d(GMD,)g(P)o (ostfac)o(h)f(1316,)h(D-5205)f(Sankt)h(Augustin)i(1,)e(German)o(y)l(,)h (Decem)o(b)q(er)168 158 y(1992.)75 252 y([13])21 b(nCUBE)16 b(Corp)q(oration.)j Fh(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i (r2.0)p Fs(,)f(Decem)o(b)q(er)f(1990.)75 346 y([14])21 b(H.)c(M)q(\177)-24 b(uhlen)o(b)q(ein)19 b(O.)d(Kr\177)-23 b(amer.)24 b(Mapping)16 b(strategies)g(in)i(message{based)e(m)o (ultipro)q(cessor)h(sys-)168 402 y(tems.)j Fh(Par)n(al)r(lel)15 b(Computing)p Fs(,)g(9:213{225,)d(1989.)75 496 y([15])21 b(P)o(arasoft)14 b(Corp)q(oration.)22 b Fh(Expr)n(ess)16 b(V)m(ersion)g(1.0:)22 b(A)17 b(Communic)n(ation)g(Envir)n(onment)e (for)j(Par-)168 552 y(al)r(lel)e(Computers)p Fs(,)f(1988.)75 646 y([16])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g (system.)22 b(In)17 b Fh(Pr)n(o)n(c)n(e)n(e)n(dings)e(of)i(the)g(Thir)n (d)g(Confer)n(enc)n(e)e(on)168 703 y(Hyp)n(er)n(cub)n(e)g(Concurr)n (ent)f(Computers)i(and)f(Applic)n(ations)p Fs(,)e(pages)h(384{390.)d(A) o(CM)j(Press,)f(1988.)75 796 y([17])21 b(A.)14 b(Skjellum)i(and)e(A.)g (Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f (comm)o(unication)h(library)168 853 y(atop)g(the)f(reactiv)o(e)h(k)o (ernel.)20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g (editors,)g Fh(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 909 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g (Confer)n(enc)n(e)p Fs(,)e(pages)i(767{776.)d(IEEE)168 966 y(Press,)d(1990.)75 1060 y([18])21 b(A.)12 b(Skjellum,)i(S.)e (Smith,)h(C.)e(Still,)j(A.)d(Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q (co)q(de)g(message)f(passing)168 1116 y(system.)20 b(T)l(ec)o(hnical)d (rep)q(ort,)d(La)o(wrence)i(Liv)o(ermore)f(National)h(Lab)q(oratory)l (,)e(Septem)o(b)q(er)i(1992.)75 1210 y([19])21 b(V.)e(Sunderam.)30 b(PVM:)18 b(a)g(framew)o(ork)f(for)h(parallel)i(distributed)h (computing.)30 b Fh(Concurr)n(ency:)168 1266 y(Pr)n(actic)n(e)16 b(and)g(Exp)n(erienc)n(e)p Fs(,)e(2\(4\):315{339)o(,)e(1990.)75 1360 y([20])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g (passing)g(in)i(a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec) o(h-)168 1417 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National) g(Lab)q(oratory)l(,)e(August)h(1992.)p eop %%Page: 112 117 112 116 bop 75 356 a Fn(App)s(endix)34 b(A)75 564 y Fu(Pro)s(cess)40 b(T)-10 b(op)s(ology)42 b(Routines)75 786 y Fs(In)16 b(this)f(app)q(endix)i(F)l(ortran)d(sp)q(eci\014cations)j(for)e(the)g (prop)q(osed)h(top)q(ology)f(functions)h(are)f(giv)o(en.)191 895 y Fo(\017)g Ft(MPI)p 338 895 16 2 v 18 w(CAR)l(T)149 b Fs(Map)15 b(a)g(cartesian)g(structure)g(\(grid/torus\))191 952 y Fo(\017)g Ft(MPI)p 338 952 V 18 w(MAKDIM)57 b Fs(Divide)17 b(a)d(pro)q(cess)i(group)f(among)f(n)i(cartesian)f(dimensions)191 1008 y Fo(\017)g Ft(MPI)p 338 1008 V 18 w(P)l(AR)l(TC)117 b Fs(P)o(artition)15 b(a)g(cartesian)g(structure)191 1065 y Fo(\017)g Ft(MPI)p 338 1065 V 18 w(GRAPH)101 b Fs(Map)15 b(a)g(graph)191 1121 y Fo(\017)g Ft(MPI)p 338 1121 V 18 w(INQMAP)72 b Fs(Chec)o(k)15 b(mapping)h(t)o(yp)q(e)191 1178 y Fo(\017)f Ft(MPI)p 338 1178 V 18 w(INQCAR)l(T)49 b Fs(Find)16 b(size)g(of)f(cartesian)g(structure)g(and)g(o)o(wn)g(p)q (osition)191 1234 y Fo(\017)g Ft(MPI)p 338 1234 V 18 w(INQGR)118 b Fs(Find)16 b(o)o(wn)f(no)q(de)g(n)o(um)o(b)q(er)h(in)g (graph)191 1291 y Fo(\017)f Ft(MPI)p 338 1291 V 18 w(INQPID)102 b Fs(Lo)q(ok)15 b(up)h(PID)f(n)o(um)o(b)q(ers)941 2828 y(112)p eop %%Page: 113 118 113 117 bop 1807 -100 a Fs(113)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(CAR)l(T)121 b Fs(Arrange)16 b(the)g(pro)q(cesses)h(of)f(a)g(pro)q(cess)h(group)f(in)h(a)f (cartesian)g(structure)573 158 y(\(torus/grid\))e(with)i Fl(n)f Fs(dimensions.)75 256 y Ft(SYNOPSIS)179 312 y Fs(in)o(teger)g(function)h(MPI)p 599 312 14 2 v 17 w(CAR)l(T)f(\(gid,)g (ndim,)h(dims,)f(p)q(erio)q(d\))179 369 y(in)o(teger)g(gid)179 425 y(in)o(teger)g(ndim)179 482 y(in)o(teger)g(dims\()p Fo(\003)p Fs(\))179 538 y(in)o(teger)g(p)q(erio)q(d\()p Fo(\003)p Fs(\))75 636 y Ft(INPUT)i(AR)o(GUMENTS)179 693 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q (cess)h(group)f(to)f(b)q(e)i(structured.)179 749 y(ndim)294 b(the)15 b(n)o(um)o(b)q(er)h(of)f(dimensions)i(of)d(the)i(torus.)179 806 y(dims)301 b(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(in)g(eac)o(h) f(co)q(ordinate)h(direction.)179 862 y(p)q(erio)q(d)269 b(p)q(erio)q(dicit)o(y)18 b(sp)q(eci\014cation)f(in)f(eac)o(h)f(co)q (ordinate)h(direction.)75 960 y Ft(DESCRIPTION)179 1016 y Fs(The)f(pro)q(cesses)g(of)g(the)g(group)g(with)g(ID)g(n)o(um)o(b)q (er)g Fl(gid)g Fs(are)f(arranged)h(in)h(a)e(cartesian)h(structure)179 1073 y(\(grid/torus\).)j(By)d(assigning)h(neigh)o(b)q(oring)g(p)q (ositions)g(in)f(the)g(logical)h(pro)q(cess)f(torus)g(to)f(neigh-)179 1129 y(b)q(oring)e(pro)q(cessors)g(in)h(the)f(hardw)o(are)f(the)h (system)g(can)g(optimize)h(the)f(lo)q(calit)o(y)h(of)f(in)o(ter{pro)q (cess)179 1186 y(comm)o(unications.)20 b(The)15 b(n)o(um)o(b)q(er)f(of) g(dimensions)i(is)f(sp)q(eci\014ed)h(b)o(y)e Fl(ndim)p Fs(.)19 b(Arra)o(y)14 b Fl(period)f Fs(sp)q(ec-)179 1242 y(i\014es)18 b(the)g(p)q(erio)q(dicit)o(y)i(in)e(eac)o(h)f(co)q (ordinate)h(direction.)29 b(If)17 b Fl(period\(i\))g Fs(is)h(set)f(to)g Ft(MPI)p 1751 1242 16 2 v 18 w(PER)179 1299 y Fs(then)j(dimension)h Fl(i)e Fs(is)h(treated)f(as)g(p)q(erio)q (dic.)34 b(If)20 b Fl(period\(i\))e Fs(is)i(set)f(to)g Ft(MPI)p 1602 1299 V 18 w(NPER)h Fs(then)179 1355 y(dimension)e Fl(i)d Fs(is)i(treated)f(as)f(non-p)q(erio)q(dic.)25 b(If)17 b(all)g Fl(period\(i\))e Fs(are)g(set)h(to)g Ft(MPI)p 1634 1355 V 18 w(PER)p Fs(,)g(then)179 1412 y(the)f(top)q(ology)g(is)h(a)f(torus.)179 1487 y(Note)c(that)h(a)g (n-dimension)i(h)o(yp)q(ercub)q(e)f(is)g(a)f(torus)f(with)h(2)g(pro)q (cesses)h(p)q(er)f(co)q(ordinate)h(direction)179 1543 y(and)i(MPI)p 359 1543 14 2 v 17 w(PER.)g(Th)o(us,)f(ndim=n,)j (dims\(1:n\)=2)e(and)g(p)q(erio)q(d\(1:n\)=MPI)p 1513 1543 V 17 w(PER.)75 1641 y Ft(RETURN)k(V)-6 b(ALUE)179 1698 y Fs(If)14 b(the)g(calling)h(pro)q(cess)f(is)g(assigned)h(a)e(p)q (osition)i(in)f(the)g(grid,)g Ft(MPI)p 1372 1698 16 2 v 18 w(CAR)l(T)f Fs(returns)h(the)g(total)179 1754 y(n)o(um)o(b)q(er)k (of)f(pro)q(cesses)h(in)h(the)f(grid.)28 b(If)18 b(the)g(calling)h(pro) q(cess)f(is)h(not)e(in)i(the)f(grid,)g(the)g(return)179 1811 y(v)m(alue)e(is)g(set)f(to)f(0.)20 b(A)15 b(v)m(alue)i(of)d(-1)h (is)h(returned)g(if)f(an)h(error)e(o)q(ccurs.)p eop %%Page: 114 119 114 118 bop 75 -100 a Fs(114)640 b Fq(APPENDIX)16 b(A.)30 b(PR)o(OCESS)16 b(TOPOLOGY)g(R)o(OUTINES)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(MAKDIM)29 b Fs(Divide)18 b(the)e(pro)q(cesses)h(of)e(a)h(pro)q(cess)h(group)f(among)f(n)i (dimensions)g(of)f(a)573 158 y(Cartesian)f(grid.)75 256 y Ft(SYNOPSIS)179 312 y Fs(in)o(teger)g(function)h(MPI)p 599 312 14 2 v 17 w(MAKDIM)f(\(gid,)g(ndim,)g(dims\))179 369 y(in)o(teger)g(gid)179 425 y(in)o(teger)g(ndim)179 482 y(in)o(teger)g(dims\()p Fo(\003)p Fs(\))75 580 y Ft(INPUT)i(AR)o(GUMENTS)179 636 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q(cess)h(group)f(to)f(b)q(e)i (structured.)179 693 y(ndim)294 b(the)15 b(n)o(um)o(b)q(er)h(of)f (dimensions)i(of)d(the)i(torus.)179 749 y(dims)301 b(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(in)g(eac)o(h)f(co)q(ordinate)h(direction.)75 847 y Ft(OUTPUT)j(AR)o(GUMENTS)179 904 y Fs(dims)301 b(n)o(um)o(b)q(er)16 b(of)e(pro)q(cesses)i(in)g(eac)o(h)f(co)q (ordinate)h(direction.)75 1002 y Ft(DESCRIPTION)179 1058 y Fs(The)11 b(pro)q(cesses)g(of)g(the)f(group)h(with)g(ID)g(n)o(um)o(b) q(er)g Fl(gid)g Fs(are)f(divided)j(as)e(ev)o(enly)h(as)e(p)q(ossible)j (among)179 1114 y Fl(ndim)k Fs(dimensions,)j(sub)s(ject)e(to)g(the)g (constrain)o(t)g(that)f(all)i(pro)q(cesses)g(in)g(the)f(group)g(are)g (used.)179 1171 y(The)e(caller)h(ma)o(y)e(further)h(constrain)g(the)g (op)q(eration)g(of)g(this)g(routine)h(b)o(y)e(sp)q(ecifying)j(elemen)o (ts)179 1227 y(of)c(arra)o(y)g Fl(dims)p Fs(.)20 b(If)15 b Fl(dims\(i\))f Fs(is)i(set)f(to)f(a)h(p)q(ositiv)o(e)h(n)o(um)o(b)q (er,)f(the)h(routine)f(will)i(\014x)e(the)h(n)o(um)o(b)q(er)179 1284 y(of)g(pro)q(cesses)g(in)i(co)q(ordinate)e(direction)i Fl(i)e Fs(to)g Fl(dims\(i\))p Fs(.)22 b(Pro)q(cesses)17 b(will)h(then)e(b)q(e)h(distributed)179 1340 y(among)d(directions)j (where)e Fl(dims\(i\))f Fs(is)i(set)f(to)g(0,)f(sub)s(ject)h(to)g(the)g (ab)q(o)o(v)o(e)g(constrain)o(ts.)179 1415 y(Note)j(that)h(an)g(error)f (will)j(o)q(ccur)e(if)g Fl(dims\(i\))f Fs(is)i(p)q(ositiv)o(e)g(and)f (do)q(es)g(not)g(ev)o(enly)h(divide)h(the)179 1472 y(n)o(um)o(b)q(er)d (of)g(pro)q(cesses)g(in)h(the)f(group.)28 b(Negativ)o(e)18 b(input)h(v)m(alues)g(of)f Fl(dims\(i\))f Fs(will)j(also)e(cause)179 1528 y(an)e(error.)22 b(All)17 b(elemen)o(ts)g(of)f Fl(dims\(*\))f Fs(m)o(ust)g(b)q(e)i(set)f(to)f(0)h(for)g(completely)h(automatic)f(pro) q(cess)179 1585 y(distribution.)179 1660 y(When)21 b(the)g(function)g (returns,)h Fl(dims\(i\))d Fs(will)k(b)q(e)e(set)f(to)g(the)h(n)o(um)o (b)q(er)g(of)f(pro)q(cesses)h(in)h(co-)179 1716 y(ordinate)h(direction) i Fl(i)p Fs(.)45 b(F)l(or)22 b Fl(dims\(i\))h Fs(not)g(set)g(b)o(y)g (the)h(user,)h Fl(dims\(i\))e Fs(will)i(b)q(e)f(ordered)179 1773 y(\(MAXIMUM,)18 b(...,)g(MINIMUM\).)g(Arra)o(y)g Fl(dim\(*\))g Fs(is)h(suitable)h(for)e(use)h(as)g(input)g(to)f(routine) 179 1829 y(MPI)p 271 1829 V 16 w(CAR)l(T.)75 1927 y Ft(RETURN)h(V)-6 b(ALUE)179 1984 y Fs(If)18 b(the)g(routine)g(completes)g(without)g (error,)f(the)h(return)f(v)m(alue)i(is)f(set)g(to)f(0.)27 b(A)18 b(v)m(alue)h(of)e(-1)g(is)179 2040 y(returned)e(if)h(an)f(error) g(o)q(ccurs.)p eop %%Page: 115 120 115 119 bop 1807 -100 a Fs(115)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(P)l(AR)l(TC)89 b Fs(P)o(artition)15 b(a)g(pro)q(cess)g(group)g(with)h(cartesian)f(structure)g(in)h (subgroups.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)f(function)h (MPI)p 599 256 14 2 v 17 w(P)l(AR)l(TC)f(\(gid,)g(lostdim\))179 312 y(in)o(teger)g(gid)179 369 y(in)o(teger)g(lostdim)75 467 y Ft(INPUT)i(AR)o(GUMENTS)179 523 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q(cess)h(group)f(to)f(b)q(e)i (partitioned.)179 580 y(lostdim)248 b(the)12 b(co)q(ordinate)g (direction)h(b)o(y)f(whic)o(h)g(the)g(pro)q(cess)g(map)f(will)j(b)q(e)e (reduced.)75 678 y Ft(DESCRIPTION)179 734 y Fs(It)19 b(is)h(assumed)f(that)g(the)g(pro)q(cesses)h(of)f(the)g(group)g(with)h (ID)f(n)o(um)o(b)q(er)h Fl(gid)e Fs(are)h(arranged)g(in)179 791 y(a)k(cartesian)h(structure)f(of)h(size)g(\(n1,n2,...\))43 b(b)o(y)24 b(a)f(previous)i(call)g(to)e(MPI)p 1580 791 V 16 w(CAR)l(T.)h(Then,)179 847 y(MPI)p 271 847 V 16 w(P)l(AR)l(TC)e(creates)f(a)f(n)o(um)o(b)q(er)i(of)f(subgroups)g(with)g (cartesian)h(structures,)f(the)h(dimen-)179 904 y(sion)c(of)f(whic)o(h) h(is)g(one)g(lo)o(w)o(er)f(than)h(its)g(paren)o(t's.)26 b(The)18 b(direction)g(of)g(splitting)h(of)e(the)g(paren)o(t)179 960 y(structure)g(is)h(sp)q(eci\014ed)i(b)o(y)e Fl(lostdim)p Fs(.)26 b(If,)18 b(for)f(example,)i Fl(lostdim)k(=)h(2)p Fs(,)17 b(then)i Fl(n2)e Fs(subgroups)179 1016 y(with)e(size)h (\(n1,n3,n4,...\))i(are)d(formed.)75 1114 y Ft(RETURN)k(V)-6 b(ALUE)179 1171 y Fs(Up)q(on)16 b(successful)h(completion)g Ft(MPI)p 852 1171 16 2 v 18 w(P)l(AR)l(TC)d Fs(returns)i(the)f(group)h (ID)g(n)o(um)o(b)q(er)f(of)h(the)f(sub-)179 1227 y(group)d(to)h(whic)o (h)g(the)g(calling)i(pro)q(cess)e(b)q(elongs.)20 b(A)13 b(v)m(alue)h(of)e(-1)h(is)g(returned)h(if)f(an)g(error)f(o)q(ccurs.)p eop %%Page: 116 121 116 120 bop 75 -100 a Fs(116)640 b Fq(APPENDIX)16 b(A.)30 b(PR)o(OCESS)16 b(TOPOLOGY)g(R)o(OUTINES)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(GRAPH)73 b Fs(Arrange)13 b(the)h(pro)q(cesses)g(of)g(a)f(pro)q(cess)h(group)f(in)i(a)e(general)i (graph)e(struc-)573 158 y(ture.)75 256 y Ft(SYNOPSIS)179 312 y Fs(in)o(teger)i(function)h(MPI)p 599 312 14 2 v 17 w(GRAPH)f(\(gid,)g(no)q(des,)g(index,)i(nglist\))179 369 y(in)o(teger)e(gid)179 425 y(in)o(teger)g(no)q(des)179 482 y(in)o(teger)g(index\()p Fo(\003)p Fs(\))179 538 y(in)o(teger)g(nglist\()p Fo(\003)p Fs(\))75 636 y Ft(INPUT)i(AR)o (GUMENTS)179 693 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g (the)g(pro)q(cess)h(group)f(to)f(b)q(e)i(structured.)179 749 y(no)q(des)282 b(the)15 b(n)o(um)o(b)q(er)h(of)f(no)q(des)g(in)i (the)e(graph.)179 806 y(index)288 b(index)17 b(v)o(ector)d(in)o(to)h(v) o(ector)g(nglist)h(\(see)f(b)q(elo)o(w\).)179 862 y(nglist)286 b(for)15 b(eac)o(h)g(graph)g(no)q(de:)20 b(list)c(of)f(neigh)o(b)q (oring)i(no)q(des)f(\(see)f(b)q(elo)o(w\).)75 960 y Ft(DESCRIPTION)179 1016 y Fs(The)k(pro)q(cesses)g(of)f(the)g(group)h(with)g(ID)f(n)o(um)o (b)q(er)h Fl(gid)f Fs(are)h(arranged)f(in)h(a)f(graph)h(structure.)179 1073 y(By)d(assigning)h(neigh)o(b)q(oring)h(graph)e(no)q(des)h(to)f (neigh)o(b)q(oring)i(pro)q(cessors)e(in)h(the)f(hardw)o(are)g(the)179 1129 y(system)k(can)h(optimize)h(the)e(lo)q(calit)o(y)i(of)f(in)o (ter{pro)q(cess)f(comm)o(unications.)37 b(The)21 b(n)o(um)o(b)q(er)g (of)179 1186 y(no)q(des)16 b(is)f(sp)q(eci\014ed)j(b)o(y)d Fl(nodes)p Fs(.)179 1242 y(Input)i(to)e(MPI)p 453 1242 V 16 w(GRAPH)h(is)h(for)e(eac)o(h)h(no)q(de)h(its)f(set)g(of)f(neigh)o (b)q(ors,)i(i.)f(e.,)f(the)h(set)g(of)g(pro)q(cesses)179 1299 y(with)g(whic)o(h)h(the)f(no)q(de)h(pro)q(cess)f(is)g(going)g(to)g (comm)o(unicate)g(most)f(frequen)o(tly)l(.)23 b(Since)17 b(there)f(is)179 1355 y(no)e(w)o(eigh)o(ting)h(of)f(no)q(de)h (connections,)g(it)g(migh)o(t)f(b)q(e)h(sensible)i(to)c(sp)q(ecify)j (only)f(those)f(neigh)o(b)q(ors)179 1412 y(with)h(whic)o(h)h(there)g (is)f(a)g(considerable)i(amoun)o(t)e(of)f(comm)o(unication.)179 1468 y(F)l(or)9 b(eac)o(h)i(no)q(de)f Fr(i)j Fs(\(1)f Fo(\024)h Fr(i)f Fo(\024)h Fr(nodes)p Fs(\))d(the)g(list)h(of)f(the)g (neigh)o(b)q(or)h(no)q(des)g(is)g(stored)e(at)h Fr(ng)r(l)q(ist)p Fs(\()p Fr(j)s Fs(\))p Fr(;)18 b(index)p Fs(\()p Fr(i)p Fs(\))11 b Fo(\024)179 1525 y Fr(j)k Fo(\024)e Fr(index)p Fs(\()p Fr(i)8 b Fs(+)h(1\))g Fo(\000)g Fs(1.)19 b(F)l(or)14 b(the)h(last)g(pro)q(cess)g Fr(index)p Fs(\()p Fr(i)8 b Fs(+)h(1\))14 b(p)q(oin)o(ts)h(to)f(the)h(\014rst)f(free)h(p)q (osition)179 1581 y(in)k(v)o(ector)f(nglist.)30 b(Note)18 b(that)g(the)g(matrix)g(is)h(symmetric,)g(i.)g(e.,)f(if)h(no)q(de)g Fh(i)f Fs(is)h(a)f(neigh)o(b)q(or)h(of)179 1637 y(no)q(de)g Fh(j)p Fs(,)g(than)g(no)q(de)h Fh(j)e Fs(is)i(also)e(a)h(neigh)o(b)q (or)h(of)e(no)q(de)h Fh(i)p Fs(.)31 b(This)19 b(prop)q(ert)o(y)g(can)g (b)q(e)g(used)h(for)e(an)179 1694 y(in)o(ternal)e(c)o(hec)o(king)g(of)f (the)g(input)h(argumen)o(ts.)75 1792 y Ft(RETURN)j(V)-6 b(ALUE)179 1848 y Fs(If)17 b(the)f(calling)j(pro)q(cess)e(is)g (assigned)g(a)g(p)q(osition)g(in)h(the)f(graph,)f Ft(MPI)p 1440 1848 16 2 v 18 w(GRAPH)g Fs(returns)h(its)179 1905 y(no)q(de)k(n)o(um)o(b)q(er.)35 b(If)21 b(it)f(is)h(not)f(in)h(the)f (graph,)h(a)f(v)m(alue)i(of)d(0)h(is)h(returned.)36 b(A)20 b(v)m(alue)i(of)d(-1)h(is)179 1961 y(returned)15 b(if)h(an)f(error)g(o) q(ccurs.)p eop %%Page: 117 122 117 121 bop 1807 -100 a Fs(117)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(INQMAP)44 b Fs(Find)16 b(out)f(mapping)h(t) o(yp)q(e)f(of)g(a)g(pro)q(cess)g(group.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)g(function)h(MPI)p 599 256 14 2 v 17 w(INQMAP)f(\(gid\))75 354 y Ft(INPUT)i(AR)o(GUMENTS)179 410 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q (cess)h(group.)75 508 y Ft(DESCRIPTION)179 565 y Fs(The)h(function)h (MPI)p 546 565 V 16 w(INQMAP)f(tells)h(the)f(calling)i(pro)q(cess)e (the)g(t)o(yp)q(e)f(of)h(mapping)g(whic)o(h)h(has)179 621 y(b)q(een)13 b(applied)g(to)e(the)h(pro)q(cess)g(group)f(with)h (the)g(ID)g(n)o(um)o(b)q(er)g Fl(gid)p Fs(,)g(of)f(whic)o(h)h(the)g (calling)i(pro)q(cess)179 678 y(is)h(a)g(mem)o(b)q(er.)75 776 y Ft(RETURN)k(V)-6 b(ALUE)179 832 y Fs(Up)q(on)16 b(successful)g(completion)g Ft(MPI)p 850 832 16 2 v 18 w(INQMAP)e Fs(returns)204 885 y(MPI)p 296 885 14 2 v 16 w(NOMAP:)148 b(if)16 b(no)f(mapping)h(has)f(b)q(een)h(assigned,)204 941 y(MPI)p 296 941 V 16 w(CAR)l(TMAP:)88 b(in)16 b(case)g(of)e(a)h (cartesian)h(mapping,)f(and)204 998 y(MPI)p 296 998 V 16 w(GRAPHMAP:)49 b(in)16 b(case)g(of)e(a)h(graph)g(mapping.)179 1050 y(A)g(v)m(alue)h(of)f(-1)g(is)h(returned)g(if)f(an)g(error)g(o)q (ccurs.)p eop %%Page: 118 123 118 122 bop 75 -100 a Fs(118)640 b Fq(APPENDIX)16 b(A.)30 b(PR)o(OCESS)16 b(TOPOLOGY)g(R)o(OUTINES)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(INQCAR)l(T)21 b Fs(Lo)q(ok)15 b(up)h(cartesian)f(structure)g(and)h(o)o(wn)e(p)q(osition.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)h(function)h(MPI)p 599 256 14 2 v 17 w(INQCAR)l(T)g(\(gid,)f(maxdim,)g(size,)h(p)q(erio)q (d,)g(p)q(os\))179 312 y(in)o(teger)f(gid)179 369 y(in)o(teger)g (maxdim)179 425 y(in)o(teger)g(size\()p Fo(\003)p Fs(\))179 482 y(in)o(teger)g(p)q(erio)q(d\()p Fo(\003)p Fs(\))179 538 y(in)o(teger)g(p)q(os\()p Fo(\003)p Fs(\))75 636 y Ft(INPUT)i(AR)o(GUMENTS)179 693 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q(cess)h(group)f(with)g(a)g (cartesian)g(structure.)179 749 y(maxdim)234 b(the)15 b(dimension)i(of)e(v)o(ectors)g Fl(size)f Fs(and)h Fl(pos)g Fs(in)h(the)g(calling)g(program.)75 847 y Ft(OUTPUT)j(AR)o(GUMENTS)179 904 y Fs(size)324 b(v)o(ector)15 b(returning)g(the)h(size)g(of)f(the)g (grid)g(in)h(eac)o(h)g(co)q(ordinate)f(direction.)179 960 y(p)q(erio)q(d)269 b(p)q(erio)q(dicit)o(y)18 b(c)o(haracteristics)d (in)h(eac)o(h)f(co)q(ordinate)h(direction.)179 1016 y(p)q(os)327 b(v)o(ector)15 b(returning)g(the)h(p)q(osition)g(of)f(the)g(calling)i (pro)q(cess)e(in)h(the)g(grid.)75 1114 y Ft(DESCRIPTION)179 1171 y Fs(After)11 b(the)i(pro)q(cesses)f(of)g(the)g(group)g Fl(gid)f Fs(ha)o(v)o(e)h(b)q(een)h(assigned)g(a)e(cartesian)i (structure)e(\(function)179 1227 y(MPI)p 271 1227 V 16 w(CAR)l(T\),)k(the)g(function)g(MPI)p 806 1227 V 17 w(INQCAR)l(T)h (tells)g(the)f(calling)h(pro)q(cess)f(the)g(parameters)f(of)179 1284 y(that)g(structure)h(and)h(its)f(o)o(wn)g(logical)h(p)q(osition.) 179 1359 y(If,)d(for)f(example,)h(a)f(three{dimensional)j(grid)e (mapping)g(has)f(b)q(een)i(applied,)g(then)f Fl(size)f Fs(returns)179 1415 y(the)f(three)h(v)m(alues)h Fl(nx,ny,nz)p Fs(,)e(and)g Fl(period\(i\))p Fs(,)g(i=1,2,3,)g(is)h(set)g(to)f Ft(MPI)p 1475 1415 16 2 v 18 w(NPER)p Fs(.)g(The)h(v)o(ector)179 1472 y Fl(pos)j Fs(con)o(tains)h(the)g(logical)h(pro)q(cess)f(co)q (ordinates)g Fl(i,j,k)f Fs(with)i(1)c Fo(\024)h Fr(i)g Fo(\024)g Fr(nx)p Fs(,)h(1)f Fo(\024)g Fr(j)i Fo(\024)e Fr(ny)r Fs(,)i(and)179 1528 y(1)j Fo(\024)g Fr(k)h Fo(\024)f Fr(nz)r Fs(.)32 b(If)19 b(the)h(calling)g(pro)q(cess)g(is)f(not)g (assigned)g(a)g(p)q(osition)h(in)g(the)f(grid)h(structure,)179 1585 y(the)c(co)q(ordinates)h(are)e(set)h(to)g(zero)q(es.)23 b(If)16 b(the)g(v)o(ectors)g Fl(size)p Fs(,)f Fl(period)h Fs(and)g Fl(pos)g Fs(are)f(to)q(o)h(short,)179 1641 y(MPI)p 271 1641 14 2 v 16 w(INQCAR)l(T)h(rep)q(orts)d(an)i(error.)75 1739 y Ft(RETURN)j(V)-6 b(ALUE)179 1796 y Fs(Up)q(on)18 b(successful)h(completion)g Ft(MPI)p 858 1796 16 2 v 18 w(INQCAR)l(T)f Fs(returns)f(the)h(n)o(um)o(b)q(er)g(of)g(dimensions) h(of)179 1852 y(the)c(cartesian)g(structure,)g Fl(ndim)p Fs(.)k(A)c(v)m(alue)i(of)e(-1)g(is)g(returned)h(if)g(an)f(error)f(o)q (ccurs.)p eop %%Page: 119 124 119 123 bop 1807 -100 a Fs(119)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(INQGR)90 b Fs(Lo)q(ok)15 b(up)h(size)g(of)f(graph)g(mapping)h(and)f(p)q(osition)h(of)f(calling)i (pro)q(cess.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)e(function)h (MPI)p 599 256 14 2 v 17 w(INQGR)g(\(gid,)f(no)q(des\))75 354 y Ft(INPUT)i(AR)o(GUMENTS)179 410 y Fs(gid)334 b(the)15 b(ID)h(n)o(um)o(b)q(er)f(of)g(the)g(pro)q(cess)h(group)f(with)g(a)g (graph)g(structure.)75 508 y Ft(OUTPUT)k(AR)o(GUMENTS)179 565 y Fs(no)q(des)282 b(the)15 b(total)g(n)o(um)o(b)q(er)h(of)e(no)q (des)i(in)g(the)f(graph)g(structure.)75 663 y Ft(DESCRIPTION)179 719 y Fs(After)24 b(the)h(pro)q(cesses)f(of)h(the)f(group)g Fl(gid)h Fs(ha)o(v)o(e)f(b)q(een)h(structured)g(as)f(a)g(graph)g (\(function)179 776 y(MPI)p 271 776 V 16 w(GRAPH\),)f(the)g(function)h (MPI)p 870 776 V 17 w(INQGR)g(tells)g(the)g(calling)h(pro)q(cess)e(the) h(size)g(of)f(the)179 832 y(graph)c(and)h(its)h(o)o(wn)e(no)q(de)i(n)o (um)o(b)q(er.)34 b(No)q(des)20 b(are)g(n)o(um)o(b)q(ered)g(from)g(1)f (to)g Fl(nodes)h Fs(\(cf.)33 b(func-)179 889 y(tion)15 b(MPI)p 364 889 V 17 w(GRAPH\).)75 987 y Ft(RETURN)k(V)-6 b(ALUE)179 1043 y Fs(Up)q(on)16 b(successful)h(completion)g Ft(MPI)p 852 1043 16 2 v 18 w(INQGR)f Fs(returns)f(the)h(no)q(de)h(n)o (um)o(b)q(er)f Fl(i)f Fs(of)g(the)h(calling)179 1099 y(pro)q(cess,)e(1)e Fo(\024)h Fr(i)f Fo(\024)h Fr(nodes)p Fs(.)20 b(If)14 b(the)g(calling)i(pro)q(cess)e(is)h(not)e(assigned)i(a) e(no)q(de)i(in)g(the)f(graph,)f(zero)179 1156 y(is)i(returned.)21 b(A)15 b(v)m(alue)h(of)f(-1)g(is)h(returned)g(if)f(an)g(error)g(o)q (ccurs.)p eop %%Page: 120 125 120 124 bop 75 -100 a Fs(120)640 b Fq(APPENDIX)16 b(A.)30 b(PR)o(OCESS)16 b(TOPOLOGY)g(R)o(OUTINES)75 45 y Ft(NAME)179 102 y(MPI)p 288 102 16 2 v 18 w(INQPID)74 b Fs(Lo)q(ok)15 b(up)h(PID)f(n)o(um)o(b)q(ers)h(of)f(selected)h(pro)q(cesses.)75 200 y Ft(SYNOPSIS)179 256 y Fs(in)o(teger)f(function)h(MPI)p 599 256 14 2 v 17 w(INQPID)g(\(gid,)f(npro)q(c,)g(co)q(ord,)g(plist\)) 179 312 y(in)o(teger)g(gid)179 369 y(in)o(teger)g(npro)q(c)179 425 y(in)o(teger)g(co)q(ord\()p Fo(\003)p Fs(\))179 482 y(in)o(teger)g(plist\()p Fo(\003)p Fs(\))75 580 y Ft(INPUT)i(AR)o (GUMENTS)179 636 y Fs(gid)334 b(the)22 b(ID)g(n)o(um)o(b)q(er)g(of)f (the)h(pro)q(cess)g(group)f(with)h(a)f(cartesian)h(or)f(graph)573 693 y(structure.)179 749 y(npro)q(c)282 b(the)22 b(n)o(um)o(b)q(er)g (of)g(pro)q(cesses)g(for)f(whic)o(h)i(the)f(PID)f(n)o(um)o(b)q(ers)i (are)e(to)g(b)q(e)573 806 y(lo)q(ok)o(ed)16 b(up)f(\(see)h(b)q(elo)o (w\).)179 862 y(co)q(ord)284 b(the)42 b(logical)h(pro)q(cess)g(co)q (ordinates)f(of)f(the)h(sp)q(eci\014ed)j(pro)q(cesses)573 918 y(\(see)15 b(b)q(elo)o(w\).)75 1016 y Ft(OUTPUT)k(AR)o(GUMENTS)179 1073 y Fs(plist)309 b(the)15 b(list)h(of)f(PID)g(n)o(um)o(b)q(ers)h(of) f(the)g(sp)q(eci\014ed)i(pro)q(cesses.)75 1171 y Ft(DESCRIPTION)179 1227 y Fs(Usually)c(a)f(pro)q(cess)h(only)g(comm)o(unicates)f(with)h(a) f(relativ)o(ely)h(small)g(n)o(um)o(b)q(er)g(of)f(other)g(pro)q(cesses) 179 1284 y(in)23 b(a)f(large)g(pro)q(cess)g(structure)g(\(a)f (cartesian)i(structure)f(or)f(a)h(graph\).)40 b(Giv)o(en)23 b(the)f(logical)179 1340 y(co)q(ordinates)d(of)f(those)g(other)g(pro)q (cesses,)h(MPI)p 1028 1340 V 16 w(INQPID)h(lo)q(oks)e(up)h(the)g (corresp)q(onding)g(PID)179 1397 y(n)o(um)o(b)q(ers.)h(Those)15 b(PID)g(n)o(um)o(b)q(ers)h(can)f(then)h(b)q(e)f(used)h(for)f(the)g (message{passing.)179 1472 y(The)f(pro)q(cesses)h(of)f(whic)o(h)h(the)f (PID)g(n)o(um)o(b)q(ers)g(are)g(required)i(are)d(sp)q(eci\014ed)k(b)o (y)d(the)g(input)h(argu-)179 1528 y(men)o(ts)g Fl(nproc)f Fs(and)h Fl(coord)p Fs(.)20 b(Tw)o(o)14 b(mec)o(hanisms)i(are)f(a)o(v)m (ailable:)243 1622 y(1.Separately)33 b(sp)q(eci\014ed)i(pro)q(cesses:) 55 b Fl(nproc)31 b Fs(is)j(set)e(to)g(the)g(n)o(um)o(b)q(er)h(of)g(pro) q(cesses)279 1679 y(1)18 b Fo(\024)g Fr(npr)q(oc)p Fs(.)30 b(V)l(ector)18 b Fl(coord)g Fs(con)o(tains)h(the)g(logical)g(co)q (ordinates)g(for)f(eac)o(h)h(pro)q(cess.)30 b(In)279 1735 y(p)q(erio)q(dic)22 b(directions)f(the)f(co)q(ordinates)g(are)f (in)o(terpreted)h(mo)q(dulo)h(the)f(n)o(um)o(b)q(er)g(of)f(pro-)279 1791 y(cesses)f(in)h(the)f(corresp)q(onding)h(direction)g(\(torus)e (top)q(ology\).)28 b(As)18 b(an)g(example,)h(assume)279 1848 y(the)e(pro)q(cesses)g(are)g(mapp)q(ed)h(as)e(a)h(t)o(w)o (o{dimensional)g(torus,)g(and)g(the)g(PID)g(n)o(um)o(b)q(ers)g(of)279 1904 y(three)j(pro)q(cesses)h(are)f(to)f(b)q(e)i(lo)q(ok)o(ed)g(up.)35 b(Then,)21 b Fl(nproc)f Fs(is)h(set)f(to)f(3,)i(and)f(the)g(\014rst)g (6)279 1961 y(en)o(tries)15 b(of)g Fl(coord)g Fs(are)f(set)h(to)g Fr(i)829 1968 y Fa(1)848 1961 y Fr(;)8 b(j)888 1968 y Fa(1)907 1961 y Fr(;)g(i)944 1968 y Fa(2)963 1961 y Fr(;)g(j)1003 1968 y Fa(2)1021 1961 y Fr(;)g(i)1058 1968 y Fa(3)1077 1961 y Fr(;)g(j)1117 1968 y Fa(3)1135 1961 y Fs(.)279 2026 y(Similarly)l(,)14 b(if)e(the)f(mapping)h(w)o(as)f(done)h(with)f (MPI)p 1153 2026 V 17 w(GRAPH,)g(the)g(en)o(tries)h(of)f(v)o(ector)g Fl(coord)279 2082 y Fs(are)k(set)g(to)f(the)h(logical)i(no)q(de)f(n)o (um)o(b)q(ers)f(of)g(the)g(sp)q(eci\014ed)j(graph)c(pro)q(cesses.)243 2155 y(2.No)30 b(individual)j(selection:)52 b(in)31 b(this)g(case)f Fl(nproc)g Fs(is)g(set)h(to)e(a)h(negativ)o(e)h(n)o(um)o(b)q(er)279 2212 y Fr(npr)q(oc)17 b Fo(\024)h(\000)p Fs(1.)29 b(V)l(ector)18 b Fl(plist)g Fs(returns)g(the)g(PID)h(n)o(um)o(b)q(ers)f(of)g(the)g Fh(\014rst)g Fl(|nproc|)f Fs(pro-)279 2268 y(cesses.)30 b(The)19 b(v)o(ector)f Fl(coord)g Fs(is)h(not)f(used)h(in)h(this)f (case.)30 b(Note)18 b(that)g(this)h(feature)g(do)q(es)279 2324 y(not)13 b(scale)i(to)f(v)o(ery)f(high)i(pro)q(cess)g(n)o(um)o(b)q (ers.)k(In)c(small)g(applications,)h(ho)o(w)o(ev)o(er,)d(it)h(can)g(b)q (e)279 2381 y(used)i(to)e(get)h(a)g(global)h(list)g(of)e(all)j(PID)e(n) o(um)o(b)q(ers)g(in)h(the)g(map.)75 2516 y Ft(RETURN)j(V)-6 b(ALUE)179 2573 y Fs(Up)q(on)15 b(successful)h(completion)g Ft(MPI)p 849 2573 16 2 v 18 w(INQPID)e Fs(returns)h(0.)20 b(A)14 b(v)m(alue)i(of)f(-1)f(is)i(returned)f(if)g(an)179 2629 y(error)f(o)q(ccurs.)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .