PTO-TILE-MODEL-MEMORY-GATHER-SCATTER

1// PTO-UNIT: {"id":"PTO-TILE-MODEL-MEMORY-GATHER-SCATTER","surface":"tile","classification":["model","memory","gather-scatter"],"depends_on":["PTO-TILE-MODEL-MEMORY-LOAD-STORE"]}2func MGATHER(destination: TileIndex, base_address: Word, indices: TileIndex,3             pad_value: TilePadValue)4begin5    let destination_tile = _Tiles[[destination]];6    let index_tile = _Tiles[[indices]];7    assert destination_tile.allocated;8    assert index_tile.allocated && index_tile.contents_defined;9    assert destination_tile.valid_rows == index_tile.valid_rows;10    assert destination_tile.valid_columns == index_tile.valid_columns;11    assert IndexedTLSUIndexDataTypeLegal(index_tile.data_type);12    assert IndexedTLSUTransferDataTypeLegal(destination_tile.data_type);13    let index_payload = index_tile.payload;14    var translated_addresses: TilePayload;15    var result_payload = destination_tile.payload;16    for row = 0 to destination_tile.valid_rows - 1 looplimit 65536 do17        for column = 0 to destination_tile.valid_columns - 1 looplimit 65536 do18            let destination_element = TileLinearIndex(destination_tile,19                row as integer {0..65535}, column as integer {0..65535});20            let index_element = TileLinearIndex(index_tile,21                row as integer {0..65535}, column as integer {0..65535});22            let address = TileMemoryByteDisplacementAddress(base_address,23                index_payload[[index_element]], index_tile.data_type);24            let probe = ProbeTileMemoryAccess(address,25                destination_tile.data_type, FALSE);26            if RaiseDataAccessFault(probe, address) then return; end;27            translated_addresses[[destination_element]] =28                probe.translated_address;29        end;30    end;31    for row = 0 to destination_tile.rows - 1 looplimit 65536 do32        for column = 0 to destination_tile.columns - 1 looplimit 65536 do33            let element = TileLinearIndex(destination_tile,34                row as integer {0..65535}, column as integer {0..65535});35            result_payload[[element]] = TilePadValueForDataType(36                pad_value, destination_tile.data_type);37        end;38    end;39    for row = 0 to destination_tile.valid_rows - 1 looplimit 65536 do40        for column = 0 to destination_tile.valid_columns - 1 looplimit 65536 do41            let element = TileLinearIndex(destination_tile,42                row as integer {0..65535}, column as integer {0..65535});43            let raw = LoadTranslatedUnsigned(translated_addresses[[element]],44                TileMemoryElementBytes(destination_tile.data_type));45            RecordLoadEvent(translated_addresses[[element]],46                TileMemoryElementBytes(destination_tile.data_type), raw,47                CurrentBundleMemoryOrder());48            result_payload[[element]] = DecodeTileMemoryElementRaw(49                raw, destination_tile.data_type, FALSE);50        end;51    end;52    _Tiles[[destination]].payload = result_payload;53    MarkTilePhysicalRegionDefined(destination);54end;55
56func MGATHER(destination: TileIndex, base_address: Word, indices: TileIndex)57begin58    MGATHER(destination, base_address, indices, TilePad_Null);59end;60
61func CommitScatterLanes(source_tile: TileInfo,62                        source_payload: TilePayload,63                        lane_order: ScatterLaneOrder,64                        lane_count: integer {0..PTO_MODEL_TILE_ELEMENTS},65                        original_addresses: TilePayload,66                        translated_addresses: TilePayload,67                        high_nibbles: bits(PTO_MODEL_TILE_ELEMENTS))68begin69    // Duplicate-address lanes have an implementation-defined winner.  B.CATR70    // atomic makes the whole block non-interleavable, but does not select an71    // internal lane order or a duplicate-address winner.72    var commit_order = lane_order;73    if lane_count > 0 then74        for position = 0 to lane_count - 175            looplimit PTO_MODEL_TILE_ELEMENTS do76            var selected_position:77                integer {0..PTO_MODEL_TILE_ELEMENTS-1} =78                    position as integer {0..PTO_MODEL_TILE_ELEMENTS-1};79            var selected = FALSE;80            for candidate_position = position to lane_count - 181                looplimit PTO_MODEL_TILE_ELEMENTS do82                if !selected then83                    if ARBITRARY: boolean then84                        selected_position = candidate_position as85                            integer {0..PTO_MODEL_TILE_ELEMENTS-1};86                        selected = TRUE;87                    end;88                end;89            end;90            let selected_element = commit_order[[selected_position]];91            commit_order[[selected_position]] = commit_order[[position]];92            commit_order[[position]] = selected_element;93            let element = UInt(commit_order[[position]]) as94                ModelTileElementIndex;95            let stored_value = StoreTileMemoryElement(96                original_addresses[[element]], translated_addresses[[element]],97                source_tile.data_type, high_nibbles[element] == '1',98                source_payload[[element]]);99            RecordStoreEvent(translated_addresses[[element]],100                TileMemoryElementBytes(source_tile.data_type), stored_value,101                CurrentBundleMemoryOrder());102        end;103    end;104end;105
106func MSCATTER(base_address: Word, source: TileIndex, indices: TileIndex)107begin108    let source_tile = _Tiles[[source]];109    let index_tile = _Tiles[[indices]];110    assert source_tile.allocated && source_tile.contents_defined;111    assert index_tile.allocated && index_tile.contents_defined;112    assert source_tile.valid_rows == index_tile.valid_rows;113    assert source_tile.valid_columns == index_tile.valid_columns;114    assert source_tile.layout == index_tile.layout;115    assert IndexedTLSUIndexDataTypeLegal(index_tile.data_type);116    assert IndexedTLSUTransferDataTypeLegal(source_tile.data_type);117    let source_payload = source_tile.payload;118    let index_payload = index_tile.payload;119    var lane_order: ScatterLaneOrder;120    var lane_count: integer {0..PTO_MODEL_TILE_ELEMENTS} = 0;121    var original_addresses: TilePayload;122    var translated_addresses: TilePayload;123    var high_nibbles: bits(PTO_MODEL_TILE_ELEMENTS);124    for row = 0 to source_tile.valid_rows - 1 looplimit 65536 do125        for column = 0 to source_tile.valid_columns - 1 looplimit 65536 do126            let source_element = TileLinearIndex(source_tile,127                row as integer {0..65535}, column as integer {0..65535});128            let index_element = TileLinearIndex(index_tile,129                row as integer {0..65535}, column as integer {0..65535});130            let address = TileMemoryByteDisplacementAddress(base_address,131                index_payload[[index_element]], index_tile.data_type);132            let probe = ProbeTileMemoryAccess(address,133                source_tile.data_type, TRUE);134            if RaiseDataAccessFault(probe, address) then return; end;135            original_addresses[[source_element]] = address;136            translated_addresses[[source_element]] = probe.translated_address;137            high_nibbles[source_element] = '0';138            lane_order[[lane_count]] =139                NaturalToWord(source_element as integer {0..262144});140            lane_count = (lane_count + 1) as141                integer {0..PTO_MODEL_TILE_ELEMENTS};142        end;143    end;144    CommitScatterLanes(source_tile, source_payload, lane_order, lane_count,145        original_addresses, translated_addresses, high_nibbles);146end;147
148type CorePEPrefetchAddresses of array [[PTO_MODEL_MEMORY_AGENTS]] of TilePayload;149
150func TPREFETCHCore(base_addresses: CorePEWords,151                   row_stride_elements: CorePEWords,152                   valid_columns: integer {1..65535},153                   valid_rows: integer {1..65535},154                   columns: integer {1..65535},155                   data_type: TileDataType)156begin157    assert valid_columns <= columns;158    assert IsNonzeroPowerOfTwo(columns);159    assert valid_rows * valid_columns <= PTO_MODEL_TILE_ELEMENTS;160    var translated_addresses: CorePEPrefetchAddresses;161    // TPREFETCH is one four-PE block attempt.  Probe every typed element of162    // every PE before recording the first event so a fault cannot expose a163    // partial request or event prefix from an earlier PE.164    for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do165        let agent = pe as MemoryAgentId;166        for row = 0 to valid_rows - 1 looplimit 65536 do167            for column = 0 to valid_columns - 1 looplimit 65536 do168                let element = (row * valid_columns + column) as169                    ModelTileElementIndex;170                let memory_index = TileMemoryStridedIndex(171                    row as integer {0..65535},172                    column as integer {0..65535},173                    row_stride_elements[[agent]]);174                let address = TileMemoryIndexedAddress(175                    base_addresses[[agent]], memory_index, data_type);176                let probe = ProbeTileMemoryAccess(address, data_type, FALSE);177                if RaiseDataAccessFault(probe, address) then return; end;178                translated_addresses[[agent]][[element]] =179                    probe.translated_address;180            end;181        end;182    end;183    for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do184        let agent = pe as MemoryAgentId;185        for row = 0 to valid_rows - 1 looplimit 65536 do186            for column = 0 to valid_columns - 1 looplimit 65536 do187                let element = (row * valid_columns + column) as188                    ModelTileElementIndex;189                let translated_address =190                    translated_addresses[[agent]][[element]];191                let value = LoadTranslatedUnsigned(translated_address,192                    TileMemoryElementBytes(data_type));193                RecordLoadEventForAgent(agent, translated_address,194                    TileMemoryElementBytes(data_type), value,195                    CurrentBundleMemoryOrder());196            end;197        end;198    end;199end;200
201// The generated direct-operation dispatcher carries one decoded base and202// stride value.  Its wrapper applies those values to all four PEs; complete203// architectural bundles use ExecuteBundleTPREFETCHOperation below to read the204// same selectors independently from each PE-private GPR file.205func TPREFETCHAllPEs(base_address: Word, row_stride_elements: Word,206                     valid_columns: integer {1..65535},207                     valid_rows: integer {1..65535},208                     columns: integer {1..65535},209                     data_type: TileDataType)210begin211    var base_addresses: CorePEWords;212    var row_strides: CorePEWords;213    for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do214        let agent = pe as MemoryAgentId;215        base_addresses[[agent]] = base_address;216        row_strides[[agent]] = row_stride_elements;217    end;218    TPREFETCHCore(base_addresses, row_strides, valid_columns, valid_rows,219        columns, data_type);220end;221
222func TPREFETCH(base_address: Word, row_stride_elements: Word,223               valid_columns: integer {1..65535},224               valid_rows: integer {1..65535},225               columns: integer {1..65535})226begin227    TPREFETCHAllPEs(base_address, row_stride_elements, valid_columns,228        valid_rows, columns, TileDataTypeFromEncoding(229            CurrentBundleTileOperationDataTypeCode()230                as TileDataTypeEncoding));231end;232
233func MGATHER_MASK(destination: TileIndex, base_address: Word, indices: TileIndex,234                  mask: TileIndex, pad_value: TilePadValue)235begin236    let destination_tile = _Tiles[[destination]];237    let index_tile = _Tiles[[indices]];238    let mask_tile = _Tiles[[mask]];239    assert destination_tile.allocated;240    assert index_tile.allocated && index_tile.contents_defined;241    assert mask_tile.allocated && mask_tile.contents_defined;242    assert destination_tile.valid_rows == index_tile.valid_rows;243    assert destination_tile.valid_columns == index_tile.valid_columns;244    assert destination_tile.valid_rows == mask_tile.valid_rows;245    assert destination_tile.valid_columns == mask_tile.valid_columns;246    assert destination_tile.layout == index_tile.layout;247    assert destination_tile.layout == mask_tile.layout;248    assert IndexedTLSUIndexDataTypeLegal(index_tile.data_type);249    assert IndexedTLSUTransferDataTypeLegal(destination_tile.data_type);250    assert TilePredicateValuesLegal(mask);251    let index_payload = index_tile.payload;252    var translated_addresses: TilePayload;253    var active_lanes: bits(PTO_MODEL_TILE_ELEMENTS) =254        Zeros{PTO_MODEL_TILE_ELEMENTS};255    var result_payload = destination_tile.payload;256    for row = 0 to destination_tile.valid_rows - 1 looplimit 65536 do257        for column = 0 to destination_tile.valid_columns - 1 looplimit 65536 do258            let destination_element = TileLinearIndex(destination_tile,259                row as integer {0..65535}, column as integer {0..65535});260            let index_element = TileLinearIndex(index_tile,261                row as integer {0..65535}, column as integer {0..65535});262            if ReadTilePredicateBit(263                mask,264                row as integer {0..65535},265                column as integer {0..65535}) then266                let address = TileMemoryByteDisplacementAddress(base_address,267                    index_payload[[index_element]], index_tile.data_type);268                let probe = ProbeTileMemoryAccess(address,269                    destination_tile.data_type, FALSE);270                if RaiseDataAccessFault(probe, address) then return; end;271                translated_addresses[[destination_element]] =272                    probe.translated_address;273                active_lanes[destination_element] = '1';274            end;275        end;276    end;277    for row = 0 to destination_tile.rows - 1 looplimit 65536 do278        for column = 0 to destination_tile.columns - 1 looplimit 65536 do279            let element = TileLinearIndex(destination_tile,280                row as integer {0..65535}, column as integer {0..65535});281            result_payload[[element]] = TilePadValueForDataType(282                pad_value, destination_tile.data_type);283        end;284    end;285    for row = 0 to destination_tile.valid_rows - 1 looplimit 65536 do286        for column = 0 to destination_tile.valid_columns - 1 looplimit 65536 do287            let element = TileLinearIndex(destination_tile,288                row as integer {0..65535}, column as integer {0..65535});289            if active_lanes[element] == '1' then290                let raw = LoadTranslatedUnsigned(translated_addresses[[element]],291                    TileMemoryElementBytes(destination_tile.data_type));292                RecordLoadEvent(translated_addresses[[element]],293                    TileMemoryElementBytes(destination_tile.data_type), raw,294                    CurrentBundleMemoryOrder());295                result_payload[[element]] = DecodeTileMemoryElementRaw(296                    raw, destination_tile.data_type, FALSE);297            end;298        end;299    end;300    _Tiles[[destination]].payload = result_payload;301    MarkTilePhysicalRegionDefined(destination);302end;303
304func MSCATTER_MASK(base_address: Word, source: TileIndex, indices: TileIndex,305                   mask: TileIndex)306begin307    let source_tile = _Tiles[[source]];308    let index_tile = _Tiles[[indices]];309    let mask_tile = _Tiles[[mask]];310    assert source_tile.allocated && source_tile.contents_defined;311    assert index_tile.allocated && index_tile.contents_defined;312    assert mask_tile.allocated && mask_tile.contents_defined;313    assert source_tile.valid_rows == index_tile.valid_rows;314    assert source_tile.valid_columns == index_tile.valid_columns;315    assert source_tile.valid_rows == mask_tile.valid_rows;316    assert source_tile.valid_columns == mask_tile.valid_columns;317    assert source_tile.layout == index_tile.layout;318    assert source_tile.layout == mask_tile.layout;319    assert IndexedTLSUIndexDataTypeLegal(index_tile.data_type);320    assert IndexedTLSUTransferDataTypeLegal(source_tile.data_type);321    assert TilePredicateValuesLegal(mask);322    let source_payload = source_tile.payload;323    let index_payload = index_tile.payload;324    var lane_order: ScatterLaneOrder;325    var lane_count: integer {0..PTO_MODEL_TILE_ELEMENTS} = 0;326    var original_addresses: TilePayload;327    var translated_addresses: TilePayload;328    var high_nibbles: bits(PTO_MODEL_TILE_ELEMENTS);329    for row = 0 to source_tile.valid_rows - 1 looplimit 65536 do330        for column = 0 to source_tile.valid_columns - 1 looplimit 65536 do331            let source_element = TileLinearIndex(source_tile,332                row as integer {0..65535}, column as integer {0..65535});333            let index_element = TileLinearIndex(index_tile,334                row as integer {0..65535}, column as integer {0..65535});335            if ReadTilePredicateBit(336                mask,337                row as integer {0..65535},338                column as integer {0..65535}) then339                let address = TileMemoryByteDisplacementAddress(base_address,340                    index_payload[[index_element]], index_tile.data_type);341                let probe = ProbeTileMemoryAccess(address,342                    source_tile.data_type, TRUE);343                if RaiseDataAccessFault(probe, address) then return; end;344                original_addresses[[source_element]] = address;345                translated_addresses[[source_element]] =346                    probe.translated_address;347                high_nibbles[source_element] = '0';348                lane_order[[lane_count]] =349                    NaturalToWord(source_element as integer {0..262144});350                lane_count = (lane_count + 1) as351                    integer {0..PTO_MODEL_TILE_ELEMENTS};352            end;353        end;354    end;355    CommitScatterLanes(source_tile, source_payload, lane_order, lane_count,356        original_addresses, translated_addresses, high_nibbles);357end;358