PTO-TILE-MODEL-MEMORY-SHARED-MOVEMENT

1// PTO-UNIT: {"id":"PTO-TILE-MODEL-MEMORY-SHARED-MOVEMENT","surface":"tile","classification":["model","memory","shared-movement"],"depends_on":["PTO-TILE-MODEL-STATE-SHARED-REGISTERS","PTO-SCALAR-MODEL-AGU-MEMORY","PTO-ARCH-MEMORY-MODEL-GLOBAL-MEMORY-ACCESS"]}2// PTO-REQ-TLSU-001, PTO-REQ-MEMORY-COMPLETION-001,3// PTO-REQ-MEMORY-TSO-001: precise, restartable direct4// TLOAD/TSTORE/MGATHER/MSCATTER and destination-free TPREFETCH.5
6type ScatterLaneOrder of array [[PTO_MODEL_TILE_ELEMENTS]] of Word;7type CorePETileInfos of array [[PTO_MODEL_MEMORY_AGENTS]] of TileInfo;8
9readonly func SharedTileElementRegion(tile: TileInfo,10                                  element: PackedTileElementIndex)11                                  => integer {0..3}12begin13    let bit_offset: integer = element * TileElementBits(tile.data_type);14    let byte_offset: integer = bit_offset DIVRM 8;15    assert byte_offset < tile.capacity_bytes;16    return ((byte_offset * 4) DIVRM tile.capacity_bytes)17        as integer {0..3};18end;19
20func SharedTileFromLocal(source: TileIndex,21                         capacity_bytes: integer {128,256,512,1024,2048,4096,8192,22                                                  16384,32768,65536,131072,23                                                  262144})24                         => TileInfo25begin26    let source_tile = _Tiles[[source]];27    assert source_tile.allocated && source_tile.contents_defined;28    assert source_tile.capacity_bytes == capacity_bytes;29    var result = source_tile;30    result.location = TileLocation_Any;31    return result;32end;33
34func TMOVLocalToShared(shared_tile_id: SharedTileID, source: TileIndex,35                       size_code: integer {1..12}, pe_mask: bits(4),36                       publish: boolean)37begin38    if pe_mask == Zeros{4} then return; end;39    let capacity_bytes = TileSizeCodeBytes(size_code);40    let shared_tile = SharedTileFromLocal(source, capacity_bytes);41    if publish && !SharedTileProspectiveFullyInitialized(42            shared_tile_id, shared_tile, pe_mask) then43        SetFault(Fault_TileLegality, ReadTPC());44        return;45    end;46    let updated = AtomicUpdateSharedTileWithPublication(47        shared_tile_id, shared_tile, pe_mask, publish);48    if !updated then SetFault(Fault_TileLegality, ReadTPC()); end;49end;50
51func TMOVSharedToLocal(destination: TileIndex, shared_tile_id: SharedTileID,52                       shared_tile: TileInfo, pe_mask: bits(4))53begin54    if pe_mask == Zeros{4} then return; end;55    let destination_tile = _Tiles[[destination]];56    assert destination_tile.capacity_bytes == shared_tile.capacity_bytes;57    assert destination_tile.rows == shared_tile.rows &&58           destination_tile.columns == shared_tile.columns &&59           destination_tile.valid_rows == shared_tile.valid_rows &&60           destination_tile.valid_columns == shared_tile.valid_columns &&61           destination_tile.data_type == shared_tile.data_type &&62           destination_tile.layout == shared_tile.layout;63    // PE_MASK selects consumer PEs, not payload quarters. Every participating64    // consumer observes the same complete published parent when no B.SUBVIEW65    // narrows the source range.66    for element = 0 to shared_tile.rows * shared_tile.columns - 167        looplimit 524288 do68        _Tiles[[destination]] = TileInfoWithLogicalElement(69            _Tiles[[destination]], element as PackedTileElementIndex,70            ReadSharedTileWord(shared_tile_id,71                element as PackedTileElementIndex));72    end;73    _Tiles[[destination]].contents_defined = TRUE;74    MarkTileValidRegionDefined(destination);75end;76
77func TLOADShared(shared_tile_id: SharedTileID, base_addresses: CorePEWords,78                 row_stride_bytes: CorePEWords,79                 size_code: integer {1..12},80                 rows: integer {1..65535}, columns: integer {1..65535},81                 valid_rows: integer {1..65535},82                 valid_columns: integer {1..65535},83                 data_type: TileDataType, layout: TileLayout,84                 pe_mask: bits(4))85begin86    if pe_mask == Zeros{4} then return; end;87    let capacity_bytes = TileSizeCodeBytes(size_code);88    if rows < valid_rows || rows >89           DerivedTileRows(capacity_bytes, columns, data_type) ||90       !TileDescriptorShapeLegal(capacity_bytes, columns, valid_rows,91           valid_columns, data_type) then92        SetFault(Fault_TileLegality, ReadTPC());93        return;94    end;95    let derived_rows = DerivedTileRows(capacity_bytes, columns, data_type);96    if derived_rows * columns >97           TileLogicalElementCapacity(capacity_bytes, data_type) then98        SetFault(Fault_TileLegality, ReadTPC());99        return;100    end;101    var tile: TileInfo;102    tile.allocated = TRUE;103    tile.contents_defined = FALSE;104    tile.defined_elements = Zeros{PTO_MODEL_TILE_ELEMENTS};105    tile.defined_valid_elements = 0;106    tile.packed_defined_elements = ZeroPackedTileDefinedElements();107    tile.capacity_bytes = capacity_bytes;108    tile.rows = derived_rows;109    tile.columns = columns;110    tile.valid_rows = valid_rows;111    tile.valid_columns = valid_columns;112    tile.data_type = data_type;113    tile.layout = layout;114    tile.location = TileLocation_Any;115    tile.cube_k_repeat = 0;116    tile.cube_n_repeat = 0;117    tile.cube_cell_count = 0;118    tile.cube_storage_bytes = 0;119    if !SharedTileUpdateCompatible(shared_tile_id, tile, pe_mask) then120        SetFault(Fault_TileLegality, ReadTPC());121        return;122    end;123    let single_issuer = PEMaskPopulation(pe_mask) == 1;124    var single_agent: MemoryAgentId = 0;125    if single_issuer then126        for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do127            let agent = pe as MemoryAgentId;128            if pe_mask[PTOPEMaskBitOfPEIdentity(agent)] == '1' then129                single_agent = agent;130            end;131        end;132    end;133    // The maximum packed witness uses zero byte stride and a fresh Shared134    // record. Check every participating issuer through the ordinary135    // translated probe before publishing the complete carrier state.136    var packed_zero_fast = PackedTileDataTypeIsFourBit(tile.data_type) &&137        !_MemoryEventCaptureEnabled &&138        !SharedTileRecord(shared_tile_id).descriptor_valid &&139        tile.valid_rows == tile.rows &&140        tile.valid_columns == tile.columns &&141        tile.rows * tile.columns ==142            PackedTileLogicalCapacity(tile.capacity_bytes, tile.data_type) &&143        tile.capacity_bytes == 262144;144    if packed_zero_fast then145        for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do146            let agent = pe as MemoryAgentId;147            if pe_mask[PTOPEMaskBitOfPEIdentity(agent)] == '1' &&148               (base_addresses[[agent]] != Zeros{PTO_XLEN} ||149                row_stride_bytes[[agent]] != Zeros{PTO_XLEN}) then150                packed_zero_fast = FALSE;151            end;152        end;153    end;154    if packed_zero_fast then155        for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do156            let agent = pe as MemoryAgentId;157            if pe_mask[PTOPEMaskBitOfPEIdentity(agent)] == '1' then158                for column = 0 to tile.valid_columns - 1 looplimit 65536 do159                    let address = TileMemoryStridedByteAddress(160                        base_addresses[[agent]], 0,161                        column as integer {0..65535},162                        row_stride_bytes[[agent]], tile.data_type);163                    let probe = ProbeTileMemoryAccess(address,164                        tile.data_type, FALSE);165                    if RaiseDataAccessFault(probe, address) then return; end;166                    if LoadTranslatedUnsigned(probe.translated_address,167                           TileMemoryElementBytes(tile.data_type)) !=168                           Zeros{PTO_XLEN} then169                        packed_zero_fast = FALSE;170                    end;171                end;172            end;173        end;174    end;175    if packed_zero_fast then176        let payload_mask = if single_issuer then '1111' else pe_mask;177        tile = TileWithPackedZeroSelectedMaxRegionDefined(tile, payload_mask);178        let updated = AtomicUpdateSharedTile(shared_tile_id, tile, pe_mask);179        assert updated;180        return;181    end;182    for row = 0 to tile.valid_rows - 1 looplimit 65536 do183        for column = 0 to tile.valid_columns - 1 looplimit 65536 do184            let element = TileLogicalLinearIndex(tile,185                row as integer {0..65535}, column as integer {0..65535});186            let region = SharedTileElementRegion(tile, element);187            let selected = single_issuer ||188                pe_mask[PTOPEMaskBitOfPEIdentity(region)] == '1';189            if selected then190                let agent = if single_issuer then single_agent191                    else region as MemoryAgentId;192                let address = TileMemoryStridedByteAddress(193                    base_addresses[[agent]], row as integer {0..65535},194                    column as integer {0..65535},195                    row_stride_bytes[[agent]], tile.data_type);196                let probe = ProbeTileMemoryAccess(address, tile.data_type, FALSE);197                if RaiseDataAccessFault(probe, address) then return; end;198            end;199        end;200    end;201    for row = 0 to tile.valid_rows - 1 looplimit 65536 do202        for column = 0 to tile.valid_columns - 1 looplimit 65536 do203            let element = TileLogicalLinearIndex(tile,204                row as integer {0..65535}, column as integer {0..65535});205            let region = SharedTileElementRegion(tile, element);206            let selected = single_issuer ||207                pe_mask[PTOPEMaskBitOfPEIdentity(region)] == '1';208            if selected then209                let agent = if single_issuer then single_agent210                    else region as MemoryAgentId;211                let high_nibble = TileMemoryStridedByteHighNibble(212                    column as integer {0..65535}, tile.data_type);213                let address = TileMemoryStridedByteAddress(214                    base_addresses[[agent]], row as integer {0..65535},215                    column as integer {0..65535},216                    row_stride_bytes[[agent]], tile.data_type);217                let translated = ProbeTileMemoryAccess(address,218                    tile.data_type, FALSE).translated_address;219                let raw = LoadTranslatedUnsigned(translated,220                    TileMemoryElementBytes(tile.data_type));221                RecordLoadEventForAgent(agent, translated,222                    TileMemoryElementBytes(tile.data_type), raw,223                    CurrentBundleMemoryOrder());224                tile = TileInfoWithLogicalElement(tile, element,225                    LoadTileMemoryElement(translated, tile.data_type,226                        high_nibble));227            end;228        end;229    end;230    if single_issuer || pe_mask == '1111' then231        tile.defined_valid_elements =232            (tile.valid_rows * tile.valid_columns) as integer {0..524288};233        tile.contents_defined = TRUE;234    end;235    let updated = AtomicUpdateSharedTile(shared_tile_id, tile, pe_mask);236    assert updated;237end;238
239func TSTOREShared(base_addresses: CorePEWords,240                  row_stride_bytes: CorePEWords,241                  shared_tile_id: SharedTileID,242                  tile: TileInfo,243                  pe_mask: bits(4))244begin245    if pe_mask == Zeros{4} then return; end;246    assert tile.allocated;247    // PE_MASK selects consumer PEs. Each selected PE stores the complete248    // parent through its own private base and byte-stride GPR values.249    for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do250        let agent = pe as MemoryAgentId;251        if pe_mask[PTOPEMaskBitOfPEIdentity(agent)] == '1' then252            for row = 0 to tile.valid_rows - 1 looplimit 65536 do253                for column = 0 to tile.valid_columns - 1 looplimit 65536 do254                    let address = TileMemoryStridedByteAddress(255                        base_addresses[[agent]], row as integer {0..65535},256                        column as integer {0..65535},257                        row_stride_bytes[[agent]], tile.data_type);258                    let probe = ProbeTileMemoryAccess(259                        address, tile.data_type, TRUE);260                    if RaiseDataAccessFault(probe, address) then return; end;261                end;262            end;263        end;264    end;265    for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do266        let agent = pe as MemoryAgentId;267        if pe_mask[PTOPEMaskBitOfPEIdentity(agent)] == '1' then268            for row = 0 to tile.valid_rows - 1 looplimit 65536 do269                for column = 0 to tile.valid_columns - 1 looplimit 65536 do270                    let element = TileLogicalLinearIndex(tile,271                        row as integer {0..65535},272                        column as integer {0..65535});273                    let address = TileMemoryStridedByteAddress(274                        base_addresses[[agent]], row as integer {0..65535},275                        column as integer {0..65535},276                        row_stride_bytes[[agent]], tile.data_type);277                    let translated = ProbeTileMemoryAccess(address,278                        tile.data_type, TRUE).translated_address;279                    let stored_value = StoreTileMemoryElement(280                        address, translated, tile.data_type,281                        TileMemoryStridedByteHighNibble(282                            column as integer {0..65535}, tile.data_type),283                        TileReadLogicalElement(tile, element));284                    RecordStoreEventForAgent(agent, translated,285                        TileMemoryElementBytes(tile.data_type), stored_value,286                        CurrentBundleMemoryOrder());287                end;288            end;289        end;290    end;291end;292
293
294func TMOVSharedToLocalPerPE(destination: TileIndex,295                           per_pe_tiles: CorePETileInfos,296                           pe_mask: bits(4))297begin298    if pe_mask == Zeros{4} then return; end;299    for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do300        let agent = pe as MemoryAgentId;301        if pe_mask[PTOPEMaskBitOfPEIdentity(agent)] == '1' then302            let source_tile = per_pe_tiles[[agent]];303            let destination_tile = _Tiles[[destination]];304            assert source_tile.allocated && source_tile.contents_defined;305            assert destination_tile.capacity_bytes == source_tile.capacity_bytes;306            assert destination_tile.rows == source_tile.rows &&307                   destination_tile.columns == source_tile.columns &&308                   destination_tile.valid_rows == source_tile.valid_rows &&309                   destination_tile.valid_columns == source_tile.valid_columns &&310                   destination_tile.data_type == source_tile.data_type &&311                   destination_tile.layout == source_tile.layout;312            for element = 0 to source_tile.rows * source_tile.columns - 1313                looplimit 524288 do314                let value = TileReadLogicalElement(source_tile,315                    element as PackedTileElementIndex);316                _Tiles[[destination]] = TileInfoWithLogicalElement(317                    _Tiles[[destination]],318                    element as PackedTileElementIndex, value);319            end;320        end;321    end;322    _Tiles[[destination]].contents_defined = pe_mask == '1111';323    if pe_mask == '1111' then MarkTileValidRegionDefined(destination); end;324end;325
326func TSTORESharedPerPE(base_addresses: CorePEWords,327                       row_stride_bytes: CorePEWords,328                       per_pe_tiles: CorePETileInfos,329                       pe_mask: bits(4))330begin331    if pe_mask == Zeros{4} then return; end;332    for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do333        let agent = pe as MemoryAgentId;334        if pe_mask[PTOPEMaskBitOfPEIdentity(agent)] == '1' then335            let tile = per_pe_tiles[[agent]];336            assert tile.allocated && tile.contents_defined;337            for row = 0 to tile.valid_rows - 1 looplimit 65536 do338                for column = 0 to tile.valid_columns - 1 looplimit 65536 do339                    let address = TileMemoryStridedByteAddress(340                        base_addresses[[agent]], row as integer {0..65535},341                        column as integer {0..65535},342                        row_stride_bytes[[agent]], tile.data_type);343                    let probe = ProbeTileMemoryAccess(address,344                        tile.data_type, TRUE);345                    if RaiseDataAccessFault(probe, address) then return; end;346                end;347            end;348        end;349    end;350    for pe = 0 to PTO_MODEL_MEMORY_AGENTS - 1 do351        let agent = pe as MemoryAgentId;352        if pe_mask[PTOPEMaskBitOfPEIdentity(agent)] == '1' then353            let tile = per_pe_tiles[[agent]];354            for row = 0 to tile.valid_rows - 1 looplimit 65536 do355                for column = 0 to tile.valid_columns - 1 looplimit 65536 do356                    let address = TileMemoryStridedByteAddress(357                        base_addresses[[agent]], row as integer {0..65535},358                        column as integer {0..65535},359                        row_stride_bytes[[agent]], tile.data_type);360                    let translated = ProbeTileMemoryAccess(address,361                        tile.data_type, TRUE).translated_address;362                    let stored_value = StoreTileMemoryElement(363                        address, translated, tile.data_type,364                        TileMemoryStridedByteHighNibble(365                            column as integer {0..65535}, tile.data_type),366                        TileReadLogicalElement(tile,367                            TileLogicalLinearIndex(tile,368                                row as integer {0..65535},369                                column as integer {0..65535})));370                    RecordStoreEventForAgent(agent, translated,371                        TileMemoryElementBytes(tile.data_type), stored_value,372                        CurrentBundleMemoryOrder());373                end;374            end;375        end;376    end;377end;378
379func TMOV(destination: TileIndex, source: TileIndex)380begin381    let source_tile = _Tiles[[source]];382    assert source_tile.allocated;383    assert TileShapesMatch(_Tiles[[destination]], source_tile);384    _Tiles[[destination]].payload = source_tile.payload;385    _Tiles[[destination]].defined_elements = source_tile.defined_elements;386    _Tiles[[destination]].packed_defined_elements =387        source_tile.packed_defined_elements;388    _Tiles[[destination]].defined_valid_elements =389        source_tile.defined_valid_elements;390    _Tiles[[destination]].contents_defined = source_tile.contents_defined;391end;392
393// The direct-operation carrier binds source to the Core4 snapshot already394// resolved from the four PE-private peer_tid values.  The bundle dispatcher395// performs collective readiness and peer-range preflight before this read-old,396// write-new local copy.  No Shared register or global-memory event is involved.397func GMOV(destination: TileIndex, source: TileIndex, peer_tid: Word)398begin399    assert UInt(peer_tid) < 4;400    let source_tile = _Tiles[[source]];401    let source_payload = source_tile.payload;402    _Tiles[[destination]].payload = source_payload;403    _Tiles[[destination]].defined_elements = source_tile.defined_elements;404    _Tiles[[destination]].packed_defined_elements =405        source_tile.packed_defined_elements;406    _Tiles[[destination]].defined_valid_elements =407        source_tile.defined_valid_elements;408    _Tiles[[destination]].contents_defined = source_tile.contents_defined;409end;410