PTO-TILE-MODEL-MEMORY-LOAD-STORE

1// PTO-UNIT: {"id":"PTO-TILE-MODEL-MEMORY-LOAD-STORE","surface":"tile","classification":["model","memory","load-store"],"depends_on":["PTO-TILE-MODEL-MEMORY-STRIDE","PTO-ARCH-MEMORY-MODEL-FAULT-PRECISION"]}2pure func DecodeTileMemoryElementRaw(raw: Word,3                                     data_type: TileDataType,4                                     high_nibble: boolean) => Word5begin6    let element_bytes = TileMemoryElementBytes(data_type);7    if TileDataTypeIsFourBit(data_type) then8        if high_nibble then9            return ZeroExtend{PTO_XLEN}(raw[7:4]);10        else11            return ZeroExtend{PTO_XLEN}(raw[3:0]);12        end;13    else14        return NormalizeLoadedValue(raw, element_bytes,15            TileDataTypeIsSigned(data_type));16    end;17end;18
19readonly func LoadTileMemoryElement(translated_address: Word,20                                    data_type: TileDataType,21                                    high_nibble: boolean) => Word22begin23    let element_bytes = TileMemoryElementBytes(data_type);24    let raw = LoadTranslatedUnsigned(translated_address, element_bytes);25    return DecodeTileMemoryElementRaw(raw, data_type, high_nibble);26end;27
28func StoreTileMemoryElement(original_address: Word,29                            translated_address: Word,30                            data_type: TileDataType,31                            high_nibble: boolean,32                            value: Word) => Word33begin34    let element_bytes = TileMemoryElementBytes(data_type);35    if TileDataTypeIsFourBit(data_type) then36        let old_byte = LoadTranslatedUnsigned(translated_address, 1);37        var stored_byte: Byte = old_byte[7:0];38        if high_nibble then stored_byte[7:4] = value[3:0];39        else stored_byte[3:0] = value[3:0];40        end;41        let stored_value = ZeroExtend{PTO_XLEN}(stored_byte);42        StoreTranslated(original_address, translated_address, 1, stored_value);43        return stored_value;44    else45        let stored_value = NormalizeMemoryAccessValue(value, element_bytes);46        StoreTranslated(original_address, translated_address, element_bytes,47            stored_value);48        return stored_value;49    end;50end;51
52func ProbeTileMemoryAccess(address: Word, data_type: TileDataType,53                           write: boolean) => DataAccessProbe54begin55    let element_bytes = TileMemoryElementBytes(data_type);56    return ProbeDataAccess(address, element_bytes, element_bytes, write);57end;58
59func TLOAD(destination: TileIndex, base_address: Word,60           row_stride_bytes: Word)61begin62    let tile = _Tiles[[destination]];63    assert tile.allocated;64    if TileLayoutIsCube(tile.layout) then65        assert TileCubeDescriptorLegal(tile);66    end;67    // Complete packed carriers make the maximum U4 shape representable, but68    // an interpreter-sized translated-address array is still only a carrier69    // cache.  The fast case is limited to ordinary reset-backed zero-stride70    // packed loads and retains the normal translated probe/fault path.71    var packed_zero_fast = PackedTileDataTypeIsFourBit(tile.data_type) &&72        !TileLayoutIsCube(tile.layout) &&73        !_MemoryEventCaptureEnabled &&74        tile.defined_valid_elements == 0 &&75        row_stride_bytes == Zeros{PTO_XLEN} &&76        tile.valid_rows == tile.rows &&77        tile.valid_columns == tile.columns &&78        tile.rows * tile.columns ==79            PackedTileLogicalCapacity(tile.capacity_bytes, tile.data_type);80    if packed_zero_fast then81        for column = 0 to tile.valid_columns - 1 looplimit 65536 do82            let address = TileMemoryStridedByteAddress(base_address,83                0, column as integer {0..65535}, row_stride_bytes,84                tile.data_type);85            let probe = ProbeTileMemoryAccess(address, tile.data_type, FALSE);86            if RaiseDataAccessFault(probe, address) then return; end;87            if LoadTranslatedUnsigned(probe.translated_address,88                   TileMemoryElementBytes(tile.data_type)) !=89                   Zeros{PTO_XLEN} then90                packed_zero_fast = FALSE;91            end;92        end;93    end;94    if packed_zero_fast then95        _Tiles[[destination]] = TileWithPackedZeroValidRegionDefined(tile);96        return;97    end;98    var result = tile;99    // Instruction-wide preflight makes tile memory faults precise and100    // restartable: no payload element changes until every access succeeds.101    for row = 0 to tile.valid_rows - 1 looplimit 65536 do102        for column = 0 to tile.valid_columns - 1 looplimit 65536 do103            let element = TileLogicalLinearIndex(tile,104                row as integer {0..65535}, column as integer {0..65535});105            let address = TileMemoryStridedByteAddress(base_address,106                row as integer {0..65535}, column as integer {0..65535},107                row_stride_bytes, tile.data_type);108            let probe = ProbeTileMemoryAccess(address, tile.data_type, FALSE);109            if RaiseDataAccessFault(probe, address) then return; end;110        end;111    end;112    for row = 0 to tile.valid_rows - 1 looplimit 65536 do113        for column = 0 to tile.valid_columns - 1 looplimit 65536 do114            let element = TileLogicalLinearIndex(tile,115                row as integer {0..65535}, column as integer {0..65535});116            let address = TileMemoryStridedByteAddress(base_address,117                row as integer {0..65535}, column as integer {0..65535},118                row_stride_bytes, tile.data_type);119            let translated = ProbeTileMemoryAccess(address, tile.data_type,120                FALSE).translated_address;121            let high_nibble = TileMemoryStridedByteHighNibble(122                column as integer {0..65535}, tile.data_type);123            let raw = LoadTranslatedUnsigned(translated,124                TileMemoryElementBytes(tile.data_type));125            RecordLoadEvent(translated,126                TileMemoryElementBytes(tile.data_type), raw,127                CurrentBundleMemoryOrder());128            result = TileInfoWithLogicalElement(result, element,129                DecodeTileMemoryElementRaw(raw, tile.data_type, high_nibble));130        end;131    end;132    result = TileWithValidRegionDefined(result);133    if TileLayoutIsCube(result.layout) then134        result = TileWithPadding(result, CurrentBundlePadValue());135    end;136    _Tiles[[destination]] = result;137end;138
139func TSTORE(base_address: Word, row_stride_bytes: Word, source: TileIndex)140begin141    let tile = _Tiles[[source]];142    assert tile.allocated && tile.contents_defined;143    if TileLayoutIsCube(tile.layout) then144        assert TileCubeDescriptorLegal(tile);145    end;146    for row = 0 to tile.valid_rows - 1 looplimit 65536 do147        for column = 0 to tile.valid_columns - 1 looplimit 65536 do148            let element = TileLogicalLinearIndex(tile,149                row as integer {0..65535}, column as integer {0..65535});150            let address = TileMemoryStridedByteAddress(base_address,151                row as integer {0..65535}, column as integer {0..65535},152                row_stride_bytes, tile.data_type);153            let probe = ProbeTileMemoryAccess(address, tile.data_type, TRUE);154            if RaiseDataAccessFault(probe, address) then return; end;155        end;156    end;157    for row = 0 to tile.valid_rows - 1 looplimit 65536 do158        for column = 0 to tile.valid_columns - 1 looplimit 65536 do159            let element = TileLogicalLinearIndex(tile,160                row as integer {0..65535}, column as integer {0..65535});161            let address = TileMemoryStridedByteAddress(base_address,162                row as integer {0..65535}, column as integer {0..65535},163                row_stride_bytes, tile.data_type);164            let translated = ProbeTileMemoryAccess(address, tile.data_type,165                TRUE).translated_address;166            let stored_value = StoreTileMemoryElement(167                address, translated, tile.data_type,168                TileMemoryStridedByteHighNibble(169                    column as integer {0..65535}, tile.data_type),170                TileReadLogicalElement(tile, element));171            RecordStoreEvent(translated,172                TileMemoryElementBytes(tile.data_type), stored_value,173                CurrentBundleMemoryOrder());174        end;175    end;176end;177