PTO-TILE-MODEL-EXECUTION-POSTPROCESS

1// PTO-UNIT: {"id":"PTO-TILE-MODEL-EXECUTION-POSTPROCESS","surface":"tile","classification":["model","execution","postprocess"],"depends_on":["PTO-TILE-MODEL-EXECUTION-CUBE","PTO-TILE-MODEL-LEGALITY-MATRIX-POSTPROCESS"]}2// Complete-bundle B.FPATR post-processing.  Numeric conversion and activation3// remain behind the named profile hooks while this unit owns operand routing,4// reductions, and atomic auxiliary-output publication.5
6impdef func TileProfileMatrixPostProcess(7    value: Word, pre_quant_mode: bits(6), relu_mode: bits(3),8    group_n_code: bits(4), output_type: TileDataType,9    quant_param: Word, relu_param: Word,10    control: NumericExecutionControl) => Word11begin12    return value;13end;14
15impdef func TileProfileMatrixPostProcessWithFlags(16    value: Word, pre_quant_mode: bits(6), relu_mode: bits(3),17    group_n_code: bits(4), output_type: TileDataType,18    quant_param: Word, relu_param: Word,19    control: NumericExecutionControl) => (Word, bits(5))20begin21    return (value, Zeros{5});22end;23
24impdef func TileProfileMatrixReductionStep(25    current: Word, candidate: Word, max_abs: boolean,26    data_type: TileDataType) => Word27begin28    return candidate;29end;30
31impdef func TileProfileMatrixReductionStepWithFlags(32    current: Word, candidate: Word, max_abs: boolean,33    data_type: TileDataType) => (Word, bits(5))34begin35    return (candidate, Zeros{5});36end;37
38readonly func BundleMatrixLocalMathematicalSourceCount() => integer {0..6}39begin40    let function = UInt(_BundleOperation.selector[4:0]);41    let left_type = TileDataTypeFromEncoding(42        CurrentBundleTileOperationDataTypeCode() as TileDataTypeEncoding);43    let right_type = if _BundleDataAttributesPresent then44        TileDataTypeFromEncoding(45            _BundleDataAttributes.data_type as TileDataTypeEncoding)46        else left_type;47    let base = TileMatrixLocalMathematicalSourceCount(48        function, left_type, right_type, BundleSharedBindingCount());49    return (base +50        (if _BundleFixedPointAttributes.c_scale_en then 1 else 0))51        as integer {0..6};52end;53
54readonly func BundleMatrixOperationIndex() => integer {0..PTO_TILE_OPERATION_COUNT-1}55begin56    let decoded = DecodeTileOperation(TileDecode_CUBE,57        BundleOperationDecodeCode(_BundleOperation));58    assert decoded != PTO_TILE_OPERATION_COUNT;59    return decoded as integer {0..PTO_TILE_OPERATION_COUNT-1};60end;61
62func MatrixRowMaxResult(input: TileInfo, destination: TileIndex,63                        rowmax_input: TileIndex, has_input: boolean,64                        intermediate_type: TileDataType)65                        => (TileInfo, bits(5))66begin67    var output = _Tiles[[destination]];68    var payload = output.payload;69    var flags = Zeros{5};70    for row = 0 to input.valid_rows - 1 looplimit 65536 do71        let first = TileStorageIndex(input, row as integer {0..65535}, 0);72        var value = input.payload[[first]];73        if _BundleFixedPointAttributes.max_abs_en then74            let (initial, initial_flags) =75                TileProfileMatrixReductionStepWithFlags(76                    value, value, TRUE, intermediate_type);77            value = initial;78            flags = flags OR initial_flags;79        end;80        for column = 1 to input.valid_columns - 1 looplimit 65536 do81            let element = TileStorageIndex(input, row as integer {0..65535}, column as integer {0..65535});82            let (next, step_flags) = TileProfileMatrixReductionStepWithFlags(83                value, input.payload[[element]],84                _BundleFixedPointAttributes.max_abs_en, intermediate_type);85            value = next;86            flags = flags OR step_flags;87        end;88        if has_input then89            let input_element = TileStorageIndex(_Tiles[[rowmax_input]], row as integer {0..65535}, 0);90            let (next, step_flags) = TileProfileMatrixReductionStepWithFlags(91                _Tiles[[rowmax_input]].payload[[input_element]], value,92                _BundleFixedPointAttributes.max_abs_en, intermediate_type);93            value = next;94            flags = flags OR step_flags;95        end;96        let output_element = TileStorageIndex(output, row as integer {0..65535}, 0);97        payload[[output_element]] = value;98    end;99    output.payload = payload;100    return (MarkLocalTileValidRegionDefined(output), flags);101end;102
103func MatrixGroupMaxResult(input: TileInfo, destination: TileIndex,104                          intermediate_type: TileDataType)105    => (TileInfo, bits(5))106begin107    var output = _Tiles[[destination]];108    var payload = output.payload;109    var flags = Zeros{5};110    let group_n = BundleFPATRGroupN(_BundleFixedPointAttributes.group_n_code);111    for row = 0 to input.valid_rows - 1 looplimit 65536 do112        for group = 0 to output.valid_columns - 1 looplimit 65536 do113            let first_column = group * group_n;114            let first = TileStorageIndex(input, row as integer {0..65535}, first_column as integer {0..65535});115            var value = input.payload[[first]];116            if _BundleFixedPointAttributes.max_abs_en then117                let (initial, initial_flags) =118                    TileProfileMatrixReductionStepWithFlags(119                        value, value, TRUE, intermediate_type);120                value = initial;121                flags = flags OR initial_flags;122            end;123            for offset = 1 to group_n - 1 looplimit 128 do124                let column = first_column + offset;125                if column < input.valid_columns then126                    let element = TileStorageIndex(input, row as integer {0..65535}, column as integer {0..65535});127                    let (next, step_flags) =128                        TileProfileMatrixReductionStepWithFlags(129                            value, input.payload[[element]],130                            _BundleFixedPointAttributes.max_abs_en,131                            intermediate_type);132                    value = next;133                    flags = flags OR step_flags;134                end;135            end;136            let output_element = TileStorageIndex(output, row as integer {0..65535}, group as integer {0..65535});137            payload[[output_element]] = value;138        end;139    end;140    output.payload = payload;141    return (MarkLocalTileValidRegionDefined(output), flags);142end;143
144func MatrixPostProcessResult(input: TileInfo,145                             intermediate_type: TileDataType)146    => (TileInfo, bits(5))147begin148    if !_BundleFixedPointAttributes.valid then149        return (input, Zeros{5});150    end;151    var result = input;152    var payload = input.payload;153    var flags = Zeros{5};154    let output_type = if UInt(155        _BundleFixedPointAttributes.pre_quant_mode) == 0 then156        intermediate_type157    else158        BundleFPATROutputType(_BundleFixedPointAttributes.pre_quant_mode);159    let operation = BundleMatrixOperationIndex();160    let operands = BundleTileInstructionOperands(operation);161    let numeric_control = ResolveTileNumericExecutionControl(operation, operands);162    let mathematical_sources = BundleMatrixLocalMathematicalSourceCount();163    let quant_source_ordinal = mathematical_sources +164        (if _BundleFixedPointAttributes.row_max_en &&165            _BundleFixedPointAttributes.row_max_init then 1 else 0);166    let relu_source_ordinal = quant_source_ordinal +167        (if BundleFPATRModeUsesVectorParameter(168            _BundleFixedPointAttributes.pre_quant_mode) then 1 else 0);169    let quant_tile = BundleMatrixSourceAt(quant_source_ordinal as integer {0..8});170    let relu_tile = BundleMatrixSourceAt(relu_source_ordinal as integer {0..8});171    for row = 0 to input.valid_rows - 1 looplimit 65536 do172        for column = 0 to input.valid_columns - 1 looplimit 65536 do173            let element = TileStorageIndex(input, row as integer {0..65535},174                column as integer {0..65535});175            let quant_param = if BundleFPATRModeUsesVectorParameter(176                _BundleFixedPointAttributes.pre_quant_mode) then177                _Tiles[[quant_tile]].payload[[TileStorageIndex(178                    _Tiles[[quant_tile]], 0,179                    column as integer {0..65535})]]180            else if BundleFPATRModeUsesScalarParameter(181                _BundleFixedPointAttributes.pre_quant_mode) then182                operands.post_quant_param183            else184                Zeros{PTO_XLEN} + 1;185            let relu_param = if BundleFPATRReluModeUsesVectorParameter(186                _BundleFixedPointAttributes.relu_mode) then187                _Tiles[[relu_tile]].payload[[TileStorageIndex(188                    _Tiles[[relu_tile]], 0,189                    column as integer {0..65535})]]190            else191                operands.post_lrelu_param;192            let (processed, element_flags) =193                TileProfileMatrixPostProcessWithFlags(194                payload[[element]],195                _BundleFixedPointAttributes.pre_quant_mode,196                _BundleFixedPointAttributes.relu_mode,197                _BundleFixedPointAttributes.group_n_code,198                output_type, quant_param, relu_param, numeric_control);199            payload[[element]] = processed;200            flags = flags OR element_flags;201        end;202    end;203    result.data_type = output_type;204    result.payload = payload;205    return (result, flags);206end;207
208func CommitMatrixResult(destination: TileIndex, result: TileInfo,209                        intermediate_type: TileDataType)210begin211    let mathematical_sources = BundleMatrixLocalMathematicalSourceCount();212    let rowmax_input = BundleMatrixSourceAt(213        mathematical_sources as integer {0..8});214    let (processed, process_flags) = MatrixPostProcessResult(215        result, intermediate_type);216    let rowmax_destination = BundleMatrixDestinationAt(1);217    let group_destination = if _BundleFixedPointAttributes.row_max_en then BundleMatrixDestinationAt(2)218        else BundleMatrixDestinationAt(1);219    let (row_result, row_flags) = if _BundleFixedPointAttributes.row_max_en then220        MatrixRowMaxResult(221            result, rowmax_destination, rowmax_input,222            _BundleFixedPointAttributes.row_max_init,223            intermediate_type)224        else (_Tiles[[0]], Zeros{5});225    let (group_result, group_flags) = if _BundleFixedPointAttributes.group_max_en then226        MatrixGroupMaxResult(result, group_destination, intermediate_type)227        else (_Tiles[[0]], Zeros{5});228    // Prepare every output from pre-commit state, then publish as one group.229    _Tiles[[destination]] = processed;230    if _BundleFixedPointAttributes.row_max_en then _Tiles[[rowmax_destination]] = row_result; end;231    if _BundleFixedPointAttributes.group_max_en then _Tiles[[group_destination]] = group_result; end;232    RecordNumericStatusFlags(process_flags OR row_flags OR group_flags);233end;234
235func CommitMatrixResult(destination: TileIndex, result: TileInfo)236begin237    CommitMatrixResult(destination, result, result.data_type);238end;239