PTO-ARCH-PROFILE-MATRIX-POSTPROCESS

1// PTO-UNIT: {"id":"PTO-ARCH-PROFILE-MATRIX-POSTPROCESS","surface":"arch","classification":["profile","matrix-postprocess"],"depends_on":["PTO-ARCH-PROFILE-MATRIX-QUANTIZATION","PTO-ARCH-PROFILE-REFERENCE-PROFILE","PTO-TILE-MODEL-EXECUTION-MATRIX-SCALE"]}2// Bit-exact B.FPATR conversion, activation, auxiliary reduction, and flags.3// NDF-BEGIN: PTO-MATRIX-POSTPROCESS-BITEXACT-0014// ndf: kind=contract level=L1 layer=architecture status=accepted5// Matrix post-processing MUST reduce the raw accumulator before conversion,6// select an activation-dependent multiplier before destination conversion,7// canonicalize special results, and publish D, enabled auxiliary outputs, and8// sticky flags as one non-faulting commit.9// NDF-END: PTO-MATRIX-POSTPROCESS-BITEXACT-00110
11implementation func TileProfileMatrixCScale(12    value: Word, exponent: bits(8)) => Word13begin14    let value_class = ClassifyFP32(value[31:0]);15    if NumericValueClassIsNaN(value_class) ||16       value_class == NumericValue_PositiveInfinity ||17       value_class == NumericValue_NegativeInfinity ||18       value_class == NumericValue_PositiveZero ||19       value_class == NumericValue_NegativeZero then20        return value;21    end;22    var scaled = ReferenceFP32FiniteValue(value[31:0]);23    for step = 1 to UInt(exponent) looplimit 255 do24        scaled = scaled / 2.0;25    end;26    let (encoded, flags) = ReferenceFP32FiniteEncoding(27        scaled, NumericRound_RNE);28    RecordNumericStatusFlags(flags);29    return encoded;30end;31
32pure func MatrixFloatingSignedZero(33    data_type: TileDataType, negative: boolean) => Word34begin35    if !negative then return Zeros{PTO_XLEN}; end;36    case data_type of37        when TileDataType_FP32 => return Zeros{PTO_XLEN} + 0x80000000;38        when TileDataType_FP16, TileDataType_BF16 =>39            return Zeros{PTO_XLEN} + 0x8000;40        when TileDataType_E4M3 => return Zeros{PTO_XLEN} + 0x80;41        when TileDataType_HiF8 => return Zeros{PTO_XLEN};42        otherwise => return Zeros{PTO_XLEN};43    end;44end;45
46pure func MatrixFloatingInfinity(47    data_type: TileDataType, negative: boolean) => Word48begin49    case data_type of50        when TileDataType_FP32 =>51            return Zeros{PTO_XLEN} +52                (if negative then 0xff800000 else 0x7f800000);53        when TileDataType_FP16 =>54            return Zeros{PTO_XLEN} +55                (if negative then 0xfc00 else 0x7c00);56        when TileDataType_BF16 =>57            return Zeros{PTO_XLEN} +58                (if negative then 0xff80 else 0x7f80);59        when TileDataType_HiF8 =>60            return Zeros{PTO_XLEN} +61                (if negative then 0xef else 0x6f);62        otherwise =>63            let (available, quiet_nan) =64                HardwareNumericCanonicalNaNResult(data_type);65            assert available;66            return quiet_nan;67    end;68end;69
70pure func MatrixFloatingLargestFinite(71    data_type: TileDataType, negative: boolean) => Word72begin73    case data_type of74        when TileDataType_FP32 =>75            return Zeros{PTO_XLEN} +76                (if negative then 0xff7fffff else 0x7f7fffff);77        when TileDataType_FP16 =>78            return Zeros{PTO_XLEN} +79                (if negative then 0xfbff else 0x7bff);80        when TileDataType_BF16 =>81            return Zeros{PTO_XLEN} +82                (if negative then 0xff7f else 0x7f7f);83        when TileDataType_E4M3 =>84            return Zeros{PTO_XLEN} +85                (if negative then 0xfe else 0x7e);86        when TileDataType_HiF8 =>87            return Zeros{PTO_XLEN} +88                (if negative then 0xee else 0x6e);89        otherwise => unreachable;90    end;91end;92
93func MatrixEncodeReal(94    value: real, data_type: TileDataType,95    control: NumericExecutionControl) => (Word, bits(5))96begin97    if TileDataTypeIsInteger(data_type) then98        return ReferenceMatrixIntegerEncoding(value, data_type, control);99    end;100    return ReferenceMatrixFloatingEncoding(value, data_type, control);101end;102
103func MatrixPostQuantSpecialValue(104    value: Word, source_type: TileDataType,105    destination_type: TileDataType,106    control: NumericExecutionControl)107    => (boolean, Word, bits(5))108begin109    if !TileDataTypeIsFloating(source_type) then110        return (FALSE, Zeros{PTO_XLEN}, Zeros{5});111    end;112    let value_class = TileNumericValueClass(source_type, value);113    if NumericValueClassIsNaN(value_class) then114        if TileDataTypeIsInteger(destination_type) then115            return (TRUE,116                if control.saturating then Zeros{PTO_XLEN}117                else TileIntegerMinimum(destination_type),118                Zeros{5} + 1);119        end;120        if control.saturating then121            return (TRUE, Zeros{PTO_XLEN},122                if value_class == NumericValue_SignalingNaN then123                    Zeros{5} + 1 else Zeros{5});124        end;125        let (available, quiet_nan) =126            HardwareNumericCanonicalNaNResult(destination_type);127        assert available;128        return (TRUE, quiet_nan,129            if value_class == NumericValue_SignalingNaN then130                Zeros{5} + 1 else Zeros{5});131    elsif NumericValueClassIsInfinity(value_class) then132        let negative = value_class == NumericValue_NegativeInfinity;133        if TileDataTypeIsInteger(destination_type) then134            let endpoint = if !control.saturating then135                TileIntegerMinimum(destination_type)136            else if negative then137                TileIntegerMinimum(destination_type)138            else139                TileIntegerMaximum(destination_type);140            return (TRUE, endpoint,141                if control.saturating then Zeros{5} + 0x14142                else Zeros{5} + 1);143        elsif control.saturating then144            return (TRUE,145                MatrixFloatingLargestFinite(destination_type, negative),146                Zeros{5} + 0x14);147        else148            return (TRUE,149                MatrixFloatingInfinity(destination_type, negative),150                if destination_type == TileDataType_E4M3 then151                    Zeros{5} + 0x14 else Zeros{5});152        end;153    end;154    return (FALSE, Zeros{PTO_XLEN}, Zeros{5});155end;156
157pure func MatrixValueClassNegative(158    value_class: NumericValueClass) => boolean159begin160    return value_class == NumericValue_NegativeNormal ||161           value_class == NumericValue_NegativeSubnormal ||162           value_class == NumericValue_NegativeInfinity ||163           value_class == NumericValue_NegativeZero;164end;165
166pure func MatrixSelectedMultiplier(167    source_negative: boolean, relu_mode: bits(3),168    quant_scale: real, relu_param: Word) => real169begin170    if !source_negative || UInt(relu_mode) == 0 then171        return quant_scale;172    elsif UInt(relu_mode) == 1 then173        return 0.0;174    end;175    return FP19FiniteValue(relu_param[18:0]);176end;177
178func MatrixActivationWithFlags(179    value: real, source_negative: boolean, relu_mode: bits(3),180    quant_scale: real, relu_param: Word) => (real, bits(5))181begin182    let multiplier = MatrixSelectedMultiplier(183        source_negative, relu_mode, quant_scale, relu_param);184    return (value * multiplier, Zeros{5});185end;186
187pure func MatrixFPATREffectiveControl(188    pre_quant_mode: bits(6), control: NumericExecutionControl)189    => NumericExecutionControl190begin191    var result = control;192    let mode = UInt(pre_quant_mode);193    if mode == 25 || mode == 28 then194        result.rounding_mode = NumericRound_RHB;195    elsif BundleFPATRModeFixedRounding(pre_quant_mode) then196        result.rounding_mode = NumericRound_RNE;197    end;198    return result;199end;200
201func MatrixPostQuantBaseWithFlags(202    value: Word, pre_quant_mode: bits(6), output_type: TileDataType,203    relu_mode: bits(3), quant_param: Word, relu_param: Word,204    control: NumericExecutionControl)205    => (Word, bits(5))206begin207    if UInt(pre_quant_mode) == 0 && UInt(relu_mode) == 0 then208        return (value, Zeros{5});209    end;210    if BundleFPATRModeIsShift(pre_quant_mode) then211        let shift = UInt(quant_param[35:32]) + 1;212        return MatrixShiftS32ToS16(213            value[31:0], shift as integer {1..16});214    end;215
216    let source_type = if UInt(pre_quant_mode) == 0 then output_type217    else if BundleFPATRModeUsesS32Accumulator(218        pre_quant_mode) then TileDataType_S32 else TileDataType_FP32;219    let source_class = if source_type == TileDataType_FP32 then220        TileNumericValueClass(source_type, value)221    else222        NumericValue_PositiveNormal;223    let source_negative = if source_type == TileDataType_S32 then224        SInt(value[31:0]) < 0225    else226        MatrixValueClassNegative(source_class);227    let scale = if BundleFPATRModeUsesScalarParameter(pre_quant_mode) ||228                   BundleFPATRModeUsesVectorParameter(pre_quant_mode) then229        FP19FiniteValue(quant_param[31:13])230    else231        1.0;232    let multiplier = MatrixSelectedMultiplier(233        source_negative, relu_mode, scale, relu_param);234    let (special, special_result, special_flags) =235        MatrixPostQuantSpecialValue(236            value, source_type, output_type, control);237    if special &&238       !(source_class == NumericValue_NegativeInfinity &&239         multiplier == 0.0) then240        return (special_result, special_flags);241    end;242
243    let source_value = if source_type == TileDataType_S32 then244        Real(SInt(value[31:0]))245    else if source_type == TileDataType_U32 then246        Real(UInt(value[31:0]))247    else if source_class == NumericValue_NegativeInfinity then248        0.0249    else250        ReferenceFP32FiniteValue(value[31:0]);251    let (activated, activation_flags) = MatrixActivationWithFlags(252        source_value, source_negative, relu_mode, scale, relu_param);253    let offset = MatrixQuantOffset(254        quant_param, BundleFPATRModeOffsetWidth(pre_quant_mode));255    let intermediate_width =256        BundleFPATRModeOffsetWidth(pre_quant_mode);257    if source_class == NumericValue_NegativeZero &&258       multiplier != 0.0 && offset == 0 &&259       TileDataTypeIsFloating(output_type) then260        return (261            MatrixFloatingSignedZero(output_type, TRUE),262            activation_flags);263    end;264    if intermediate_width != 0 then265        let (encoded, encoding_flags) = MatrixQuantizedAffine(266            activated, 1.0, offset, intermediate_width,267            output_type, control);268        return (encoded, activation_flags OR encoding_flags);269    end;270    let (encoded, encoding_flags) = MatrixEncodeReal(271        activated + Real(offset), output_type, control);272    return (encoded, activation_flags OR encoding_flags);273end;274
275implementation func TileProfileMatrixPostProcessWithFlags(276    value: Word, pre_quant_mode: bits(6), relu_mode: bits(3),277    group_n_code: bits(4), output_type: TileDataType,278    quant_param: Word, relu_param: Word,279    control: NumericExecutionControl) => (Word, bits(5))280begin281    let effective_control = MatrixFPATREffectiveControl(282        pre_quant_mode, control);283    return MatrixPostQuantBaseWithFlags(284        value, pre_quant_mode, output_type, relu_mode,285        quant_param, relu_param, effective_control);286end;287
288implementation func TileProfileMatrixPostProcess(289    value: Word, pre_quant_mode: bits(6), relu_mode: bits(3),290    group_n_code: bits(4), output_type: TileDataType,291    quant_param: Word, relu_param: Word,292    control: NumericExecutionControl) => Word293begin294    let (result, -) = TileProfileMatrixPostProcessWithFlags(295        value, pre_quant_mode, relu_mode, group_n_code,296        output_type, quant_param, relu_param, control);297    return result;298end;299
300implementation func TileProfileMatrixReductionStep(301    current: Word, candidate: Word, max_abs: boolean,302    data_type: TileDataType) => Word303begin304    let (result, -) = TileProfileMatrixReductionStepWithFlags(305        current, candidate, max_abs, data_type);306    return result;307end;308
309pure func MatrixReductionAbsoluteWithFlags(310    value: Word, data_type: TileDataType) => (Word, bits(5))311begin312    if data_type == TileDataType_U32 then313        return (ZeroExtend{PTO_XLEN}(value[31:0]), Zeros{5});314    elsif data_type == TileDataType_S32 then315        if value[31] == '0' then316            return (SignExtend{PTO_XLEN}(value[31:0]), Zeros{5});317        end;318        let magnitude = Zeros{32} - value[31:0];319        if value[31:0] == '10000000000000000000000000000000' then320            return (Zeros{PTO_XLEN} + 0x7fffffff, Zeros{5} + 4);321        end;322        return (323            SignExtend{PTO_XLEN}(magnitude),324            Zeros{5});325    end;326    let (result, invalid) = TileFixedUnaryValue(327        TileUnary_ABS, data_type, value);328    return (result, if invalid then Zeros{5} + 1 else Zeros{5});329end;330
331implementation func TileProfileMatrixReductionStepWithFlags(332    current: Word, candidate: Word, max_abs: boolean,333    data_type: TileDataType) => (Word, bits(5))334begin335    let (lhs_abs, lhs_flags) = if max_abs then336        MatrixReductionAbsoluteWithFlags(current, data_type)337    else338        (current, Zeros{5});339    let (rhs_abs, rhs_flags) = if max_abs then340        MatrixReductionAbsoluteWithFlags(candidate, data_type)341    else342        (candidate, Zeros{5});343    let lhs = if max_abs then lhs_abs else current;344    let rhs = if max_abs then rhs_abs else candidate;345    let (selected, -, flags) = TileReductionStepWithFlags(346        TileReduction_MAX, data_type, lhs, rhs);347    return (selected, flags OR lhs_flags OR rhs_flags);348end;349