PTO-BLOCK-MODEL-DISPATCH-DESTINATION-SHAPE

1// PTO-UNIT: {"id":"PTO-BLOCK-MODEL-DISPATCH-DESTINATION-SHAPE","surface":"block","classification":["model","dispatch","destination-shape"],"depends_on":["PTO-BLOCK-MODEL-DISPATCH-CELL-REARRANGEMENT-SCHEMA","PTO-BLOCK-MODEL-DISPATCH-DESTINATION-AUXILIARY","PTO-BLOCK-MODEL-DISPATCH-EXPANSION-SCHEMA","PTO-BLOCK-MODEL-DISPATCH-HISTOGRAM-SCHEMA","PTO-BLOCK-MODEL-DISPATCH-NUMERIC-CONTROL","PTO-BLOCK-MODEL-DISPATCH-QUANTIZATION-SCHEMA","PTO-BLOCK-MODEL-DISPATCH-REDUCTION-SCHEMA","PTO-BLOCK-MODEL-DISPATCH-SORTING-SCHEMA","PTO-BLOCK-MODEL-DISPATCH-TCVT-SCHEMA","PTO-BLOCK-MODEL-DISPATCH-TILE-SCALAR-SCHEMA","PTO-TILE-MODEL-STATE-SHARED-REGISTERS"]}2
3readonly func BundleDestinationValidRows(shape_source_valid: boolean,4                                         shape_source: TileIndex)5                                         => integer {0..65535}6begin7    let index = BundleDimensionIndexOfRegister(BundleDimension_LB1);8    if UInt(_BundleDimensions[[index]]) >= 1 &&9       UInt(_BundleDimensions[[index]]) <= 65535 then10        return UInt(_BundleDimensions[[index]]) as integer {1..65535};11    elsif shape_source_valid && TileDescriptorConfigured(shape_source) then12        return _Tiles[[shape_source]].valid_rows;13    elsif BundleSharedBindingCount() == 1 &&14          SharedTileDescriptorLegal(BundleSharedBindingId(0)) then15        return SharedTileRecord(BundleSharedBindingId(0)).tile.valid_rows;16    else17        return 1;18    end;19end;20readonly func BundleDestinationValidColumns(shape_source_valid: boolean,21                                            shape_source: TileIndex)22                                            => integer {0..65535}23begin24    let index = BundleDimensionIndexOfRegister(BundleDimension_LB0);25    if UInt(_BundleDimensions[[index]]) >= 1 &&26       UInt(_BundleDimensions[[index]]) <= 65535 then27        return UInt(_BundleDimensions[[index]]) as integer {1..65535};28    elsif shape_source_valid && TileDescriptorConfigured(shape_source) then29        return _Tiles[[shape_source]].valid_columns;30    elsif BundleSharedBindingCount() == 1 &&31          SharedTileDescriptorLegal(BundleSharedBindingId(0)) then32        return SharedTileRecord(BundleSharedBindingId(0)).tile.valid_columns;33    else34        return 1;35    end;36end;37readonly func BundleDestinationPhysicalColumns(shape_source_valid: boolean,38                                               shape_source: TileIndex)39                                               => integer {0..65535}40begin41    let index = BundleDimensionIndexOfRegister(BundleDimension_LB2);42    if UInt(_BundleDimensions[[index]]) >= 1 &&43       UInt(_BundleDimensions[[index]]) <= 65535 then44        return UInt(_BundleDimensions[[index]]) as integer {1..65535};45    elsif shape_source_valid && TileDescriptorConfigured(shape_source) then46        return _Tiles[[shape_source]].columns;47    elsif BundleSharedBindingCount() == 1 &&48          SharedTileDescriptorLegal(BundleSharedBindingId(0)) then49        return SharedTileRecord(BundleSharedBindingId(0)).tile.columns;50    else51        return BundleDestinationValidColumns(FALSE, 0);52    end;53end;54readonly func BundleLocalDestinationCapacityGroupFits() => boolean55begin56    var additional0: integer = 0;57    var additional1: integer = 0;58    var additional2: integer = 0;59    var additional3: integer = 0;60    for binding = 0 to PTO_BUNDLE_TILE_BINDING_COUNT - 1 do61        if _BundleTileBindings[[binding]].valid &&62           _BundleTileBindings[[binding]].destination_valid &&63           !_BundleTileBindings[[binding]].destination_allocated_by_bundle then64            let capacity_bytes = BundleLocalDestinationAllocationBytes(65                binding as BundleTileBindingIndex);66            let mask = _BundleTileBindings[[binding]].pe_mask;67            if mask[PTOPEMaskBitOfPEIdentity(0)] == '1' then68                additional0 = additional0 + capacity_bytes;69            end;70            if mask[PTOPEMaskBitOfPEIdentity(1)] == '1' then71                additional1 = additional1 + capacity_bytes;72            end;73            if mask[PTOPEMaskBitOfPEIdentity(2)] == '1' then74                additional2 = additional2 + capacity_bytes;75            end;76            if mask[PTOPEMaskBitOfPEIdentity(3)] == '1' then77                additional3 = additional3 + capacity_bytes;78            end;79        end;80    end;81    return TileCapacityInUseForPE(0) + additional0 <=82               TileCapacityLimitBytes() &&83           TileCapacityInUseForPE(1) + additional1 <=84               TileCapacityLimitBytes() &&85           TileCapacityInUseForPE(2) + additional2 <=86               TileCapacityLimitBytes() &&87           TileCapacityInUseForPE(3) + additional3 <=88               TileCapacityLimitBytes();89end;90
91func ResolveBundleTileDestinationsWithShapeAndType(92    explicit_shape: boolean,93    explicit_valid_rows: integer {0..65535},94    explicit_valid_columns: integer {0..65535},95    explicit_columns: integer {0..65535},96    explicit_primary_type: boolean,97    primary_type: TileDataType) => boolean98begin99    let (effective_type_valid, selected_type) =100        ResolveBundleEffectiveDataType();101    if !effective_type_valid then102        SetFault(Fault_TileLegality, ReadTPC());103        return FALSE;104    end;105    var reserved: array [[PTO_TILE_REGISTER_COUNT]] of boolean;106    var resolved: array [[PTO_BUNDLE_TILE_BINDING_COUNT]] of TileIndex;107    for index = 0 to PTO_TILE_REGISTER_COUNT - 1 do108        reserved[[index]] = FALSE;109    end;110    for binding = 0 to PTO_BUNDLE_TILE_BINDING_COUNT - 1 do111        resolved[[binding]] = 0;112        if _BundleTileBindings[[binding]].valid &&113           _BundleTileBindings[[binding]].destination_valid &&114            !_BundleTileBindings[[binding]].destination_allocated_by_bundle then115            let hand =116                UInt(_BundleTileBindings[[binding]].destination_hand);117            var found = FALSE;118            for offset = 0 to 15 do119                let raw_index: integer = hand * 16 + offset;120                if !found && !_Tiles[[raw_index]].allocated &&121                   !reserved[[raw_index]] then122                    resolved[[binding]] = raw_index as TileIndex;123                    reserved[[raw_index]] = TRUE;124                    found = TRUE;125                end;126            end;127            if !found then128                SetFault(Fault_TileAllocation, ReadTPC());129                return FALSE;130            end;131        end;132    end;133    if !BundleLocalDestinationCapacityGroupFits() then134        SetFault(Fault_TileAllocation, ReadTPC());135        return FALSE;136    end;137
138    let matrix = _BundleOperation.valid &&139        _BundleOperation.operation_class == BundleOperation_TileMatrix;140    let accumulator_type = TileMatrixAccumulatorDataType(selected_type);141    let matrix_output_type = if matrix &&142        UInt(_BundleFixedPointAttributes.pre_quant_mode) == 0 then143        accumulator_type144    else if matrix then145        BundleFPATROutputType(_BundleFixedPointAttributes.pre_quant_mode)146    else selected_type;147    let primary_output_type = if explicit_primary_type then148        primary_type149    else150        matrix_output_type;151    var shape_source_valid = FALSE;152    var shape_source: TileIndex = 0;153    for binding = 0 to PTO_BUNDLE_TILE_BINDING_COUNT - 1 do154        if !shape_source_valid && _BundleTileBindings[[binding]].valid then155            if _BundleTileBindings[[binding]].source0_valid then156                shape_source = BundleTileSourceIndex(157                    binding as BundleTileBindingIndex, FALSE);158                shape_source_valid = TRUE;159            elsif _BundleTileBindings[[binding]].source1_valid then160                shape_source = BundleTileSourceIndex(161                    binding as BundleTileBindingIndex, TRUE);162                shape_source_valid = TRUE;163            end;164        end;165    end;166
167    // Validate every derived descriptor before allocating any destination.168    // This preserves precise all-or-nothing B.IOT allocation when a size code169    // is too small for its logical shape.170    var destination_ordinal: integer = 0;171    for binding = 0 to PTO_BUNDLE_TILE_BINDING_COUNT - 1 do172        if _BundleTileBindings[[binding]].valid &&173           _BundleTileBindings[[binding]].destination_valid &&174           !_BundleTileBindings[[binding]].destination_allocated_by_bundle then175            let valid_rows = if explicit_shape then explicit_valid_rows else176                BundleDestinationValidRows(shape_source_valid, shape_source);177            let valid_columns = if explicit_shape then178                explicit_valid_columns else BundleDestinationValidColumns(179                    shape_source_valid, shape_source);180            let columns = if explicit_shape then explicit_columns else181                BundleDestinationPhysicalColumns(182                    shape_source_valid, shape_source);183            let destination_type = if destination_ordinal == 0 then184                primary_output_type else if matrix then accumulator_type185                else TileDataType_U32;186            let auxiliary_row = matrix &&187                _BundleFixedPointAttributes.row_max_en &&188                destination_ordinal == 1;189            let auxiliary_group = matrix &&190                _BundleFixedPointAttributes.group_max_en &&191                ((!_BundleFixedPointAttributes.row_max_en &&192                  destination_ordinal == 1) ||193                 (_BundleFixedPointAttributes.row_max_en &&194                  destination_ordinal == 2));195            let auxiliary_columns = if auxiliary_row then 1196                else if auxiliary_group then197                    BundleGroupMaxColumns(columns)198                else columns;199            let auxiliary_valid_columns = if auxiliary_row then 1200                else if auxiliary_group then201                    BundleGroupMaxColumns(valid_columns)202                else valid_columns;203            let capacity_bytes = BundleLocalDestinationAllocationBytes(204                binding as BundleTileBindingIndex);205            let rows = DerivedTileRows(capacity_bytes, auxiliary_columns,206                destination_type);207            if !TileDescriptorShapeLegal(capacity_bytes, auxiliary_columns,208                   valid_rows, auxiliary_valid_columns, destination_type) ||209               rows * auxiliary_columns >210                   TileLogicalElementCapacity(capacity_bytes,211                       destination_type) then212                SetFault(Fault_TileAllocation, ReadTPC());213                return FALSE;214            end;215            destination_ordinal = destination_ordinal + 1;216        end;217    end;218
219    destination_ordinal = 0;220    for binding = 0 to PTO_BUNDLE_TILE_BINDING_COUNT - 1 do221        if _BundleTileBindings[[binding]].valid &&222           _BundleTileBindings[[binding]].destination_valid &&223           !_BundleTileBindings[[binding]].destination_allocated_by_bundle then224            let valid_rows = if explicit_shape then explicit_valid_rows else225                BundleDestinationValidRows(shape_source_valid, shape_source);226            let valid_columns = if explicit_shape then227                explicit_valid_columns else BundleDestinationValidColumns(228                    shape_source_valid, shape_source);229            let columns = if explicit_shape then explicit_columns else230                BundleDestinationPhysicalColumns(231                    shape_source_valid, shape_source);232            let destination_type = if destination_ordinal == 0 then233                primary_output_type else if matrix then accumulator_type234                else TileDataType_U32;235            let auxiliary_row = matrix &&236                _BundleFixedPointAttributes.row_max_en &&237                destination_ordinal == 1;238            let auxiliary_group = matrix &&239                _BundleFixedPointAttributes.group_max_en &&240                ((!_BundleFixedPointAttributes.row_max_en &&241                  destination_ordinal == 1) ||242                 (_BundleFixedPointAttributes.row_max_en &&243                  destination_ordinal == 2));244            let auxiliary_columns = if auxiliary_row then 1245                else if auxiliary_group then246                    BundleGroupMaxColumns(columns)247                else columns;248            let auxiliary_valid_columns = if auxiliary_row then 1249                else if auxiliary_group then250                    BundleGroupMaxColumns(valid_columns)251                else valid_columns;252            let capacity_bytes = BundleLocalDestinationAllocationBytes(253                binding as BundleTileBindingIndex);254            ConfigureTileForMask(resolved[[binding]],255                capacity_bytes, valid_rows, auxiliary_columns, valid_rows,256                auxiliary_valid_columns, destination_type,257                CurrentBundleTileLayout(), TileLocation_Any,258                _BundleTileBindings[[binding]].pe_mask);259            _BundleTileBindings[[binding]].destination = resolved[[binding]];260            _BundleTileBindings[[binding]].destination_allocated_by_bundle =261                TRUE;262            destination_ordinal = destination_ordinal + 1;263        end;264    end;265    return TRUE;266end;267
268func ResolveBundleTileDestinationsWithShape(269    explicit_shape: boolean,270    explicit_valid_rows: integer {0..65535},271    explicit_valid_columns: integer {0..65535},272    explicit_columns: integer {0..65535}) => boolean273begin274    return ResolveBundleTileDestinationsWithShapeAndType(275        explicit_shape,276        explicit_valid_rows,277        explicit_valid_columns,278        explicit_columns,279        FALSE,280        TileDataType_FP64);281end;282
283func ResolveBundleTileDestinations() => boolean284begin285    return ResolveBundleTileDestinationsWithShape(FALSE, 0, 0, 0);286end;287
288func ResolveBundlePredicateDestination() => boolean289begin290    var destination_binding: BundleTileBindingIndex = 0;291    var destination_seen = FALSE;292    for binding = 0 to PTO_BUNDLE_TILE_BINDING_COUNT - 1 do293        if !destination_seen &&294           _BundleTileBindings[[binding]].valid &&295           _BundleTileBindings[[binding]].destination_valid then296            destination_binding = binding as BundleTileBindingIndex;297            destination_seen = TRUE;298        end;299    end;300    if !destination_seen then301        SetFault(Fault_BundleControl, ReadTPC());302        return FALSE;303    end;304
305    let binding = _BundleTileBindings[[destination_binding]];306    let source = binding.source0;307    let source_tile = _Tiles[[source]];308    let capacity_bytes = BundleLocalDestinationAllocationBytes(309        destination_binding);310    if source_tile.storage_kind != TileStorage_Numeric ||311       source_tile.rows * source_tile.columns >312           TileLogicalElementCapacity(source_tile.capacity_bytes,313               source_tile.data_type) ||314       PredicateTileStorageBytes(315           source_tile.rows,316           source_tile.columns) > capacity_bytes then317        SetFault(Fault_TileAllocation, ReadTPC());318        return FALSE;319    end;320    if !LocalTileAllocationFits(binding.pe_mask, capacity_bytes) then321        SetFault(Fault_TileAllocation, ReadTPC());322        return FALSE;323    end;324
325    let hand = UInt(binding.destination_hand);326    var resolved: TileIndex = 0;327    var found = FALSE;328    for offset = 0 to 15 do329        let raw_index: integer = hand * 16 + offset;330        if !found && !_Tiles[[raw_index]].allocated then331            resolved = raw_index as TileIndex;332            found = TRUE;333        end;334    end;335    if !found then336        SetFault(Fault_TileAllocation, ReadTPC());337        return FALSE;338    end;339
340    ConfigurePredicateTileForMask(341        resolved,342        capacity_bytes,343        source_tile.rows,344        source_tile.columns,345        source_tile.valid_rows,346        source_tile.valid_columns,347        binding.pe_mask);348    _BundleTileBindings[[destination_binding]].destination = resolved;349    _BundleTileBindings[[destination_binding]].destination_allocated_by_bundle =350        TRUE;351    return TRUE;352end;353
354func ResolveBundleTileDestinationsForOperation(355    operation: integer {0..PTO_TILE_OPERATION_COUNT-1}) => boolean356begin357    // CUBE matrix handlers own the primary CUBE destination shape and358    // atomic allocation group.  Leave the binding unresolved here so the359    // selected handler can apply its authoritative M/N/layout/type rules;360    // the generic RowMajor resolver would mark the destination as already361    // allocated and prevent that conversion.362    if _BundleOperation.valid &&363       _BundleOperation.operation_class == BundleOperation_TileMatrix then364        return TRUE;365    end;366    let decoded_operation = TileOperationOfIndex(operation);367    if decoded_operation == TileOperation_TPERMUTE ||368       decoded_operation == TileOperation_TSHUF ||369       decoded_operation == TileOperation_TPACK ||370       decoded_operation == TileOperation_TUNPACK then371        return ResolveBundleCellRearrangementDestination();372    end;373    if TileOperationOfIndex(operation) == TileOperation_TIMG2COL then374        let resolved = ResolveBundleTileDestinations();375        if resolved then376            MarkBundleTIMG2COLDestinationsMatrix();377        end;378        return resolved;379    end;380    if TileOperationOfIndex(operation) == TileOperation_TCONCAT then381        let (legal, valid_rows, valid_columns, physical_columns,382             data_type) = BundleTCONCATDestinationShape();383        if !legal then384            SetFault(Fault_TileAllocation, ReadTPC());385            return FALSE;386        end;387        return ResolveBundleTileDestinationsWithShapeAndType(388            TRUE, valid_rows, valid_columns, physical_columns,389            TRUE, data_type);390    end;391    if TileOperationUsesClosedSortingSchema(operation) then392        let decoded = TileOperationOfIndex(operation);393        let source_left = BundleSortingSourceAt(0);394        if decoded == TileOperation_TSORT then395            let source = _Tiles[[source_left]];396            return ResolveBundleTileDestinationsWithShapeAndType(397                TRUE,398                source.valid_rows,399                source.valid_columns,400                source.columns,401                TRUE,402                source.data_type);403        end;404        let source_right = BundleSortingSourceAt(1);405        let output_columns =406            _Tiles[[source_left]].valid_columns +407            _Tiles[[source_right]].valid_columns;408        if output_columns < 1 || output_columns > 32768 then409            SetFault(Fault_TileAllocation, ReadTPC());410            return FALSE;411        end;412        let physical_columns = SmallestTilePhysicalColumns(413            output_columns as integer {1..65535});414        if physical_columns == 0 then415            SetFault(Fault_TileAllocation, ReadTPC());416            return FALSE;417        end;418        return ResolveBundleTileDestinationsWithShapeAndType(419            TRUE,420            1,421            output_columns as integer {1..65535},422            physical_columns,423            TRUE,424            _Tiles[[source_left]].data_type);425    end;426    if TileOperationUsesClosedHistogramSchema(operation) then427        let source = _BundleTileBindings[[0]].source0;428        return ResolveBundleTileDestinationsWithShape(429            TRUE,430            _Tiles[[source]].valid_rows,431            256,432            256);433    end;434    if TileOperationUsesClosedReductionSchema(operation) then435        let source = _BundleTileBindings[[0]].source0;436        let source_tile = _Tiles[[source]];437        let row_reduction =438            TileOperationUsesClosedRowReductionSchema(operation);439        let destination_type =440            if TileReductionOperationReturnsIndex(operation) then441                TileDataType_U32442            else443                source_tile.data_type;444        let valid_rows =445            if row_reduction then source_tile.valid_rows else 1;446        let valid_columns =447            if row_reduction then 1 else source_tile.valid_columns;448        let columns =449            if row_reduction then 1 else source_tile.columns;450        return ResolveBundleTileDestinationsWithShapeAndType(451            TRUE,452            valid_rows,453            valid_columns,454            columns,455            TRUE,456            destination_type);457    end;458    if TileOperationUsesClosedTCMPSchema(operation) ||459       TileOperationUsesClosedTCMPSSchema(operation) then460        return ResolveBundlePredicateDestination();461    end;462    if TileExpansionOperationIsExponentialDifference(operation) then463        let (types_legal, -, destination_type) =464            SelectedBundleExpansionExponentialDifferenceTypes();465        if !types_legal then466            SetFault(Fault_TileLegality, ReadTPC());467            return FALSE;468        end;469        let valid_columns = UInt(_BundleDimensions[[0]])470            as integer {1..65535};471        let valid_rows = if _BundleDimensionPresent[[1]] then472            UInt(_BundleDimensions[[1]]) as integer {1..65535} else 1;473        let columns = if _BundleDimensionPresent[[2]] then474            UInt(_BundleDimensions[[2]]) as integer {1..65535}475            else valid_columns;476        return ResolveBundleTileDestinationsWithShapeAndType(477            TRUE, valid_rows, valid_columns, columns,478            TRUE, destination_type);479    end;480    if !TileOperationUsesClosedBinarySchema(operation) &&481       !TileOperationUsesClosedUnarySchema(operation) &&482       !TileOperationUsesClosedTFMASchema(operation) &&483       !TileOperationUsesClosedQuantizationSchema(operation) &&484       !TileOperationUsesClosedGenerationSchema(operation) &&485       !TileOperationUsesClosedExpansionSchema(operation) &&486       !TileOperationUsesClosedTCVTSchema(operation) &&487       !TileOperationUsesClosedComparisonSchema(operation) &&488       !TileOperationUsesClosedTileScalarSchema(operation) then489        return ResolveBundleTileDestinations();490    end;491    let valid_columns = UInt(_BundleDimensions[[0]])492        as integer {1..65535};493    let valid_rows = if _BundleDimensionPresent[[1]] then494        UInt(_BundleDimensions[[1]]) as integer {1..65535} else 1;495    let columns = if _BundleDimensionPresent[[2]] then496        UInt(_BundleDimensions[[2]]) as integer {1..65535}497        else valid_columns;498    return ResolveBundleTileDestinationsWithShape(499        TRUE, valid_rows, valid_columns, columns);500end;501