The complete ASL owner is shown directly below.
12345678910
11pure func MatrixQuantParameter(fp19: bits(19), offset: Word,12 offset_width: integer {0,5,9,17}) => Word13begin14 var result = Zeros{PTO_XLEN};15 result[31:13] = fp19;16 case offset_width of17 when 0 => return result;18 when 5 => result[41:37] = offset[4:0];19 when 9 => result[45:37] = offset[8:0];20 when 17 => result[53:37] = offset[16:0];21 end;22 return result;23end;24
25pure func MatrixShiftParameter(code: integer {0..15}) => Word26begin27 var result = Zeros{PTO_XLEN};28 result[35:32] = Zeros{4} + code;29 return result;30end;31
32pure func MatrixQuantOffset(parameter: Word,33 width: integer {0,5,9,17}) => integer34begin35 case width of36 when 0 => return 0;37 when 5 => return SInt(parameter[41:37]);38 when 9 => return SInt(parameter[45:37]);39 when 17 => return SInt(parameter[53:37]);40 end;41end;42
43pure func MatrixMagnitudeRoundingMode(44 mode: NumericRoundingMode, negative: boolean) => NumericRoundingMode45begin46 if !negative then return mode; end;47 if mode == NumericRound_RTP then return NumericRound_RTM;48 elsif mode == NumericRound_RTM then return NumericRound_RTP;49 end;50 return mode;51end;52
53func MatrixRoundMagnitude(54 value: real, mode: NumericRoundingMode,55 negative: boolean) => integer56begin57 if negative && mode == NumericRound_RHB then58 let lower = RoundDown(value);59 let fraction = value - Real(lower);60 if fraction <= 0.5 then return lower;61 else return lower + 1;62 end;63 end;64 return FloatingToInteger(65 value, MatrixMagnitudeRoundingMode(mode, negative));66end;67
68func MatrixRoundAndSaturateSigned(69 value: real, width: integer {5,9,17},70 rounding_mode: NumericRoundingMode)71 => (integer {-65536..65535}, bits(5))72begin73 let rounded = FloatingToInteger(value, rounding_mode);74 let minimum = if width == 5 then -1675 else if width == 9 then -25676 else -65536;77 let maximum = if width == 5 then 1578 else if width == 9 then 25579 else 65535;80 let overflow = rounded < minimum || rounded > maximum;81 var selected = rounded;82 if selected < minimum then selected = minimum;83 elsif selected > maximum then selected = maximum;84 end;85 let flags = if overflow then Zeros{5} + 0x1486 else if Real(rounded) != value then Zeros{5} + 0x1087 else Zeros{5};88 return (89 selected as integer {-65536..65535},90 flags);91end;92
93func MatrixShiftS32ToS16(94 value: bits(32), shift: integer {1..16}) => (Word, bits(5))95begin96 let shifted = SInt(ASR(value, shift));97 if shifted < -32768 then98 return (Zeros{PTO_XLEN} + 0xffffffffffff8000,99 Zeros{5} + 0x14);100 elsif shifted > 32767 then101 return (Zeros{PTO_XLEN} + 0x7fff, Zeros{5} + 0x14);102 end;103 return (104 SignExtend{PTO_XLEN}(Zeros{16} + shifted),105 Zeros{5});106end;107
108func ReferenceMatrixIntegerEncoding(109 value: real, destination_type: TileDataType,110 control: NumericExecutionControl) => (Word, bits(5))111begin112 let rounded = FloatingToInteger(value, control.rounding_mode);113 let minimum = ReferenceIntegerValue(114 TileIntegerMinimum(destination_type), destination_type);115 let maximum = ReferenceIntegerValue(116 TileIntegerMaximum(destination_type), destination_type);117 let overflow = rounded < minimum || rounded > maximum;118 var selected = rounded;119 if control.saturating && overflow then120 if selected < minimum then selected = minimum;121 else selected = maximum;122 end;123 end;124 let flags = if overflow then Zeros{5} + 0x14125 else if Real(rounded) != value then Zeros{5} + 0x10126 else Zeros{5};127 return (128 NormalizeTileInteger(Zeros{PTO_XLEN} + selected,129 destination_type),130 flags);131end;132
133func MatrixQuantizedAffine(134 value: real, scale: real, offset: integer,135 intermediate_width: integer {0,5,9,17},136 output_type: TileDataType,137 control: NumericExecutionControl) => (Word, bits(5))138begin139 if intermediate_width == 0 then140 return ReferenceMatrixIntegerEncoding(141 value * scale + Real(offset), output_type, control);142 end;143 let width = intermediate_width as integer {5,9,17};144 let (intermediate, intermediate_flags) =145 MatrixRoundAndSaturateSigned(146 value * scale, width, control.rounding_mode);147 let (encoded, final_flags) = ReferenceMatrixIntegerEncoding(148 Real(intermediate + offset), output_type, control);149 return (encoded, intermediate_flags OR final_flags);150end;151
152func ReferenceBinary16Encoding(153 value: real, destination_type: TileDataType,154 control: NumericExecutionControl) => (Word, bits(5))155begin156 assert destination_type == TileDataType_FP16 ||157 destination_type == TileDataType_BF16;158 if value == 0.0 then return (Zeros{PTO_XLEN}, Zeros{5}); end;159 let negative = value < 0.0;160 var normalized = if negative then -value else value;161 var exponent: integer {-149..128} = 0;162 for step = 1 to 128 looplimit 128 do163 if normalized >= 2.0 && exponent < 128 then164 normalized = normalized / 2.0;165 exponent = (exponent + 1) as integer {-149..128};166 end;167 end;168 for step = 1 to 149 looplimit 149 do169 if normalized < 1.0 && exponent > -149 then170 normalized = normalized * 2.0;171 exponent = (exponent - 1) as integer {-149..128};172 end;173 end;174
175 let bf16 = destination_type == TileDataType_BF16;176 let fraction_bits = if bf16 then 7 else 10;177 let fraction_scale = if bf16 then 128 else 1024;178 let bias = if bf16 then 127 else 15;179 let maximum_exponent = if bf16 then 127 else 15;180 let minimum_exponent = if bf16 then -126 else -14;181 let minimum_subnormal_exponent = if bf16 then -133 else -24;182 let sign = if negative then 0x8000 else 0;183
184 if exponent > maximum_exponent then185 let overflow = if control.saturating then186 (if bf16 then 0x7f7f else 0x7bff)187 else188 (if bf16 then 0x7f80 else 0x7c00);189 return (Zeros{PTO_XLEN} + sign + overflow,190 Zeros{5} + 0x14);191 end;192
193 if exponent < minimum_exponent then194 let scaled = (if negative then -value else value) /195 ReferencePowerOfTwo(196 minimum_subnormal_exponent as integer {-149..127});197 var rounded = MatrixRoundMagnitude(198 scaled, control.rounding_mode, negative);199 if rounded >= fraction_scale then200 return (Zeros{PTO_XLEN} + sign + fraction_scale,201 if Real(rounded) == scaled then Zeros{5}202 else Zeros{5} + 0x18);203 end;204 if rounded < 0 then rounded = 0; end;205 return (Zeros{PTO_XLEN} + sign + rounded,206 if Real(rounded) == scaled then Zeros{5}207 else Zeros{5} + 0x18);208 end;209
210 let scaled = normalized * Real(fraction_scale);211 var rounded = MatrixRoundMagnitude(212 scaled, control.rounding_mode, negative);213 var encoded_exponent = exponent + bias;214 if rounded == 2 * fraction_scale then215 rounded = fraction_scale;216 assert encoded_exponent <= 254;217 encoded_exponent =218 (encoded_exponent + 1) as integer {-134..255};219 end;220 if encoded_exponent >= 2 * bias + 1 then221 let overflow = if control.saturating then222 (if bf16 then 0x7f7f else 0x7bff)223 else224 (if bf16 then 0x7f80 else 0x7c00);225 return (Zeros{PTO_XLEN} + sign + overflow,226 Zeros{5} + 0x14);227 end;228 let fraction = rounded - fraction_scale;229 let encoded = encoded_exponent * fraction_scale + fraction;230 return (Zeros{PTO_XLEN} + sign + encoded,231 if Real(rounded) == scaled then Zeros{5}232 else Zeros{5} + 0x10);233end;234
235pure func ReferenceBinary16FiniteValue(236 value: Word, data_type: TileDataType) => real237begin238 assert data_type == TileDataType_FP16 || data_type == TileDataType_BF16;239 let bf16 = data_type == TileDataType_BF16;240 let sign = value[15];241 let exponent = if bf16 then UInt(value[14:7])242 else UInt(value[14:10]);243 let fraction = if bf16 then UInt(value[6:0])244 else UInt(value[9:0]);245 let fraction_scale = if bf16 then 128 else 1024;246 let bias = if bf16 then 127 else 15;247 let minimum_subnormal_exponent = if bf16 then -133 else -24;248 let maximum_field = if bf16 then 255 else 31;249 assert exponent != maximum_field;250 var magnitude: real = 0.0;251 if exponent == 0 then252 magnitude = Real(fraction) * ReferencePowerOfTwo(253 minimum_subnormal_exponent as integer {-149..127});254 else255 magnitude = (1.0 + Real(fraction) / Real(fraction_scale)) *256 ReferencePowerOfTwo(257 (exponent - bias) as integer {-126..127});258 end;259 if sign == '1' then return -magnitude; end;260 return magnitude;261end;262
263pure func ReferenceFP8FiniteValue(264 data_type: TileDataType, code: bits(8)) => real265begin266 assert data_type == TileDataType_E4M3 ||267 data_type == TileDataType_HiF8;268 let negative = code[7] == '1';269 var magnitude: real = 0.0;270 if data_type == TileDataType_E4M3 then271 let exponent = UInt(code[6:3]);272 let fraction = UInt(code[2:0]);273 if exponent == 0 then274 magnitude = Real(fraction) * ReferencePowerOfTwo(-9);275 else276 magnitude = (1.0 + Real(fraction) / 8.0) *277 ReferencePowerOfTwo(278 (exponent - 7) as integer {-6..8});279 end;280 else281 let body = UInt(code[6:0]);282 if body <= 7 then283 if body == 0 then magnitude = 0.0;284 else285 magnitude = ReferencePowerOfTwo(286 (body - 23) as integer {-22..-16});287 end;288 elsif body <= 15 then289 magnitude = 1.0 + Real(body - 8) / 8.0;290 elsif body <= 31 then291 let exponent_code = UInt(code[3]);292 let exponent = if exponent_code == 0 then 1 else -1;293 magnitude = (1.0 + Real(UInt(code[2:0])) / 8.0) *294 ReferencePowerOfTwo(exponent as integer {-1..1});295 elsif body <= 63 then296 let exponent_code = UInt(code[4:3]);297 let absolute = 2 + UInt(code[3]);298 let exponent = if exponent_code < 2 then absolute else -absolute;299 magnitude = (1.0 + Real(UInt(code[2:0])) / 8.0) *300 ReferencePowerOfTwo(exponent as integer {-3..3});301 elsif body <= 95 then302 let exponent_code = UInt(code[4:2]);303 let absolute = 4 + UInt(code[3:2]);304 let exponent = if exponent_code < 4 then absolute else -absolute;305 magnitude = (1.0 + Real(UInt(code[1:0])) / 4.0) *306 ReferencePowerOfTwo(exponent as integer {-7..7});307 else308 let exponent_code = UInt(code[4:1]);309 let absolute = 8 + UInt(code[3:1]);310 let exponent = if exponent_code < 8 then absolute else -absolute;311 magnitude = (1.0 + Real(UInt(code[0])) / 2.0) *312 ReferencePowerOfTwo(exponent as integer {-15..15});313 end;314 end;315 if negative then return -magnitude; end;316 return magnitude;317end;318
319pure func ReferenceNearestFP8CandidateBetter(320 target: real, candidate: real, candidate_code: integer {0..255},321 best: real, best_code: integer {0..255},322 mode: NumericRoundingMode) => boolean323begin324 let candidate_distance = if candidate >= target then325 candidate - target else target - candidate;326 let best_distance = if best >= target then best - target else target - best;327 if candidate_distance < best_distance then return TRUE;328 elsif candidate_distance > best_distance then return FALSE;329 end;330 if mode == NumericRound_RNE then331 return candidate_code MOD 2 == 0 && best_code MOD 2 != 0;332 elsif mode == NumericRound_RNA then333 let candidate_magnitude = if candidate < 0.0 then -candidate else candidate;334 let best_magnitude = if best < 0.0 then -best else best;335 return candidate_magnitude > best_magnitude;336 elsif mode == NumericRound_RTO then337 return candidate_code MOD 2 != 0 && best_code MOD 2 == 0;338 else339 return candidate > best;340 end;341end;342
343func ReferenceFP8Encoding(344 value: real, destination_type: TileDataType,345 control: NumericExecutionControl) => (Word, bits(5))346begin347 assert destination_type == TileDataType_E4M3 ||348 destination_type == TileDataType_HiF8;349 if value == 0.0 then return (Zeros{PTO_XLEN}, Zeros{5}); end;350 let negative = value < 0.0;351 let magnitude = if negative then -value else value;352 let maximum = if destination_type == TileDataType_E4M3 then353 448.0 else 32768.0;354 if magnitude > maximum then355 let result = if control.saturating then356 (if destination_type == TileDataType_E4M3 then357 (if negative then 0xfe else 0x7e)358 else (if negative then 0xee else 0x6e))359 else if destination_type == TileDataType_E4M3 then 0x7f360 else if negative then 0xef else 0x6f;361 return (Zeros{PTO_XLEN} + result, Zeros{5} + 0x14);362 end;363
364 var best_set = FALSE;365 var best_code: integer {0..255} = 0;366 var best_value: real = 0.0;367 for code = 0 to 255 do368 let candidate_bits = Zeros{8} + code;369 let value_class = TileNumericValueClass(370 destination_type, Zeros{PTO_XLEN} + code);371 if !NumericValueClassIsNaN(value_class) &&372 !NumericValueClassIsInfinity(value_class) &&373 value_class != NumericValue_InvalidEncoding then374 let candidate = ReferenceFP8FiniteValue(375 destination_type, candidate_bits);376 var eligible = TRUE;377 if control.rounding_mode == NumericRound_RTP then378 eligible = candidate >= value;379 elsif control.rounding_mode == NumericRound_RTM then380 eligible = candidate <= value;381 elsif control.rounding_mode == NumericRound_RTZ ||382 control.rounding_mode == NumericRound_RTO then383 eligible = if negative then candidate <= 0.0 && candidate >= value384 else candidate >= 0.0 && candidate <= value;385 end;386 if eligible then387 var better = !best_set;388 if best_set then389 if control.rounding_mode == NumericRound_RTP then390 better = candidate < best_value;391 elsif control.rounding_mode == NumericRound_RTM then392 better = candidate > best_value;393 elsif control.rounding_mode == NumericRound_RTZ ||394 control.rounding_mode == NumericRound_RTO then395 better = if negative then candidate < best_value396 else candidate > best_value;397 else398 better = ReferenceNearestFP8CandidateBetter(399 value, candidate, code, best_value, best_code,400 control.rounding_mode);401 end;402 end;403 if better then404 best_set = TRUE;405 best_code = code;406 best_value = candidate;407 end;408 end;409 end;410 end;411 assert best_set;412
413 if control.rounding_mode == NumericRound_RTO && best_value != value &&414 best_code MOD 2 == 0 then415 var odd_set = FALSE;416 var odd_code: integer {0..255} = best_code;417 var odd_value: real = best_value;418 for code = 0 to 255 do419 if code MOD 2 == 1 then420 let candidate_class = TileNumericValueClass(421 destination_type, Zeros{PTO_XLEN} + code);422 if !NumericValueClassIsNaN(candidate_class) &&423 !NumericValueClassIsInfinity(candidate_class) then424 let candidate = ReferenceFP8FiniteValue(425 destination_type, Zeros{8} + code);426 let away = if negative then candidate < best_value427 else candidate > best_value;428 if away && (!odd_set ||429 (if negative then candidate > odd_value430 else candidate < odd_value)) then431 odd_set = TRUE;432 odd_code = code;433 odd_value = candidate;434 end;435 end;436 end;437 end;438 if odd_set then439 best_code = odd_code;440 best_value = odd_value;441 end;442 end;443
444 let minimum_normal = if destination_type == TileDataType_E4M3 then445 ReferencePowerOfTwo(-6) else ReferencePowerOfTwo(-15);446 let inexact = best_value != value;447 let underflow = inexact && magnitude < minimum_normal;448 return (Zeros{PTO_XLEN} + best_code,449 if underflow then Zeros{5} + 0x18450 else if inexact then Zeros{5} + 0x10451 else Zeros{5});452end;453
454func ReferenceMatrixFloatingEncoding(455 value: real, destination_type: TileDataType,456 control: NumericExecutionControl) => (Word, bits(5))457begin458 if destination_type == TileDataType_FP32 then459 let (result, flags) = ReferenceFP32FiniteEncoding(460 value, control.rounding_mode);461 if control.saturating && (flags AND (Zeros{5} + 4)) != Zeros{5} then462 let sign = result AND (Zeros{PTO_XLEN} + 0x80000000);463 return (sign OR (Zeros{PTO_XLEN} + 0x7f7fffff), flags);464 end;465 return (result, flags);466 elsif destination_type == TileDataType_FP16 ||467 destination_type == TileDataType_BF16 then468 return ReferenceBinary16Encoding(value, destination_type, control);469 else470 return ReferenceFP8Encoding(value, destination_type, control);471 end;472end;473