diff --git a/libs/vulcan-gpu/abi.zig b/libs/vulcan-gpu/abi.zig index 20a830a..f8515f8 100644 --- a/libs/vulcan-gpu/abi.zig +++ b/libs/vulcan-gpu/abi.zig @@ -370,7 +370,7 @@ pub fn layoutSharedFrame(allocator: std.mem.Allocator, func: *const Function, a: const al = switch (func.opcode(inst)) { .alloca => |al| al, .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select => continue, - .struct_new, .extract, .convert, .unary, .global_addr => continue, + .struct_new, .extract, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .global_addr => continue, .call, .call_indirect, .load, .store, .prefetch, .@"if" => continue, .va_start, .va_arg, .va_end, .dot, .matmul, .barrier, .atomic_rmw => continue, }; diff --git a/libs/vulcan-gpu/offload.zig b/libs/vulcan-gpu/offload.zig index f0bb531..688f2e8 100644 --- a/libs/vulcan-gpu/offload.zig +++ b/libs/vulcan-gpu/offload.zig @@ -158,6 +158,13 @@ pub fn lowerToLoopNest( block: [3]u32, ) Error!Function { if (func.blockCount() == 0) return error.Unsupported; + for (0..func.blockCount()) |bi| { + for (func.blockInsts(@enumFromInt(bi))) |inst| switch (func.opcode(inst)) { + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, + else => {}, + }; + } if (returnsValue(func)) return error.Unsupported; if (entryIsBranchTarget(func)) return error.Unsupported; if (hasBlockAttribute(func)) return error.Unsupported; @@ -726,6 +733,58 @@ test "a value-returning kernel is rejected" { ); } +test "both low float directions are rejected before loop-nest construction" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |decode_direction| { + var kernel = Function.init(allocator); + defer kernel.deinit(); + const u16_t = try kernel.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const f32_t = try kernel.types.intern(.{ .float = .f32 }); + const ptr_t = try kernel.types.ptrGlobal(); + const entry = try kernel.appendBlock(); + const source = try kernel.appendBlockParam(entry, if (decode_direction) u16_t else f32_t); + const output = try kernel.appendBlockParam(entry, ptr_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = .bf16 }; + const result = try kernel.appendInst(entry, if (decode_direction) f32_t else u16_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + try kernel.appendStore(entry, result, output); + kernel.setTerminator(entry, .{ .ret = ir.function.Ret.none() }); + try std.testing.expectError(error.Unsupported, lowerToLoopNest(allocator, &kernel, .{ 1, 1, 1 })); + } +} + +test "both nvfp4 directions are rejected before loop-nest construction" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |dequantize| { + var kernel = Function.init(allocator); + defer kernel.deinit(); + const u8_t = try kernel.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try kernel.types.intern(.{ .float = .f32 }); + const ptr_t = try kernel.types.ptrGlobal(); + const entry = try kernel.appendBlock(); + const payload = try kernel.appendBlockParam(entry, u8_t); + const block_scale = try kernel.appendBlockParam(entry, u8_t); + const value = try kernel.appendBlockParam(entry, f32_t); + const global_scale = try kernel.appendBlockParam(entry, f32_t); + const output = try kernel.appendBlockParam(entry, ptr_t); + const conversion: ir.function.NvFp4Convert = .{ + .value = if (dequantize) payload else value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + }; + const result = try kernel.appendInst(entry, if (dequantize) f32_t else u8_t, if (dequantize) .{ .dequantize_nvfp4 = conversion } else .{ .quantize_nvfp4 = conversion }); + try kernel.appendStore(entry, result, output); + kernel.setTerminator(entry, .{ .ret = ir.function.Ret.none() }); + const before = try ir.bitcode.encode(allocator, &kernel); + defer allocator.free(before); + try std.testing.expectError(error.Unsupported, lowerToLoopNest(allocator, &kernel, .{ 1, 1, 1 })); + const after = try ir.bitcode.encode(allocator, &kernel); + defer allocator.free(after); + try std.testing.expectEqualSlices(u8, before, after); + } +} + test "a builtin parameter that is not a 32-bit integer is rejected" { // The induction variables are i32, and an arithmetic op needs both operands to share a // type, so a wider builtin has no correct lowering here. diff --git a/libs/vulcan-ir.zig b/libs/vulcan-ir.zig index a51f4e1..ae7d264 100644 --- a/libs/vulcan-ir.zig +++ b/libs/vulcan-ir.zig @@ -6,6 +6,8 @@ const std = @import("std"); pub const entity = @import("vulcan-ir/entity.zig"); pub const types = @import("vulcan-ir/types.zig"); +pub const low_float = @import("vulcan-ir/low_float.zig"); +pub const nvfp4 = @import("vulcan-ir/nvfp4.zig"); pub const function = @import("vulcan-ir/function.zig"); pub const builder = @import("vulcan-ir/builder.zig"); pub const parser = @import("vulcan-ir/parser.zig"); diff --git a/libs/vulcan-ir/bitcode.zig b/libs/vulcan-ir/bitcode.zig index 1177fdd..50b1f00 100644 --- a/libs/vulcan-ir/bitcode.zig +++ b/libs/vulcan-ir/bitcode.zig @@ -12,6 +12,7 @@ const function = @import("function.zig"); const types = @import("types.zig"); const parser = @import("parser.zig"); const attribute = @import("attribute.zig"); +const nvfp4 = @import("nvfp4.zig"); const Function = function.Function; const Value = function.Value; @@ -94,6 +95,10 @@ const op_va_end: u8 = 22; const op_fconst128: u8 = 23; const op_barrier: u8 = 24; const op_atomic_rmw: u8 = 25; +const op_decode_low_float: u8 = 26; +const op_encode_low_float: u8 = 27; +const op_dequantize_nvfp4: u8 = 28; +const op_quantize_nvfp4: u8 = 29; // An `atomic_rmw` record flag bit: the compare operand follows the two ordinary operand // slots. Written from the field and read back into it, so a record round-trips whatever the @@ -408,6 +413,36 @@ fn writeInst(w: *Writer, func: *const Function, inst: Inst, serial: []const u32, try w.u8v(op_convert); try w.u32v(sv(serial, cv.value)); }, + .decode_low_float => |cv| { + try w.u8v(op_decode_low_float); + comptime { + std.debug.assert(@intFromEnum(function.LowFloatFormat.bf16) == 0); + std.debug.assert(@intFromEnum(function.LowFloatFormat.f8_e4m3) == 1); + std.debug.assert(@intFromEnum(function.LowFloatFormat.f8_e5m2) == 2); + } + try w.u8v(@intFromEnum(cv.format)); + try w.u32v(sv(serial, cv.value)); + }, + .encode_low_float => |cv| { + try w.u8v(op_encode_low_float); + try w.u8v(@intFromEnum(cv.format)); + try w.u32v(sv(serial, cv.value)); + }, + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + comptime { + std.debug.assert(@intFromEnum(nvfp4.ScaleApplication.multiply) == 0); + std.debug.assert(@intFromEnum(nvfp4.ScaleApplication.divide) == 1); + } + try w.u8v(if (std.meta.activeTag(func.opcode(inst)) == .dequantize_nvfp4) + op_dequantize_nvfp4 + else + op_quantize_nvfp4); + try w.u8v(@intFromEnum(cv.block_application)); + try w.u8v(@intFromEnum(cv.global_application)); + try w.u32v(sv(serial, cv.value)); + try w.u32v(sv(serial, cv.block_scale)); + try w.u32v(sv(serial, cv.global_scale)); + }, .unary => |u| { try w.u8v(op_unary); try w.u8v(@intFromEnum(u.op)); @@ -895,6 +930,12 @@ const Fixup = struct { }, .extract => |*e| e.aggregate = next(&i, self.slots, serial), .convert => |*cv| cv.value = next(&i, self.slots, serial), + .decode_low_float, .encode_low_float => |*cv| cv.value = next(&i, self.slots, serial), + .dequantize_nvfp4, .quantize_nvfp4 => |*cv| { + cv.value = next(&i, self.slots, serial); + cv.block_scale = next(&i, self.slots, serial); + cv.global_scale = next(&i, self.slots, serial); + }, .unary => |*u| u.value = next(&i, self.slots, serial), .load => |*l| l.ptr = next(&i, self.slots, serial), .store => |*st| { @@ -959,6 +1000,28 @@ const Fixup = struct { } }; +fn canonicalLowFloatResult( + func: *const Function, + result_type: Type, + format: function.LowFloatFormat, + decode_direction: bool, +) bool { + return switch (func.types.type_kind(result_type)) { + .float => |float| decode_direction and float == .f32, + .int => |int| !decode_direction and int.signedness == .unsigned and + int.bits == format.payloadBits(), + else => false, + }; +} + +fn canonicalNvFp4Result(func: *const Function, result_type: Type, dequantize_direction: bool) bool { + return switch (func.types.type_kind(result_type)) { + .float => |float| dequantize_direction and float == .f32, + .int => |int| !dequantize_direction and int.signedness == .unsigned and int.bits == 8, + else => false, + }; +} + fn readInst(r: *Reader, func: *Function, block: Block, type_map: []const Type, block_count: u32, dummy: Value, serial: *std.ArrayList(Value), inst_serial: *std.ArrayList(Inst), fixups: *std.ArrayList(Fixup), allocator: std.mem.Allocator) Error!void { const has_result = (try r.take(u8)) != 0; const rty: Type = if (has_result) try mapType(type_map, type_map.len, try r.take(u32)) else undefined; @@ -1015,6 +1078,43 @@ fn readInst(r: *Reader, func: *Function, block: Block, type_map: []const Type, b try slots.append(allocator, try r.take(u32)); break :blk try appendRes(func, block, serial, rty, .{ .convert = .{ .value = dummy } }); }, + op_decode_low_float, op_encode_low_float => blk: { + if (!has_result) return error.MalformedBitcode; + const format = std.enums.fromInt(function.LowFloatFormat, try r.take(u8)) orelse + return error.MalformedBitcode; + if (!canonicalLowFloatResult(func, rty, format, tag == op_decode_low_float)) + return error.MalformedBitcode; + try slots.append(allocator, try r.take(u32)); + const conversion: function.LowFloatConvert = .{ .value = dummy, .format = format }; + break :blk if (tag == op_decode_low_float) + try appendRes(func, block, serial, rty, .{ .decode_low_float = conversion }) + else + try appendRes(func, block, serial, rty, .{ .encode_low_float = conversion }); + }, + op_dequantize_nvfp4, op_quantize_nvfp4 => blk: { + if (!has_result) return error.MalformedBitcode; + const block_application = std.enums.fromInt(nvfp4.ScaleApplication, try r.take(u8)) orelse + return error.MalformedBitcode; + const global_application = std.enums.fromInt(nvfp4.ScaleApplication, try r.take(u8)) orelse + return error.MalformedBitcode; + const dequantize_direction = tag == op_dequantize_nvfp4; + if (!canonicalNvFp4Result(func, rty, dequantize_direction)) + return error.MalformedBitcode; + try slots.append(allocator, try r.take(u32)); + try slots.append(allocator, try r.take(u32)); + try slots.append(allocator, try r.take(u32)); + const conversion: function.NvFp4Convert = .{ + .value = dummy, + .block_scale = dummy, + .global_scale = dummy, + .block_application = block_application, + .global_application = global_application, + }; + break :blk if (dequantize_direction) + try appendRes(func, block, serial, rty, .{ .dequantize_nvfp4 = conversion }) + else + try appendRes(func, block, serial, rty, .{ .quantize_nvfp4 = conversion }); + }, op_unary => blk: { const uop = std.enums.fromInt(function.UnaryOp, try r.take(u8)) orelse return error.MalformedBitcode; try slots.append(allocator, try r.take(u32)); @@ -1376,6 +1476,246 @@ test "round-trips a prefetch through bitcode" { try std.testing.expectEqualStrings(a, b); } +test "low float bitcode pins tags and rejects malformed records" { + const allocator = std.testing.allocator; + + try std.testing.expectEqual(@as(u8, 26), op_decode_low_float); + try std.testing.expectEqual(@as(u8, 27), op_encode_low_float); + try std.testing.expectEqual(@as(u8, 0), @intFromEnum(function.LowFloatFormat.bf16)); + try std.testing.expectEqual(@as(u8, 1), @intFromEnum(function.LowFloatFormat.f8_e4m3)); + try std.testing.expectEqual(@as(u8, 2), @intFromEnum(function.LowFloatFormat.f8_e5m2)); + + const formats = [_]function.LowFloatFormat{ .bf16, .f8_e4m3, .f8_e5m2 }; + inline for (.{ true, false }) |decode_direction| { + for (formats) |format| { + var func = Function.init(allocator); + defer func.deinit(); + const payload_t = try func.types.intern(.{ .int = .{ + .signedness = .unsigned, + .bits = format.payloadBits(), + } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) payload_t else f32_t); + const conversion: function.LowFloatConvert = .{ .value = source, .format = format }; + const result = try func.appendInst( + block, + if (decode_direction) f32_t else payload_t, + if (decode_direction) + .{ .decode_low_float = conversion } + else + .{ .encode_low_float = conversion }, + ); + func.setTerminator(block, .{ .ret = function.Ret.one(result) }); + + const bytes = try encode(allocator, &func); + defer allocator.free(bytes); + var decoded = try decode(allocator, bytes); + defer decoded.deinit(); + + const decoded_block: Block = @enumFromInt(0); + const decoded_inst = decoded.blockInsts(decoded_block)[0]; + const decoded_result = decoded.instResult(decoded_inst).?; + const decoded_source = decoded.blockParams(decoded_block)[0]; + const expected_result_t = if (decode_direction) + try decoded.types.intern(.{ .float = .f32 }) + else + try decoded.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + try std.testing.expectEqual(expected_result_t, decoded.valueType(decoded_result)); + switch (decoded.opcode(decoded_inst)) { + .decode_low_float => |cv| { + try std.testing.expect(decode_direction); + try std.testing.expectEqual(format, cv.format); + try std.testing.expectEqual(decoded_source, cv.value); + }, + .encode_low_float => |cv| { + try std.testing.expect(!decode_direction); + try std.testing.expectEqual(format, cv.format); + try std.testing.expectEqual(decoded_source, cv.value); + }, + else => return error.TestUnexpectedResult, + } + + const tag = if (decode_direction) op_decode_low_float else op_encode_low_float; + var record_at: ?usize = null; + var i: usize = 5; + while (i + 5 < bytes.len) : (i += 1) { + if (bytes[i] == tag and bytes[i + 1] == @intFromEnum(format) and + std.mem.eql(u8, bytes[i + 2 .. i + 6], &std.mem.toBytes(@as(u32, 0)))) + { + record_at = i; + break; + } + } + try std.testing.expect(record_at != null); + const at = record_at.?; + try std.testing.expectEqual(@as(u8, 1), bytes[at - 5]); + + // Every patch alters only the named field of an otherwise valid record. + if (format == .bf16) { + { + const malformed = try allocator.dupe(u8, bytes); + defer allocator.free(malformed); + malformed[at] = 0xff; + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed)); + } + { + const malformed = try allocator.dupe(u8, bytes); + defer allocator.free(malformed); + malformed[at + 1] = 0xff; + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed)); + } + { + var malformed: std.ArrayList(u8) = .empty; + defer malformed.deinit(allocator); + try malformed.appendSlice(allocator, bytes[0 .. at - 5]); + try malformed.append(allocator, 0); + // A result-less record has no result-type field: the opcode follows the + // zero has-result byte immediately. Leaving the old four type bytes in + // place would only exercise unknown-opcode handling instead of this + // opcode's explicit missing-result guard. + try malformed.appendSlice(allocator, bytes[at..]); + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed.items)); + } + { + const malformed = try allocator.dupe(u8, bytes); + defer allocator.free(malformed); + const wrong_result_type: u32 = @intFromEnum(if (decode_direction) payload_t else f32_t); + std.mem.writeInt(u32, @ptrCast(malformed[at - 4 .. at].ptr), wrong_result_type, .little); + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed)); + } + { + const malformed = try allocator.dupe(u8, bytes); + defer allocator.free(malformed); + @memset(malformed[at + 2 .. at + 6], 0xff); + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed)); + } + try std.testing.expectError(error.MalformedBitcode, decode(allocator, bytes[0 .. at + 3])); + } + } + } +} + +test "NVFP4 bitcode pins fields and rejects every malformed record field" { + const allocator = std.testing.allocator; + try std.testing.expectEqual(@as(u8, 28), op_dequantize_nvfp4); + try std.testing.expectEqual(@as(u8, 29), op_quantize_nvfp4); + try std.testing.expectEqual(@as(u8, 0), @intFromEnum(nvfp4.ScaleApplication.multiply)); + try std.testing.expectEqual(@as(u8, 1), @intFromEnum(nvfp4.ScaleApplication.divide)); + + inline for (.{ true, false }) |dequantize_direction| { + inline for (.{ nvfp4.ScaleApplication.multiply, nvfp4.ScaleApplication.divide }) |block_application| { + inline for (.{ nvfp4.ScaleApplication.multiply, nvfp4.ScaleApplication.divide }) |global_application| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const value = try func.appendBlockParam(block, if (dequantize_direction) u8_t else f32_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: function.NvFp4Convert = .{ + .value = value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = block_application, + .global_application = global_application, + }; + const result = try func.appendInst(block, if (dequantize_direction) f32_t else u8_t, if (dequantize_direction) + .{ .dequantize_nvfp4 = conversion } + else + .{ .quantize_nvfp4 = conversion }); + const inst = func.definingInst(result).?; + try func.addAttr(.{ .inst = inst }, .{ .custom = .{ + .namespace = "test", + .key = "nvfp4", + .value = .flag, + } }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ + .namespace = "test", + .key = "result", + .value = .flag, + } }); + func.setTerminator(block, .{ .ret = function.Ret.one(result) }); + + const bytes = try encode(allocator, &func); + defer allocator.free(bytes); + var decoded = try decode(allocator, bytes); + defer decoded.deinit(); + const decoded_inst = decoded.blockInsts(@enumFromInt(0))[0]; + const decoded_result = decoded.instResult(decoded_inst).?; + const decoded_conversion = switch (decoded.opcode(decoded_inst)) { + .dequantize_nvfp4 => |cv| cv, + .quantize_nvfp4 => |cv| cv, + else => return error.TestUnexpectedResult, + }; + const params = decoded.blockParams(@enumFromInt(0)); + try std.testing.expectEqual(params[0], decoded_conversion.value); + try std.testing.expectEqual(params[1], decoded_conversion.block_scale); + try std.testing.expectEqual(params[2], decoded_conversion.global_scale); + try std.testing.expectEqual(block_application, decoded_conversion.block_application); + try std.testing.expectEqual(global_application, decoded_conversion.global_application); + var inst_attrs = decoded.attributesOf(.{ .inst = decoded_inst }); + try std.testing.expect(inst_attrs.next() != null); + var result_attrs = decoded.attributesOf(.{ .value = decoded_result }); + try std.testing.expect(result_attrs.next() != null); + + const tag = if (dequantize_direction) op_dequantize_nvfp4 else op_quantize_nvfp4; + const pattern = [_]u8{ + tag, + @intFromEnum(block_application), + @intFromEnum(global_application), + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 2, + 0, + 0, + 0, + }; + const at = std.mem.indexOf(u8, bytes, &pattern) orelse return error.TestUnexpectedResult; + if (block_application == .multiply and global_application == .multiply) { + { + var malformed: std.ArrayList(u8) = .empty; + defer malformed.deinit(allocator); + try malformed.appendSlice(allocator, bytes[0 .. at - 5]); + try malformed.append(allocator, 0); + try malformed.appendSlice(allocator, bytes[at..]); + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed.items)); + } + for ([_]usize{ at + 1, at + 2 }) |policy_at| { + const malformed = try allocator.dupe(u8, bytes); + defer allocator.free(malformed); + malformed[policy_at] = 2; + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed)); + } + { + const malformed = try allocator.dupe(u8, bytes); + defer allocator.free(malformed); + const wrong_type: u32 = @intFromEnum(if (dequantize_direction) u8_t else f32_t); + std.mem.writeInt(u32, @ptrCast(malformed[at - 4 .. at].ptr), wrong_type, .little); + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed)); + } + for ([_]usize{ at + 3, at + 7, at + 11 }) |serial_at| { + const malformed = try allocator.dupe(u8, bytes); + defer allocator.free(malformed); + @memset(malformed[serial_at .. serial_at + 4], 0xff); + try std.testing.expectError(error.MalformedBitcode, decode(allocator, malformed)); + } + for ([_]usize{ at + 2, at + 3, at + 7, at + 11, at + 15 }) |end| { + try std.testing.expectError(error.MalformedBitcode, decode(allocator, bytes[0..end])); + } + } + } + } + } +} + test "round-trips va_start/va_arg/va_end through bitcode" { const allocator = std.testing.allocator; diff --git a/libs/vulcan-ir/expand.zig b/libs/vulcan-ir/expand.zig index 2e98707..cbea8a6 100644 --- a/libs/vulcan-ir/expand.zig +++ b/libs/vulcan-ir/expand.zig @@ -21,6 +21,9 @@ const std = @import("std"); const function = @import("function.zig"); const types = @import("types.zig"); +const verify = @import("verify.zig"); +const bitcode = @import("bitcode.zig"); +const nvfp4 = @import("nvfp4.zig"); const Function = function.Function; const Value = function.Value; @@ -29,6 +32,1319 @@ const Inst = function.Inst; const BinOp = function.BinOp; const MatMul = function.MatMul; +const NumericBuilder = struct { + func: *Function, + out: *std.ArrayList(Inst), + allocator: std.mem.Allocator, + u8_t: types.Type, + u16_t: types.Type, + u32_t: types.Type, + f32_t: types.Type, + bool_t: types.Type, + + fn emit(self: NumericBuilder, ty: types.Type, opcode: function.Opcode) std.mem.Allocator.Error!Value { + const value = try self.func.createInst(ty, opcode); + try self.out.append(self.allocator, self.func.definingInst(value).?); + return value; + } + + fn constant(self: NumericBuilder, value: u32) std.mem.Allocator.Error!Value { + return self.emit(self.u32_t, .{ .iconst = value }); + } + + fn binary(self: NumericBuilder, op: BinOp, lhs: Value, rhs: Value) std.mem.Allocator.Error!Value { + return self.emit(self.u32_t, .{ .arith = .{ .op = op, .lhs = lhs, .rhs = rhs } }); + } + + fn floatBinary(self: NumericBuilder, op: BinOp, lhs: Value, rhs: Value) std.mem.Allocator.Error!Value { + std.debug.assert(op == .mul or op == .div); + std.debug.assert(self.func.valueType(lhs) == self.f32_t); + std.debug.assert(self.func.valueType(rhs) == self.f32_t); + return self.emit(self.f32_t, .{ .arith = .{ .op = op, .lhs = lhs, .rhs = rhs } }); + } + + fn compare(self: NumericBuilder, op: function.CmpOp, lhs: Value, rhs: Value) std.mem.Allocator.Error!Value { + return self.emit(self.bool_t, .{ .icmp = .{ .op = op, .lhs = lhs, .rhs = rhs } }); + } + + fn select(self: NumericBuilder, cond: Value, then_value: Value, else_value: Value) std.mem.Allocator.Error!Value { + return self.emit(self.u32_t, .{ .select = .{ .cond = cond, .then = then_value, .@"else" = else_value } }); + } + + fn convert(self: NumericBuilder, ty: types.Type, value: Value) std.mem.Allocator.Error!Value { + return self.emit(ty, .{ .convert = .{ .value = value } }); + } + + fn reinterpret(self: NumericBuilder, ty: types.Type, value: Value) std.mem.Allocator.Error!Value { + return self.emit(ty, .{ .unary = .{ .op = .reinterpret, .value = value } }); + } + + fn masked(self: NumericBuilder, value: Value, mask: u32) std.mem.Allocator.Error!Value { + return self.binary(.bit_and, value, try self.constant(mask)); + } + + fn shifted(self: NumericBuilder, op: BinOp, value: Value, amount: u32) std.mem.Allocator.Error!Value { + return self.binary(op, value, try self.constant(amount)); + } + + fn equalConstant(self: NumericBuilder, value: Value, expected: u32) std.mem.Allocator.Error!Value { + return self.compare(.eq, value, try self.constant(expected)); + } + + fn roundRight(self: NumericBuilder, value: Value, shift: Value) std.mem.Allocator.Error!Value { + const one = try self.constant(1); + const retained = try self.binary(.shr, value, shift); + const shifted_one = try self.binary(.shl, one, shift); + const mask = try self.binary(.sub, shifted_one, one); + const discarded = try self.binary(.bit_and, value, mask); + const shift_minus_one = try self.binary(.sub, shift, one); + const halfway = try self.binary(.shl, one, shift_minus_one); + const greater = try self.compare(.gt, discarded, halfway); + const equal = try self.compare(.eq, discarded, halfway); + const odd_bits = try self.binary(.bit_and, retained, one); + const odd = try self.compare(.ne, odd_bits, try self.constant(0)); + const odd_as_int = try self.select(odd, one, try self.constant(0)); + const tie_and_odd = try self.select(equal, odd_as_int, try self.constant(0)); + const increment = try self.select(greater, one, tie_and_odd); + return self.binary(.add, retained, increment); + } +}; + +/// Replace all low-float storage conversions with target-independent integer operations. +pub fn expandLowFloat(allocator: std.mem.Allocator, func: *Function) std.mem.Allocator.Error!bool { + var has_low_float = false; + for (0..func.blockCount()) |block_index| { + const block: Block = @enumFromInt(block_index); + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .decode_low_float, .encode_low_float => { + has_low_float = true; + break; + }, + else => {}, + }; + if (has_low_float) break; + } + if (!has_low_float) return false; + + var changed = false; + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const bool_t = try func.types.intern(.bool); + + for (0..func.blockCount()) |block_index| { + const block: Block = @enumFromInt(block_index); + var contains_low_float = false; + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .decode_low_float, .encode_low_float => contains_low_float = true, + else => {}, + }; + if (!contains_low_float) continue; + changed = true; + + const original = try allocator.dupe(Inst, func.blockInsts(block)); + defer allocator.free(original); + var out: std.ArrayList(Inst) = .empty; + defer out.deinit(allocator); + const builder: NumericBuilder = .{ + .func = func, + .out = &out, + .allocator = allocator, + .u8_t = u8_t, + .u16_t = u16_t, + .u32_t = u32_t, + .f32_t = f32_t, + .bool_t = bool_t, + }; + + for (original) |inst| switch (func.opcode(inst)) { + .decode_low_float => |conversion| { + const old_result = func.instResult(inst).?; + const expanded = try expandDecodeLowFloat(builder, conversion); + func.replaceAllUses(old_result, expanded); + func.retargetAttrs(.{ .value = old_result }, .{ .value = expanded }); + func.retargetAttrs(.{ .inst = inst }, .{ .inst = func.definingInst(expanded).? }); + }, + .encode_low_float => |conversion| { + const old_result = func.instResult(inst).?; + const expanded = try expandEncodeLowFloat(builder, conversion, inst); + func.replaceAllUses(old_result, expanded); + func.retargetAttrs(.{ .value = old_result }, .{ .value = expanded }); + }, + else => try out.append(allocator, inst), + }; + try func.setBlockInsts(block, out.items); + } + return changed; +} + +/// Replace scalar NVFP4 conversions with integer classification and ordered f32 scale operations. +pub fn expandNvFp4(allocator: std.mem.Allocator, func: *Function) std.mem.Allocator.Error!bool { + var has_nvfp4 = false; + for (0..func.blockCount()) |block_index| { + for (func.blockInsts(@enumFromInt(block_index))) |inst| switch (func.opcode(inst)) { + .dequantize_nvfp4, .quantize_nvfp4 => { + has_nvfp4 = true; + break; + }, + else => {}, + }; + if (has_nvfp4) break; + } + if (!has_nvfp4) return false; + + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const bool_t = try func.types.intern(.bool); + + for (0..func.blockCount()) |block_index| { + const block: Block = @enumFromInt(block_index); + var contains_nvfp4 = false; + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .dequantize_nvfp4, .quantize_nvfp4 => contains_nvfp4 = true, + else => {}, + }; + if (!contains_nvfp4) continue; + + const original = try allocator.dupe(Inst, func.blockInsts(block)); + defer allocator.free(original); + var out: std.ArrayList(Inst) = .empty; + defer out.deinit(allocator); + const builder: NumericBuilder = .{ + .func = func, + .out = &out, + .allocator = allocator, + .u8_t = u8_t, + .u16_t = u16_t, + .u32_t = u32_t, + .f32_t = f32_t, + .bool_t = bool_t, + }; + + for (original) |inst| switch (func.opcode(inst)) { + .dequantize_nvfp4 => |conversion| { + const old_result = func.instResult(inst).?; + const expanded = try expandDequantizeNvFp4(builder, conversion); + func.replaceAllUses(old_result, expanded.result); + func.retargetAttrs(.{ .inst = inst }, .{ .inst = expanded.semantic_inst }); + func.retargetAttrs(.{ .value = old_result }, .{ .value = expanded.result }); + }, + .quantize_nvfp4 => |conversion| { + const old_result = func.instResult(inst).?; + const expanded = try expandQuantizeNvFp4(builder, conversion); + func.replaceAllUses(old_result, expanded.result); + func.retargetAttrs(.{ .inst = inst }, .{ .inst = expanded.semantic_inst }); + func.retargetAttrs(.{ .value = old_result }, .{ .value = expanded.result }); + }, + else => try out.append(allocator, inst), + }; + try func.setBlockInsts(block, out.items); + } + return true; +} + +/// Replace scalar f32 division with target-independent u32 operations. +pub fn expandF32Div(allocator: std.mem.Allocator, func: *Function) std.mem.Allocator.Error!bool { + var has_division = false; + for (0..func.blockCount()) |block_index| { + for (func.blockInsts(@enumFromInt(block_index))) |inst| switch (func.opcode(inst)) { + .arith => |arith| if (arith.op == .div) switch (func.types.type_kind(func.valueType(func.instResult(inst).?))) { + .float => |kind| if (kind == .f32) { + has_division = true; + break; + }, + else => {}, + }, + else => {}, + }; + if (has_division) break; + } + if (!has_division) return false; + + const f32_t = try func.types.intern(.{ .float = .f32 }); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const bool_t = try func.types.intern(.bool); + + for (0..func.blockCount()) |block_index| { + const block: Block = @enumFromInt(block_index); + var contains_division = false; + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .arith => |arith| if (arith.op == .div and func.valueType(func.instResult(inst).?) == f32_t) { + contains_division = true; + break; + }, + else => {}, + }; + if (!contains_division) continue; + + const original = try allocator.dupe(Inst, func.blockInsts(block)); + defer allocator.free(original); + var out: std.ArrayList(Inst) = .empty; + defer out.deinit(allocator); + const builder: NumericBuilder = .{ + .func = func, + .out = &out, + .allocator = allocator, + .u8_t = u8_t, + .u16_t = u16_t, + .u32_t = u32_t, + .f32_t = f32_t, + .bool_t = bool_t, + }; + + for (original) |inst| switch (func.opcode(inst)) { + .arith => |arith| if (arith.op == .div and func.valueType(func.instResult(inst).?) == f32_t) { + const old_result = func.instResult(inst).?; + const expanded = try expandF32Division(builder, arith.lhs, arith.rhs); + func.replaceAllUses(old_result, expanded); + func.retargetAttrs(.{ .value = old_result }, .{ .value = expanded }); + func.retargetAttrs(.{ .inst = inst }, .{ .inst = func.definingInst(expanded).? }); + } else try out.append(allocator, inst), + else => try out.append(allocator, inst), + }; + try func.setBlockInsts(block, out.items); + } + return true; +} + +const NormalizedF32 = struct { + significand: Value, + exponent: Value, +}; + +fn normalizeF32(builder: NumericBuilder, magnitude: Value) std.mem.Allocator.Error!NormalizedF32 { + const fraction = try builder.masked(magnitude, 0x007f_ffff); + const raw_exponent = try builder.masked(try builder.shifted(.shr, magnitude, 23), 0xff); + const is_subnormal = try builder.equalConstant(raw_exponent, 0); + var significand = try builder.select( + is_subnormal, + fraction, + try builder.binary(.bit_or, fraction, try builder.constant(0x0080_0000)), + ); + var exponent = try builder.select( + is_subnormal, + try builder.constant(257), + try builder.binary(.add, raw_exponent, try builder.constant(256)), + ); + for ([_]u32{ 16, 8, 4, 2, 1 }) |shift| { + const needs_shift = try builder.compare(.lt, significand, try builder.constant(@as(u32, 0x0100_0000) >> @intCast(shift))); + significand = try builder.select(needs_shift, try builder.shifted(.shl, significand, shift), significand); + exponent = try builder.select(needs_shift, try builder.binary(.sub, exponent, try builder.constant(shift)), exponent); + } + return .{ .significand = significand, .exponent = exponent }; +} + +fn boolAsU32(builder: NumericBuilder, condition: Value) std.mem.Allocator.Error!Value { + return builder.select(condition, try builder.constant(1), try builder.constant(0)); +} + +fn flagSet(builder: NumericBuilder, flag: Value) std.mem.Allocator.Error!Value { + return builder.compare(.ne, flag, try builder.constant(0)); +} + +fn roundDivisionQuotient(builder: NumericBuilder, quotient: Value, remainder: Value, shift: Value) std.mem.Allocator.Error!Value { + const one = try builder.constant(1); + const retained = try builder.binary(.shr, quotient, shift); + const shifted_one = try builder.binary(.shl, one, shift); + const discarded = try builder.binary(.bit_and, quotient, try builder.binary(.sub, shifted_one, one)); + const halfway = try builder.binary(.shl, one, try builder.binary(.sub, shift, one)); + const above_half = try boolAsU32(builder, try builder.compare(.gt, discarded, halfway)); + const at_half = try boolAsU32(builder, try builder.compare(.eq, discarded, halfway)); + const has_remainder = try boolAsU32(builder, try builder.compare(.ne, remainder, try builder.constant(0))); + const odd = try boolAsU32(builder, try builder.compare(.ne, try builder.binary(.bit_and, retained, one), try builder.constant(0))); + const half_increment = try builder.select(try flagSet(builder, has_remainder), one, odd); + const tie_increment = try builder.select(try flagSet(builder, at_half), half_increment, try builder.constant(0)); + const increment = try builder.select(try flagSet(builder, above_half), one, tie_increment); + return builder.binary(.add, retained, increment); +} + +fn expandF32Division(builder: NumericBuilder, numerator: Value, denominator: Value) std.mem.Allocator.Error!Value { + const numerator_bits = try builder.reinterpret(builder.u32_t, numerator); + const denominator_bits = try builder.reinterpret(builder.u32_t, denominator); + const numerator_magnitude = try builder.masked(numerator_bits, 0x7fff_ffff); + const denominator_magnitude = try builder.masked(denominator_bits, 0x7fff_ffff); + const sign = try builder.masked(try builder.binary(.bit_xor, numerator_bits, denominator_bits), 0x8000_0000); + const normalized_numerator = try normalizeF32(builder, numerator_magnitude); + const normalized_denominator = try normalizeF32(builder, denominator_magnitude); + + const ratio_below_one = try boolAsU32(builder, try builder.compare(.lt, normalized_numerator.significand, normalized_denominator.significand)); + const dividend = try builder.select( + try flagSet(builder, ratio_below_one), + try builder.shifted(.shl, normalized_numerator.significand, 1), + normalized_numerator.significand, + ); + const exponent_bias = try builder.constant(383); + var result_exponent = try builder.binary( + .add, + try builder.binary(.sub, normalized_numerator.exponent, normalized_denominator.exponent), + exponent_bias, + ); + result_exponent = try builder.binary(.sub, result_exponent, ratio_below_one); + + var quotient = try builder.constant(0); + var remainder = dividend; + for (0..27) |step| { + const bit = try builder.compare(.ge, remainder, normalized_denominator.significand); + const subtrahend = try builder.select(bit, normalized_denominator.significand, try builder.constant(0)); + remainder = try builder.binary(.sub, remainder, subtrahend); + quotient = try builder.binary(.bit_or, try builder.shifted(.shl, quotient, 1), try boolAsU32(builder, bit)); + if (step != 26) remainder = try builder.shifted(.shl, remainder, 1); + } + + const is_tiny = try boolAsU32(builder, try builder.compare(.le, result_exponent, try builder.constant(256))); + const raw_tiny_shift = try builder.binary(.sub, try builder.constant(260), result_exponent); + const shift_too_large = try boolAsU32(builder, try builder.compare(.gt, raw_tiny_shift, try builder.constant(31))); + const tiny_shift = try builder.select(try flagSet(builder, shift_too_large), try builder.constant(31), raw_tiny_shift); + const rounding_shift = try builder.select(try flagSet(builder, is_tiny), tiny_shift, try builder.constant(3)); + const rounded = try roundDivisionQuotient(builder, quotient, remainder, rounding_shift); + + const rounded_carry = try boolAsU32(builder, try builder.compare(.ge, rounded, try builder.constant(0x0100_0000))); + const normal_significand = try builder.select(try flagSet(builder, rounded_carry), try builder.shifted(.shr, rounded, 1), rounded); + const normal_exponent = try builder.binary( + .add, + try builder.binary(.sub, result_exponent, try builder.constant(256)), + rounded_carry, + ); + const normal_bits = try builder.binary( + .bit_or, + try builder.shifted(.shl, normal_exponent, 23), + try builder.masked(normal_significand, 0x007f_ffff), + ); + const finite_bits = try builder.select(try flagSet(builder, is_tiny), rounded, normal_bits); + const overflow_before_round = try boolAsU32(builder, try builder.compare(.ge, result_exponent, try builder.constant(511))); + const overflow_after_round = try boolAsU32(builder, try builder.compare(.ge, normal_exponent, try builder.constant(255))); + const rounded_overflow = try builder.select(try flagSet(builder, is_tiny), try builder.constant(0), overflow_after_round); + const overflow = try builder.select(try flagSet(builder, overflow_before_round), try builder.constant(1), rounded_overflow); + var result_bits = try builder.select( + try builder.compare(.ne, overflow, try builder.constant(0)), + try builder.constant(0x7f80_0000), + finite_bits, + ); + + const numerator_zero = try boolAsU32(builder, try builder.equalConstant(numerator_magnitude, 0)); + const denominator_zero = try boolAsU32(builder, try builder.equalConstant(denominator_magnitude, 0)); + const numerator_infinity = try boolAsU32(builder, try builder.equalConstant(numerator_magnitude, 0x7f80_0000)); + const denominator_infinity = try boolAsU32(builder, try builder.equalConstant(denominator_magnitude, 0x7f80_0000)); + const numerator_nan = try boolAsU32(builder, try builder.compare(.gt, numerator_magnitude, try builder.constant(0x7f80_0000))); + const denominator_nan = try boolAsU32(builder, try builder.compare(.gt, denominator_magnitude, try builder.constant(0x7f80_0000))); + result_bits = try builder.select(try flagSet(builder, denominator_infinity), try builder.constant(0), result_bits); + result_bits = try builder.select(try flagSet(builder, numerator_infinity), try builder.constant(0x7f80_0000), result_bits); + result_bits = try builder.select(try flagSet(builder, denominator_zero), try builder.constant(0x7f80_0000), result_bits); + result_bits = try builder.select(try flagSet(builder, numerator_zero), try builder.constant(0), result_bits); + result_bits = try builder.binary(.bit_or, sign, result_bits); + + const both_zero = try builder.select(try flagSet(builder, numerator_zero), denominator_zero, try builder.constant(0)); + const both_infinite = try builder.select(try flagSet(builder, numerator_infinity), denominator_infinity, try builder.constant(0)); + var invalid = try builder.select(try flagSet(builder, numerator_nan), try builder.constant(1), denominator_nan); + invalid = try builder.binary(.bit_or, invalid, both_zero); + invalid = try builder.binary(.bit_or, invalid, both_infinite); + result_bits = try builder.select( + try builder.compare(.ne, invalid, try builder.constant(0)), + try builder.constant(0x7fc0_0000), + result_bits, + ); + return builder.reinterpret(builder.f32_t, result_bits); +} + +fn appendF32DivisionFixture(func: *Function, block: Block) std.mem.Allocator.Error!Value { + const f32_t = try func.types.intern(.{ .float = .f32 }); + const numerator = try func.appendBlockParam(block, f32_t); + const denominator = try func.appendBlockParam(block, f32_t); + return func.appendInst(block, f32_t, .{ .arith = .{ .op = .div, .lhs = numerator, .rhs = denominator } }); +} + +test "expandF32Div replaces uses attributes and every scalar division in program order" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const entry = try func.appendBlock(); + const first = try appendF32DivisionFixture(&func, entry); + const first_inst = func.definingInst(first).?; + try func.addAttr(.{ .inst = first_inst }, .{ .custom = .{ .namespace = "debug", .key = "line", .value = .{ .int = 45 } } }); + try func.addAttr(.{ .value = first }, .{ .custom = .{ .namespace = "test", .key = "division", .value = .flag } }); + const second = try func.appendInst(entry, f32_t, .{ .arith = .{ .op = .div, .lhs = first, .rhs = func.blockParams(entry)[1] } }); + const exit = try func.appendBlock(); + try func.setJump(entry, exit, &.{second}); + const forwarded = try func.appendBlockParam(exit, f32_t); + func.setTerminator(exit, .{ .ret = function.Ret.one(forwarded) }); + + try std.testing.expect(try expandF32Div(allocator, &func)); + try std.testing.expect(!(try expandF32Div(allocator, &func))); + var diagnostics = try verify.verify(allocator, &func, .low); + defer diagnostics.deinit(); + try std.testing.expect(diagnostics.ok()); + for (0..func.blockCount()) |block_index| { + for (func.blockInsts(@enumFromInt(block_index))) |inst| switch (func.opcode(inst)) { + .arith => |arithmetic| if (arithmetic.op == .div and + func.valueType(func.instResult(inst).?) == f32_t) + { + return error.TestUnexpectedResult; + }, + else => {}, + }; + } + const replacement = func.blockArgs(func.terminator(entry).?.jump)[0]; + try std.testing.expect(replacement != second); + var migrated_value_attrs: usize = 0; + var migrated_inst_attrs: usize = 0; + for (func.blockInsts(entry)) |inst| { + const live_result = func.instResult(inst).?; + var value_attrs = func.attributesOf(.{ .value = live_result }); + if (value_attrs.next()) |attribute| { + try std.testing.expectEqualStrings("division", attribute.custom.key); + migrated_value_attrs += 1; + } + var inst_attrs = func.attributesOf(.{ .inst = inst }); + if (inst_attrs.next()) |attribute| { + try std.testing.expectEqualStrings("line", attribute.custom.key); + migrated_inst_attrs += 1; + } + } + try std.testing.expectEqual(@as(usize, 1), migrated_value_attrs); + try std.testing.expectEqual(@as(usize, 1), migrated_inst_attrs); + var old_value_attrs = func.attributesOf(.{ .value = first }); + var old_inst_attrs = func.attributesOf(.{ .inst = first_inst }); + try std.testing.expect(old_value_attrs.next() == null); + try std.testing.expect(old_inst_attrs.next() == null); +} + +test "expandF32Div emits only bounded ordinary integer operations" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const block = try func.appendBlock(); + const result = try appendF32DivisionFixture(&func, block); + func.setTerminator(block, .{ .ret = function.Ret.one(result) }); + try std.testing.expect(try expandF32Div(allocator, &func)); + + var restoring_compares: usize = 0; + var dynamic_shifts: usize = 0; + var clamped_shift_uses: usize = 0; + var count_minus_one_uses: usize = 0; + var max_predicate_distance: usize = 0; + const insts = func.blockInsts(block); + for (insts, 0..) |inst, inst_index| switch (func.opcode(inst)) { + .arith => |arithmetic| { + const result_kind = func.types.type_kind(func.valueType(func.instResult(inst).?)); + try std.testing.expect(std.meta.activeTag(result_kind) == .int); + try std.testing.expectEqual(@as(u16, 32), result_kind.int.bits); + if (arithmetic.op == .shl or arithmetic.op == .shr) switch (func.opcode(func.definingInst(arithmetic.rhs).?)) { + .iconst => |amount| try std.testing.expect(amount <= 31), + .select => |amount| { + dynamic_shifts += 1; + clamped_shift_uses += 1; + switch (func.opcode(func.definingInst(amount.@"else").?)) { + .iconst => |normal_count| try std.testing.expectEqual(@as(i64, 3), normal_count), + else => return error.TestUnexpectedResult, + } + switch (func.opcode(func.definingInst(amount.then).?)) { + .select => |clamp| { + switch (func.opcode(func.definingInst(clamp.then).?)) { + .iconst => |maximum| try std.testing.expectEqual(@as(i64, 31), maximum), + else => return error.TestUnexpectedResult, + } + try std.testing.expect(func.opcode(func.definingInst(clamp.@"else").?) == .arith); + }, + else => return error.TestUnexpectedResult, + } + }, + .arith => |count| { + dynamic_shifts += 1; + try std.testing.expectEqual(function.BinOp.sub, count.op); + switch (func.opcode(func.definingInst(count.rhs).?)) { + .iconst => |one| try std.testing.expectEqual(@as(i64, 1), one), + else => return error.TestUnexpectedResult, + } + try std.testing.expect(func.opcode(func.definingInst(count.lhs).?) == .select); + count_minus_one_uses += 1; + }, + else => return error.TestUnexpectedResult, + }; + }, + .icmp => |comparison| { + if (comparison.op == .ge) restoring_compares += 1; + try std.testing.expect(std.meta.activeTag(func.types.type_kind(func.valueType(comparison.lhs))) == .int); + try std.testing.expect(std.meta.activeTag(func.types.type_kind(func.valueType(comparison.rhs))) == .int); + }, + .select => |selection| { + try std.testing.expect(std.meta.activeTag(func.types.type_kind(func.valueType(selection.then))) == .int); + try std.testing.expect(std.meta.activeTag(func.types.type_kind(func.valueType(selection.@"else"))) == .int); + const predicate_inst = func.definingInst(selection.cond).?; + var predicate_index: ?usize = null; + for (insts[0..inst_index], 0..) |candidate, candidate_index| { + if (candidate == predicate_inst) predicate_index = candidate_index; + } + const distance = inst_index - (predicate_index orelse return error.TestUnexpectedResult); + max_predicate_distance = @max(max_predicate_distance, distance); + }, + .unary => |unary| try std.testing.expectEqual(function.UnaryOp.reinterpret, unary.op), + .call, .call_indirect, .convert => return error.TestUnexpectedResult, + else => {}, + }; + try std.testing.expectEqual(@as(usize, 30), restoring_compares); + try std.testing.expectEqual(@as(usize, 3), dynamic_shifts); + try std.testing.expectEqual(@as(usize, 2), clamped_shift_uses); + try std.testing.expectEqual(@as(usize, 1), count_minus_one_uses); + // The restoring predicate has the longest span: compare, guarded subtract, quotient + // shift, then conversion to a u32 flag. Freeze that eight-instruction ceiling so a + // refactor cannot hoist transient predicates across the unrolled division. + try std.testing.expectEqual(@as(usize, 8), max_predicate_distance); +} + +test "expandF32Div leaves division-free logical IR byte-for-byte unchanged" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const block = try func.appendBlock(); + const value = try func.appendInst(block, u32_t, .{ .iconst = 45 }); + try func.addAttr(.{ .value = value }, .{ .custom = .{ .namespace = "test", .key = "unchanged", .value = .flag } }); + func.setTerminator(block, .{ .ret = function.Ret.one(value) }); + const text_before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_before); + const bits_before = try bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + try std.testing.expect(!(try expandF32Div(allocator, &func))); + const text_after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_after); + const bits_after = try bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try std.testing.expectEqualStrings(text_before, text_after); + try std.testing.expectEqualSlices(u8, bits_before, bits_after); +} + +const NvFp4Expansion = struct { + result: Value, + semantic_inst: Inst, +}; + +fn scaleOp(application: nvfp4.ScaleApplication, inverse: bool) BinOp { + return switch (application) { + .multiply => if (inverse) .div else .mul, + .divide => if (inverse) .mul else .div, + }; +} + +fn expandDequantizeNvFp4(builder: NumericBuilder, conversion: function.NvFp4Convert) std.mem.Allocator.Error!NvFp4Expansion { + const carrier = try builder.convert(builder.u32_t, conversion.value); + const nibble = try builder.masked(carrier, 0x0f); + const sign = try builder.shifted(.shl, try builder.masked(nibble, 0x08), 28); + const magnitude_index = try builder.masked(nibble, 0x07); + var magnitude = try builder.constant(0); + const magnitude_bits = [_]u32{ + 0x3f00_0000, + 0x3f80_0000, + 0x3fc0_0000, + 0x4000_0000, + 0x4040_0000, + 0x4080_0000, + 0x40c0_0000, + }; + for (magnitude_bits, 1..) |bits, index| { + magnitude = try builder.select( + try builder.equalConstant(magnitude_index, @intCast(index)), + try builder.constant(bits), + magnitude, + ); + } + const value_bits = try builder.binary(.bit_or, sign, magnitude); + const value = try builder.reinterpret(builder.f32_t, value_bits); + + const block_payload = try builder.convert(builder.u32_t, conversion.block_scale); + const block_bits = try decodeE4M3(builder, block_payload); + const block_scale = try builder.reinterpret(builder.f32_t, block_bits); + const local = try builder.floatBinary(scaleOp(conversion.block_application, false), value, block_scale); + const scaled = try builder.floatBinary(scaleOp(conversion.global_application, false), local, conversion.global_scale); + const semantic_inst = builder.func.definingInst(scaled).?; + + // This bit round trip blocks a backend from fusing the final scale with an original consumer. + const scaled_bits = try builder.reinterpret(builder.u32_t, scaled); + const result = try builder.reinterpret(builder.f32_t, scaled_bits); + return .{ .result = result, .semantic_inst = semantic_inst }; +} + +fn expandQuantizeNvFp4(builder: NumericBuilder, conversion: function.NvFp4Convert) std.mem.Allocator.Error!NvFp4Expansion { + const block_payload = try builder.convert(builder.u32_t, conversion.block_scale); + const block_bits = try decodeE4M3(builder, block_payload); + const block_scale = try builder.reinterpret(builder.f32_t, block_bits); + const global_unscaled = try builder.floatBinary(scaleOp(conversion.global_application, true), conversion.value, conversion.global_scale); + const local_unscaled = try builder.floatBinary(scaleOp(conversion.block_application, true), global_unscaled, block_scale); + const bits = try builder.reinterpret(builder.u32_t, local_unscaled); + const semantic_inst = builder.func.definingInst(bits).?; + + const sign = try builder.masked(try builder.shifted(.shr, bits, 28), 0x08); + const magnitude = try builder.masked(bits, 0x7fff_ffff); + var payload = try builder.constant(0x07); + const thresholds = [_]struct { relation: function.CmpOp, bits: u32, payload: u32 }{ + .{ .relation = .le, .bits = 0x40a0_0000, .payload = 0x06 }, + .{ .relation = .lt, .bits = 0x4060_0000, .payload = 0x05 }, + .{ .relation = .le, .bits = 0x4020_0000, .payload = 0x04 }, + .{ .relation = .lt, .bits = 0x3fe0_0000, .payload = 0x03 }, + .{ .relation = .le, .bits = 0x3fa0_0000, .payload = 0x02 }, + .{ .relation = .lt, .bits = 0x3f40_0000, .payload = 0x01 }, + .{ .relation = .le, .bits = 0x3e80_0000, .payload = 0x00 }, + }; + for (thresholds) |threshold| { + payload = try builder.select( + try builder.compare(threshold.relation, magnitude, try builder.constant(threshold.bits)), + try builder.constant(threshold.payload), + payload, + ); + } + const signed_payload = try builder.binary(.bit_or, sign, payload); + const is_nan = try builder.compare(.gt, magnitude, try builder.constant(0x7f80_0000)); + const canonical = try builder.select(is_nan, try builder.constant(0x07), signed_payload); + return .{ + .result = try builder.convert(builder.u8_t, canonical), + .semantic_inst = semantic_inst, + }; +} + +fn expandDecodeLowFloat(builder: NumericBuilder, conversion: function.LowFloatConvert) std.mem.Allocator.Error!Value { + const payload = try builder.convert(builder.u32_t, conversion.value); + const bits = switch (conversion.format) { + .bf16 => try builder.shifted(.shl, payload, 16), + .f8_e4m3 => try decodeE4M3(builder, payload), + .f8_e5m2 => try decodeE5M2(builder, payload), + }; + return builder.reinterpret(builder.f32_t, bits); +} + +fn decodeE4M3(builder: NumericBuilder, payload: Value) std.mem.Allocator.Error!Value { + const sign = try builder.shifted(.shl, try builder.masked(payload, 0x80), 24); + const exponent = try builder.masked(try builder.shifted(.shr, payload, 3), 0x0f); + const fraction = try builder.masked(payload, 0x07); + const normal_exponent = try builder.shifted(.shl, try builder.binary(.add, exponent, try builder.constant(120)), 23); + const normal_fraction = try builder.shifted(.shl, fraction, 20); + const normal = try builder.binary(.bit_or, sign, try builder.binary(.bit_or, normal_exponent, normal_fraction)); + + var subnormal = try builder.constant(0); + const subnormal_bits = [_]u32{ 0x3b00_0000, 0x3b80_0000, 0x3bc0_0000, 0x3c00_0000, 0x3c20_0000, 0x3c40_0000, 0x3c60_0000 }; + for (subnormal_bits, 1..) |bits, raw_fraction| { + subnormal = try builder.select(try builder.equalConstant(fraction, @intCast(raw_fraction)), try builder.constant(bits), subnormal); + } + subnormal = try builder.binary(.bit_or, sign, subnormal); + const finite = try builder.select(try builder.equalConstant(exponent, 0), subnormal, normal); + const nan = try builder.binary(.bit_or, sign, try builder.constant(0x7fc0_0000)); + const exponent_is_max = try builder.equalConstant(exponent, 0x0f); + const fraction_is_nan = try builder.equalConstant(fraction, 0x07); + const fraction_is_nan_int = try builder.select(fraction_is_nan, try builder.constant(1), try builder.constant(0)); + const is_nan_int = try builder.select(exponent_is_max, fraction_is_nan_int, try builder.constant(0)); + const is_nan = try builder.compare(.ne, is_nan_int, try builder.constant(0)); + return builder.select(is_nan, nan, finite); +} + +fn decodeE5M2(builder: NumericBuilder, payload: Value) std.mem.Allocator.Error!Value { + const sign = try builder.shifted(.shl, try builder.masked(payload, 0x80), 24); + const exponent = try builder.masked(try builder.shifted(.shr, payload, 2), 0x1f); + const fraction = try builder.masked(payload, 0x03); + const normal_exponent = try builder.shifted(.shl, try builder.binary(.add, exponent, try builder.constant(112)), 23); + const normal_fraction = try builder.shifted(.shl, fraction, 21); + const normal = try builder.binary(.bit_or, sign, try builder.binary(.bit_or, normal_exponent, normal_fraction)); + + var subnormal = try builder.constant(0); + const subnormal_bits = [_]u32{ 0x3780_0000, 0x3800_0000, 0x3840_0000 }; + for (subnormal_bits, 1..) |bits, raw_fraction| { + subnormal = try builder.select(try builder.equalConstant(fraction, @intCast(raw_fraction)), try builder.constant(bits), subnormal); + } + subnormal = try builder.binary(.bit_or, sign, subnormal); + const finite = try builder.select(try builder.equalConstant(exponent, 0), subnormal, normal); + const quiet_fraction = try builder.shifted(.shl, try builder.binary(.bit_or, fraction, try builder.constant(2)), 21); + const nan = try builder.binary(.bit_or, sign, try builder.binary(.bit_or, try builder.constant(0x7f80_0000), quiet_fraction)); + const infinity = try builder.binary(.bit_or, sign, try builder.constant(0x7f80_0000)); + const special = try builder.select(try builder.equalConstant(fraction, 0), infinity, nan); + return builder.select(try builder.equalConstant(exponent, 0x1f), special, finite); +} + +fn expandEncodeLowFloat(builder: NumericBuilder, conversion: function.LowFloatConvert, old_inst: Inst) std.mem.Allocator.Error!Value { + const bits = try builder.reinterpret(builder.u32_t, conversion.value); + builder.func.retargetAttrs(.{ .inst = old_inst }, .{ .inst = builder.func.definingInst(bits).? }); + const payload = switch (conversion.format) { + .bf16 => try encodeBf16(builder, bits), + .f8_e4m3 => try encodeFp8(builder, bits, true), + .f8_e5m2 => try encodeFp8(builder, bits, false), + }; + return builder.convert(if (conversion.format == .bf16) builder.u16_t else builder.u8_t, payload); +} + +fn encodeBf16(builder: NumericBuilder, bits: Value) std.mem.Allocator.Error!Value { + const exponent = try builder.masked(bits, 0x7f80_0000); + const fraction = try builder.masked(bits, 0x007f_ffff); + const retained = try builder.shifted(.shr, bits, 16); + const retained_lsb = try builder.masked(retained, 1); + const rounded = try builder.shifted(.shr, try builder.binary(.add, bits, try builder.binary(.add, try builder.constant(0x7fff), retained_lsb)), 16); + const nan = try builder.binary(.bit_or, retained, try builder.constant(0x40)); + const special = try builder.select(try builder.compare(.ne, fraction, try builder.constant(0)), nan, retained); + return builder.select(try builder.equalConstant(exponent, 0x7f80_0000), special, rounded); +} + +fn encodeFp8(builder: NumericBuilder, bits: Value, e4m3: bool) std.mem.Allocator.Error!Value { + const sign = try builder.masked(try builder.shifted(.shr, bits, 24), 0x80); + const source_exponent = try builder.masked(try builder.shifted(.shr, bits, 23), 0xff); + const source_fraction = try builder.masked(bits, 0x007f_ffff); + const significand = try builder.binary(.bit_or, source_fraction, try builder.constant(0x0080_0000)); + const threshold: u32 = if (e4m3) 121 else 113; + const shift_bias: u32 = if (e4m3) 141 else 134; + const original_shift = try builder.binary(.sub, try builder.constant(shift_bias), source_exponent); + const shift_too_large = try builder.compare(.gt, original_shift, try builder.constant(24)); + const bounded_shift = try builder.select(shift_too_large, try builder.constant(24), original_shift); + const uses_subnormal_path = try builder.compare(.lt, source_exponent, try builder.constant(threshold)); + const safe_shift = try builder.select(uses_subnormal_path, bounded_shift, try builder.constant(24)); + const subnormal_rounded = try builder.roundRight(significand, safe_shift); + const subnormal = try builder.select(shift_too_large, try builder.constant(0), subnormal_rounded); + + const normal_shift = try builder.constant(if (e4m3) 20 else 21); + const normal_rounded = try builder.roundRight(significand, normal_shift); + const carry_value: u32 = if (e4m3) 16 else 8; + const retained_value: u32 = if (e4m3) 8 else 4; + const carry = try builder.equalConstant(normal_rounded, carry_value); + const rounded = try builder.select(carry, try builder.constant(retained_value), normal_rounded); + const exponent_base = try builder.binary(.sub, source_exponent, try builder.constant(if (e4m3) 120 else 112)); + const carry_increment = try builder.select(carry, try builder.constant(1), try builder.constant(0)); + const target_exponent = try builder.binary(.add, exponent_base, carry_increment); + const scaled_exponent = try builder.shifted(.shl, target_exponent, if (e4m3) 3 else 2); + const normal = try builder.binary(.add, scaled_exponent, try builder.binary(.sub, rounded, try builder.constant(retained_value))); + const finite_magnitude = try builder.select(try builder.compare(.lt, source_exponent, try builder.constant(threshold)), subnormal, normal); + const max_finite: u32 = if (e4m3) 0x7e else 0x7b; + const saturated = try builder.select(try builder.compare(.gt, finite_magnitude, try builder.constant(max_finite)), try builder.constant(max_finite), finite_magnitude); + const finite = try builder.binary(.bit_or, sign, saturated); + + const nan = if (e4m3) + try builder.binary(.bit_or, sign, try builder.constant(0x7f)) + else blk: { + const retained_nan = try builder.masked(try builder.shifted(.shr, source_fraction, 21), 0x03); + const payload = try builder.binary(.bit_or, try builder.constant(0x7e), retained_nan); + break :blk try builder.binary(.bit_or, sign, payload); + }; + const infinity = try builder.binary(.bit_or, sign, try builder.constant(if (e4m3) 0x7e else 0x7c)); + const special = try builder.select(try builder.compare(.ne, source_fraction, try builder.constant(0)), nan, infinity); + const non_special = try builder.select(try builder.equalConstant(source_exponent, 0), sign, finite); + return builder.select(try builder.equalConstant(source_exponent, 0xff), special, non_special); +} + +fn appendNvFp4Fixture( + func: *Function, + block: Block, + dequantize_direction: bool, + block_application: nvfp4.ScaleApplication, + global_application: nvfp4.ScaleApplication, +) !Value { + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const value = try func.appendBlockParam(block, f32_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: function.NvFp4Convert = .{ + .value = if (dequantize_direction) payload else value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = block_application, + .global_application = global_application, + }; + return func.appendInst(block, if (dequantize_direction) f32_t else u8_t, if (dequantize_direction) + .{ .dequantize_nvfp4 = conversion } + else + .{ .quantize_nvfp4 = conversion }); +} + +fn constantU32(func: *const Function, value: Value) ?u32 { + const inst = func.definingInst(value) orelse return null; + return switch (func.opcode(inst)) { + .iconst => |constant| @bitCast(@as(i32, @truncate(constant))), + else => null, + }; +} + +test "expandNvFp4 emits ordered integer classification and two f32 scale operations" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |dequantize_direction| { + inline for (.{ nvfp4.ScaleApplication.multiply, nvfp4.ScaleApplication.divide }) |block_application| { + inline for (.{ nvfp4.ScaleApplication.multiply, nvfp4.ScaleApplication.divide }) |global_application| { + var func = Function.init(allocator); + defer func.deinit(); + const block = try func.appendBlock(); + const original = try appendNvFp4Fixture(&func, block, dequantize_direction, block_application, global_application); + func.setTerminator(block, .{ .ret = function.Ret.one(original) }); + const original_params = try allocator.dupe(Value, func.blockParams(block)); + defer allocator.free(original_params); + + try std.testing.expect(try expandNvFp4(allocator, &func)); + try std.testing.expect(!(try expandNvFp4(allocator, &func))); + var diags = try verify.verify(allocator, &func, .low); + defer diags.deinit(); + try std.testing.expect(diags.ok()); + + var float_arith: [2]Inst = undefined; + var float_arith_count: usize = 0; + var carrier_masked = false; + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .dequantize_nvfp4, .quantize_nvfp4, .decode_low_float, .encode_low_float, .call, .call_indirect => return error.TestUnexpectedResult, + .arith => |arith| { + const result = func.instResult(inst).?; + if (func.valueType(result) == try func.types.intern(.{ .float = .f32 })) { + try std.testing.expect(float_arith_count < float_arith.len); + float_arith[float_arith_count] = inst; + float_arith_count += 1; + } else if (dequantize_direction and arith.op == .bit_and and + constantU32(&func, arith.rhs) == 0x0f) + { + const widened_inst = func.definingInst(arith.lhs) orelse continue; + carrier_masked = carrier_masked or switch (func.opcode(widened_inst)) { + .convert => |conversion| conversion.value == original_params[0], + else => false, + }; + } + }, + .icmp => |compare| { + try std.testing.expect(func.types.type_kind(func.valueType(compare.lhs)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(compare.rhs)) == .int); + }, + .select => |select| { + try std.testing.expect(func.types.type_kind(func.valueType(select.then)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(select.@"else")) == .int); + }, + .convert => |conversion| { + try std.testing.expect(func.types.type_kind(func.valueType(conversion.value)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(func.instResult(inst).?)) == .int); + }, + .unary => |unary| try std.testing.expectEqual(function.UnaryOp.reinterpret, unary.op), + else => {}, + }; + try std.testing.expectEqual(@as(usize, 2), float_arith_count); + const first_result = func.instResult(float_arith[0]).?; + const first = func.opcode(float_arith[0]).arith; + const second = func.opcode(float_arith[1]).arith; + try std.testing.expectEqual(first_result, second.lhs); + try std.testing.expectEqual( + scaleOp(if (dequantize_direction) block_application else global_application, !dequantize_direction), + first.op, + ); + try std.testing.expectEqual( + scaleOp(if (dequantize_direction) global_application else block_application, !dequantize_direction), + second.op, + ); + + const replacement = func.terminator(block).?.ret.values[0]; + if (dequantize_direction) { + try std.testing.expect(carrier_masked); + const final_reinterpret = func.opcode(func.definingInst(replacement).?).unary; + const bits_reinterpret = func.opcode(func.definingInst(final_reinterpret.value).?).unary; + try std.testing.expectEqual(first_result, second.lhs); + try std.testing.expectEqual(func.instResult(float_arith[1]).?, bits_reinterpret.value); + } else { + const narrow = func.opcode(func.definingInst(replacement).?).convert; + try std.testing.expectEqual(.int, std.meta.activeTag(func.types.type_kind(func.valueType(narrow.value)))); + try std.testing.expectEqual(@as(u16, 8), func.types.type_kind(func.valueType(replacement)).int.bits); + } + } + } + } +} + +test "expandNvFp4 preserves program order and replaces uses across blocks" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const entry = try func.appendBlock(); + const next = try func.appendBlock(); + const first = try appendNvFp4Fixture(&func, entry, true, .multiply, .divide); + const entry_params = func.blockParams(entry); + const addend = entry_params[2]; + const sum = try func.appendInst(entry, f32_t, .{ .arith = .{ .op = .add, .lhs = first, .rhs = addend } }); + try func.setJump(entry, next, &.{ sum, entry_params[1], entry_params[3] }); + const forwarded = try func.appendBlockParam(next, f32_t); + const next_block_scale = try func.appendBlockParam(next, func.valueType(entry_params[1])); + const next_global_scale = try func.appendBlockParam(next, f32_t); + const second = try func.appendInst(next, try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }), .{ .quantize_nvfp4 = .{ + .value = forwarded, + .block_scale = next_block_scale, + .global_scale = next_global_scale, + .block_application = .divide, + .global_application = .multiply, + } }); + func.setTerminator(next, .{ .ret = function.Ret.one(second) }); + + try std.testing.expect(try expandNvFp4(allocator, &func)); + var diags = try verify.verify(allocator, &func, .low); + defer diags.deinit(); + try std.testing.expect(diags.ok()); + for (0..func.blockCount()) |block_index| { + for (func.blockInsts(@enumFromInt(block_index))) |inst| switch (func.opcode(inst)) { + .dequantize_nvfp4, .quantize_nvfp4 => return error.TestUnexpectedResult, + else => {}, + }; + } + const jump_arg = func.blockArgs(func.terminator(entry).?.jump)[0]; + try std.testing.expectEqual(sum, jump_arg); + try std.testing.expect(func.opcode(func.definingInst(sum).?).arith.lhs != first); +} + +test "expandNvFp4 preserves program order and replaces uses for multiple conversions in one block" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const first = try appendNvFp4Fixture(&func, block, true, .multiply, .divide); + const second = try appendNvFp4Fixture(&func, block, true, .divide, .multiply); + const sum = try func.appendInst(block, f32_t, .{ .arith = .{ .op = .add, .lhs = first, .rhs = second } }); + func.setTerminator(block, .{ .ret = function.Ret.one(sum) }); + + try std.testing.expect(try expandNvFp4(allocator, &func)); + var diags = try verify.verify(allocator, &func, .low); + defer diags.deinit(); + try std.testing.expect(diags.ok()); + + var float_ops: [5]BinOp = undefined; + var float_op_count: usize = 0; + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .dequantize_nvfp4, .quantize_nvfp4 => return error.TestUnexpectedResult, + .arith => |arith| if (func.valueType(func.instResult(inst).?) == f32_t) { + try std.testing.expect(float_op_count < float_ops.len); + float_ops[float_op_count] = arith.op; + float_op_count += 1; + }, + else => {}, + }; + try std.testing.expectEqual(float_ops.len, float_op_count); + try std.testing.expectEqualSlices(BinOp, &.{ .mul, .div, .div, .mul, .add }, &float_ops); + + const rewritten_sum = func.opcode(func.definingInst(sum).?).arith; + try std.testing.expect(rewritten_sum.lhs != first); + try std.testing.expect(rewritten_sum.rhs != second); + const instructions = func.blockInsts(block); + const first_at = std.mem.indexOfScalar(Inst, instructions, func.definingInst(rewritten_sum.lhs).?).?; + const second_at = std.mem.indexOfScalar(Inst, instructions, func.definingInst(rewritten_sum.rhs).?).?; + const sum_at = std.mem.indexOfScalar(Inst, instructions, func.definingInst(sum).?).?; + try std.testing.expect(first_at < second_at); + try std.testing.expect(second_at < sum_at); +} + +test "expandNvFp4 migrates attributes to semantic boundaries" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |dequantize_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const block = try func.appendBlock(); + const original = try appendNvFp4Fixture(&func, block, dequantize_direction, .multiply, .divide); + const original_inst = func.definingInst(original).?; + try func.addAttr(.{ .inst = original_inst }, .{ .custom = .{ + .namespace = "debug", + .key = "line", + .value = .{ .int = 43 }, + } }); + try func.addAttr(.{ .value = original }, .{ .custom = .{ + .namespace = "test", + .key = "semantic", + .value = .flag, + } }); + func.setTerminator(block, .{ .ret = function.Ret.one(original) }); + + try std.testing.expect(try expandNvFp4(allocator, &func)); + const replacement = func.terminator(block).?.ret.values[0]; + var value_attrs = func.attributesOf(.{ .value = replacement }); + try std.testing.expectEqualStrings("semantic", value_attrs.next().?.custom.key); + var old_value_attrs = func.attributesOf(.{ .value = original }); + var old_inst_attrs = func.attributesOf(.{ .inst = original_inst }); + try std.testing.expect(old_value_attrs.next() == null); + try std.testing.expect(old_inst_attrs.next() == null); + + var boundary: ?Inst = null; + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .arith => |arith| if (dequantize_direction and + func.valueType(func.instResult(inst).?) == try func.types.intern(.{ .float = .f32 }) and + (arith.op == .mul or arith.op == .div)) + { + boundary = inst; + }, + .unary => |unary| if (!dequantize_direction and unary.op == .reinterpret and + func.types.type_kind(func.valueType(func.instResult(inst).?)) == .int) + { + boundary = inst; + }, + else => {}, + }; + try std.testing.expect(boundary != null); + var inst_attrs = func.attributesOf(.{ .inst = boundary.? }); + try std.testing.expectEqualStrings("line", inst_attrs.next().?.custom.key); + } +} + +test "expandNvFp4 keeps reinterpret barriers before original add and subtract consumers" { + const allocator = std.testing.allocator; + inline for (.{ BinOp.add, BinOp.sub }) |consumer_op| { + var func = Function.init(allocator); + defer func.deinit(); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const decoded = try appendNvFp4Fixture(&func, block, true, .multiply, .multiply); + const addend = func.blockParams(block)[2]; + const consumed = try func.appendInst(block, f32_t, .{ .arith = .{ .op = consumer_op, .lhs = decoded, .rhs = addend } }); + func.setTerminator(block, .{ .ret = function.Ret.one(consumed) }); + + try std.testing.expect(try expandNvFp4(allocator, &func)); + const consumer = func.opcode(func.definingInst(consumed).?).arith; + const final_reinterpret = func.opcode(func.definingInst(consumer.lhs).?).unary; + const bits_reinterpret = func.opcode(func.definingInst(final_reinterpret.value).?).unary; + const final_scale = func.opcode(func.definingInst(bits_reinterpret.value).?).arith; + try std.testing.expectEqual(function.UnaryOp.reinterpret, final_reinterpret.op); + try std.testing.expectEqual(function.UnaryOp.reinterpret, bits_reinterpret.op); + try std.testing.expectEqual(BinOp.mul, final_scale.op); + } +} + +test "expandNvFp4 leaves unrelated logical IR unchanged" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const block = try func.appendBlock(); + const value = try func.appendInst(block, u32_t, .{ .iconst = 43 }); + try func.addAttr(.{ .value = value }, .{ .custom = .{ + .namespace = "test", + .key = "unchanged", + .value = .flag, + } }); + func.setTerminator(block, .{ .ret = function.Ret.one(value) }); + const before = try bitcode.encode(allocator, &func); + defer allocator.free(before); + const text_before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_before); + + try std.testing.expect(!(try expandNvFp4(allocator, &func))); + const after = try bitcode.encode(allocator, &func); + defer allocator.free(after); + const text_after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_after); + try std.testing.expectEqualSlices(u8, before, after); + try std.testing.expectEqualStrings(text_before, text_after); +} + +fn appendLowFloatFixture(func: *Function, block: Block, format: function.LowFloatFormat) ![2]Value { + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const payload = try func.appendBlockParam(block, payload_t); + const source_bits = try func.appendBlockParam(block, u32_t); + const source = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = source_bits } }); + const decoded = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .format = format, .value = payload } }); + const decoded_bits = try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = decoded } }); + const encoded = try func.appendInst(block, payload_t, .{ .encode_low_float = .{ .format = format, .value = source } }); + const encoded_wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = encoded } }); + return .{ decoded_bits, encoded_wide }; +} + +test "expandLowFloat rewrites all formats and directions into integer operations" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const block = try func.appendBlock(); + var results: [6]Value = undefined; + var index: usize = 0; + for (std.enums.values(function.LowFloatFormat)) |format| { + const pair = try appendLowFloatFixture(&func, block, format); + results[index] = pair[0]; + results[index + 1] = pair[1]; + index += 2; + } + var combined = results[0]; + for (results[1..]) |result| { + combined = try func.appendInst(block, func.valueType(result), .{ .arith = .{ .op = .bit_xor, .lhs = combined, .rhs = result } }); + } + func.setTerminator(block, .{ .ret = function.Ret.one(combined) }); + + try std.testing.expect(try expandLowFloat(allocator, &func)); + try std.testing.expect(!(try expandLowFloat(allocator, &func))); + var diags = try verify.verify(allocator, &func, .low); + defer diags.deinit(); + try std.testing.expect(diags.ok()); + + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .decode_low_float, .encode_low_float, .call => return error.TestUnexpectedResult, + .arith => |op| { + try std.testing.expect(func.types.type_kind(func.valueType(func.instResult(inst).?)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(op.lhs)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(op.rhs)) == .int); + }, + .icmp => |op| { + try std.testing.expect(func.types.type_kind(func.valueType(op.lhs)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(op.rhs)) == .int); + }, + .select => |op| { + try std.testing.expect(func.types.type_kind(func.valueType(func.instResult(inst).?)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(op.then)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(op.@"else")) == .int); + }, + .convert => |op| { + try std.testing.expect(func.types.type_kind(func.valueType(func.instResult(inst).?)) == .int); + try std.testing.expect(func.types.type_kind(func.valueType(op.value)) == .int); + }, + .unary => |op| { + try std.testing.expectEqual(function.UnaryOp.reinterpret, op.op); + const result_kind = func.types.type_kind(func.valueType(func.instResult(inst).?)); + const source_kind = func.types.type_kind(func.valueType(op.value)); + try std.testing.expect((result_kind == .float and source_kind == .int) or + (result_kind == .int and source_kind == .float)); + }, + else => {}, + }; +} + +test "expandLowFloat replaces uses across blocks in program order" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const entry = try func.appendBlock(); + const next = try func.appendBlock(); + const payload = try func.appendBlockParam(entry, u8_t); + const decoded = try func.appendInst(entry, f32_t, .{ .decode_low_float = .{ .format = .f8_e4m3, .value = payload } }); + try func.setJump(entry, next, &.{decoded}); + const forwarded = try func.appendBlockParam(next, f32_t); + const encoded = try func.appendInst(next, u8_t, .{ .encode_low_float = .{ .format = .f8_e5m2, .value = forwarded } }); + func.setTerminator(next, .{ .ret = function.Ret.one(encoded) }); + + try std.testing.expect(try expandLowFloat(allocator, &func)); + var diags = try verify.verify(allocator, &func, .low); + defer diags.deinit(); + try std.testing.expect(diags.ok()); + for (0..func.blockCount()) |block_index| { + for (func.blockInsts(@enumFromInt(block_index))) |inst| switch (func.opcode(inst)) { + .decode_low_float, .encode_low_float => return error.TestUnexpectedResult, + else => {}, + }; + } +} + +test "expandLowFloat migrates live value and debug attributes" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u16_t); + const decoded = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .format = .bf16, .value = payload } }); + const original_inst = func.definingInst(decoded).?; + try func.addAttr(.{ .value = decoded }, .{ .custom = .{ .namespace = "test", .key = "semantic", .value = .flag } }); + try func.addAttr(.{ .inst = original_inst }, .{ .custom = .{ .namespace = "debug", .key = "line", .value = .{ .int = 41 } } }); + func.setTerminator(block, .{ .ret = function.Ret.one(decoded) }); + + try std.testing.expect(try expandLowFloat(allocator, &func)); + const replacement = func.terminator(block).?.ret.values[0]; + const boundary = func.definingInst(replacement).?; + var value_attrs = func.attributesOf(.{ .value = replacement }); + var inst_attrs = func.attributesOf(.{ .inst = boundary }); + var old_value_attrs = func.attributesOf(.{ .value = decoded }); + var old_inst_attrs = func.attributesOf(.{ .inst = original_inst }); + try std.testing.expect(value_attrs.next() != null); + try std.testing.expect(inst_attrs.next() != null); + try std.testing.expect(old_value_attrs.next() == null); + try std.testing.expect(old_inst_attrs.next() == null); + var diags = try verify.verify(allocator, &func, .low); + defer diags.deinit(); + try std.testing.expect(diags.ok()); +} + +test "expandLowFloat moves encode attributes to the result and f32 boundary" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, f32_t); + const encoded = try func.appendInst(block, u8_t, .{ .encode_low_float = .{ .format = .f8_e5m2, .value = source } }); + const original_inst = func.definingInst(encoded).?; + try func.addAttr(.{ .value = encoded }, .{ .custom = .{ .namespace = "test", .key = "semantic", .value = .flag } }); + try func.addAttr(.{ .inst = original_inst }, .{ .custom = .{ .namespace = "debug", .key = "line", .value = .{ .int = 42 } } }); + func.setTerminator(block, .{ .ret = function.Ret.one(encoded) }); + + try std.testing.expect(try expandLowFloat(allocator, &func)); + const replacement = func.terminator(block).?.ret.values[0]; + var boundary: ?Inst = null; + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .unary => |op| if (op.op == .reinterpret and op.value == source) { + boundary = inst; + }, + else => {}, + }; + try std.testing.expect(boundary != null); + var value_attrs = func.attributesOf(.{ .value = replacement }); + var inst_attrs = func.attributesOf(.{ .inst = boundary.? }); + var old_value_attrs = func.attributesOf(.{ .value = encoded }); + var old_inst_attrs = func.attributesOf(.{ .inst = original_inst }); + try std.testing.expect(value_attrs.next() != null); + try std.testing.expect(inst_attrs.next() != null); + try std.testing.expect(old_value_attrs.next() == null); + try std.testing.expect(old_inst_attrs.next() == null); + var diags = try verify.verify(allocator, &func, .low); + defer diags.deinit(); + try std.testing.expect(diags.ok()); +} + +test "expandLowFloat clamps eager FP8 subnormal shifts outside their path" { + const allocator = std.testing.allocator; + for ([_]struct { format: function.LowFloatFormat, shift_bias: u32 }{ + .{ .format = .f8_e4m3, .shift_bias = 141 }, + .{ .format = .f8_e5m2, .shift_bias = 134 }, + }) |case| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const bits = try func.appendInst(block, u32_t, .{ .iconst = @as(i64, case.shift_bias << 23) }); + const source = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = bits } }); + const encoded = try func.appendInst(block, u8_t, .{ .encode_low_float = .{ .format = case.format, .value = source } }); + func.setTerminator(block, .{ .ret = function.Ret.one(encoded) }); + + try std.testing.expect(try expandLowFloat(allocator, &func)); + var guarded_shift_count: usize = 0; + for (func.blockInsts(block)) |inst| switch (func.opcode(inst)) { + .arith => |op| if (op.op == .shr or op.op == .shl) { + const amount_inst = func.definingInst(op.rhs) orelse continue; + const amount_select = switch (func.opcode(amount_inst)) { + .select => |select| select, + else => continue, + }; + const fallback_inst = func.definingInst(amount_select.@"else").?; + try std.testing.expectEqual(@as(i64, 24), func.opcode(fallback_inst).iconst); + guarded_shift_count += 1; + }, + else => {}, + }; + try std.testing.expectEqual(@as(usize, 2), guarded_shift_count); + } +} + +test "expandLowFloat leaves a function without conversions byte-for-byte unchanged" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const block = try func.appendBlock(); + const value = try func.appendInst(block, u32_t, .{ .iconst = 41 }); + func.setTerminator(block, .{ .ret = function.Ret.one(value) }); + const before = try bitcode.encode(allocator, &func); + defer allocator.free(before); + + try std.testing.expect(!(try expandLowFloat(allocator, &func))); + const after = try bitcode.encode(allocator, &func); + defer allocator.free(after); + try std.testing.expectEqualSlices(u8, before, after); +} + /// Rewrite every `arith` with op `.mulh` in `func` into an equivalent limb sequence. Returns /// whether anything was rewritten. pub fn expandMulh(allocator: std.mem.Allocator, func: *Function) std.mem.Allocator.Error!bool { diff --git a/libs/vulcan-ir/function.zig b/libs/vulcan-ir/function.zig index 137f6fb..5bb7f57 100644 --- a/libs/vulcan-ir/function.zig +++ b/libs/vulcan-ir/function.zig @@ -6,6 +6,8 @@ const std = @import("std"); const types = @import("types.zig"); const attribute = @import("attribute.zig"); +const low_float = @import("low_float.zig"); +const nvfp4 = @import("nvfp4.zig"); const Type = types.Type; const TypeTable = types.TypeTable; @@ -103,6 +105,21 @@ pub const Extract = struct { aggregate: Value, index: u32 }; /// direction is read from the source value's type versus the result type. pub const Convert = struct { value: Value }; +pub const LowFloatFormat = low_float.Format; + +pub const LowFloatConvert = struct { + value: Value, + format: LowFloatFormat, +}; + +pub const NvFp4Convert = struct { + value: Value, + block_scale: Value, + global_scale: Value, + block_application: nvfp4.ScaleApplication, + global_application: nvfp4.ScaleApplication, +}; + /// A single-operand operation. `reinterpret` reinterprets the bits as the result /// type (int<->float, same width), the rest are floating-point math on the result /// type. @@ -558,6 +575,10 @@ pub const Opcode = union(enum) { extract: Extract, /// Convert a value to the result type (int<->float numeric conversion). convert: Convert, + decode_low_float: LowFloatConvert, + encode_low_float: LowFloatConvert, + dequantize_nvfp4: NvFp4Convert, + quantize_nvfp4: NvFp4Convert, /// A single-operand op (bit reinterpret, or floating-point math) on the result type. unary: Unary, /// Reserve a stack slot and produce its address. Result type is `ptr`. @@ -798,6 +819,14 @@ pub const Function = struct { return self.attributes.items; } + /// Move every attribute from one live IR entity to another. Attribute payload ownership does + /// not change. This is for replacement passes which remove the old entity instead of cloning it. + pub fn retargetAttrs(self: *Function, from: AttrTarget, to: AttrTarget) void { + for (self.attributes.items) |*entry| { + if (std.meta.eql(entry.target, from)) entry.target = to; + } + } + /// True when the memory operation `inst` carries an `endian` attribute, in either of the /// two places the verifier accepts it: on the instruction, or on the instruction's result /// value. A load is tagged on its result, a store on the instruction, and this answers for @@ -1377,6 +1406,12 @@ pub const Function = struct { }, .extract => |*e| e.aggregate = r(from, to, e.aggregate), .convert => |*cv| cv.value = r(from, to, cv.value), + .decode_low_float, .encode_low_float => |*cv| cv.value = r(from, to, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |*cv| { + cv.value = r(from, to, cv.value); + cv.block_scale = r(from, to, cv.block_scale); + cv.global_scale = r(from, to, cv.global_scale); + }, .unary => |*u| u.value = r(from, to, u.value), .load => |*l| l.ptr = r(from, to, l.ptr), .store => |*st| { @@ -1634,6 +1669,22 @@ pub const Function = struct { .struct_new => |sn| .{ .struct_new = .{ .fields = try self.remapValueList(allocator, sn.fields, map) } }, .extract => |ex| .{ .extract = .{ .aggregate = remapValue(map, ex.aggregate), .index = ex.index } }, .convert => |cv| .{ .convert = .{ .value = remapValue(map, cv.value) } }, + .decode_low_float => |cv| .{ .decode_low_float = .{ .value = remapValue(map, cv.value), .format = cv.format } }, + .encode_low_float => |cv| .{ .encode_low_float = .{ .value = remapValue(map, cv.value), .format = cv.format } }, + .dequantize_nvfp4 => |cv| .{ .dequantize_nvfp4 = .{ + .value = remapValue(map, cv.value), + .block_scale = remapValue(map, cv.block_scale), + .global_scale = remapValue(map, cv.global_scale), + .block_application = cv.block_application, + .global_application = cv.global_application, + } }, + .quantize_nvfp4 => |cv| .{ .quantize_nvfp4 = .{ + .value = remapValue(map, cv.value), + .block_scale = remapValue(map, cv.block_scale), + .global_scale = remapValue(map, cv.global_scale), + .block_application = cv.block_application, + .global_application = cv.global_application, + } }, .unary => |u| .{ .unary = .{ .op = u.op, .value = remapValue(map, u.value) } }, .call => |c| .{ .call = .{ .symbol = c.symbol, @@ -2110,6 +2161,25 @@ fn printInst(self: *const Function, w: *std.Io.Writer, inst: Inst) std.Io.Writer self.types.fmt(self.valueType(data.result.?)), self.valueName(cv.value), }), + .decode_low_float => |cv| try w.print("let v{d} = decode_low_float {s}, v{d}", .{ + self.valueName(data.result.?), + @tagName(cv.format), + self.valueName(cv.value), + }), + .encode_low_float => |cv| try w.print("let v{d} = encode_low_float {s}, v{d}", .{ + self.valueName(data.result.?), + @tagName(cv.format), + self.valueName(cv.value), + }), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| try w.print("let v{d} = {s} {s}, {s}, v{d}, v{d}, v{d}", .{ + self.valueName(data.result.?), + if (std.meta.activeTag(data.op) == .dequantize_nvfp4) "dequantize_nvfp4" else "quantize_nvfp4", + @tagName(cv.block_application), + @tagName(cv.global_application), + self.valueName(cv.value), + self.valueName(cv.block_scale), + self.valueName(cv.global_scale), + }), .unary => |u| try w.print("let v{d} = {s} {f}, v{d}", .{ self.valueName(data.result.?), @tagName(u.op), @@ -3017,6 +3087,113 @@ test "cloneBlock leaves external references unchanged" { try std.testing.expectEqual(ext, dst_arith.rhs); // external operand: identical, unchanged } +test "cloneBlock remaps low float operands without changing formats" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + + const f32_t = try func.types.intern(.{ .float = .f32 }); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const src = try func.appendBlock(); + const input = try func.appendBlockParam(src, f32_t); + const payload = try func.appendInst(src, u8_t, .{ .encode_low_float = .{ + .value = input, + .format = .f8_e4m3, + } }); + _ = try func.appendInst(src, f32_t, .{ .decode_low_float = .{ + .value = payload, + .format = .f8_e5m2, + } }); + + var map: std.AutoHashMapUnmanaged(Value, Value) = .empty; + defer map.deinit(allocator); + const dst = try func.cloneBlock(allocator, src, &map); + const insts = func.blockInsts(dst); + const encode_op = func.opcode(insts[0]).encode_low_float; + const decode_op = func.opcode(insts[1]).decode_low_float; + try std.testing.expectEqual(LowFloatFormat.f8_e4m3, encode_op.format); + try std.testing.expectEqual(LowFloatFormat.f8_e5m2, decode_op.format); + try std.testing.expectEqual(map.get(input).?, encode_op.value); + try std.testing.expectEqual(map.get(payload).?, decode_op.value); + try std.testing.expect(encode_op.value != input); + try std.testing.expect(decode_op.value != payload); +} + +test "NVFP4 replacement and clone cover every operand and policy" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const src = try func.appendBlock(); + const payload = try func.appendBlockParam(src, u8_t); + const block_scale = try func.appendBlockParam(src, u8_t); + const global_scale = try func.appendBlockParam(src, f32_t); + const decoded = try func.appendInst(src, f32_t, .{ .dequantize_nvfp4 = .{ + .value = payload, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + } }); + const decoded_inst = func.definingInst(decoded).?; + try func.addAttr(.{ .inst = decoded_inst }, .{ .custom = .{ + .namespace = "test", + .key = "nvfp4_inst", + .value = .flag, + } }); + try func.addAttr(.{ .value = decoded }, .{ .custom = .{ + .namespace = "test", + .key = "nvfp4_result", + .value = .flag, + } }); + _ = try func.appendInst(src, u8_t, .{ .quantize_nvfp4 = .{ + .value = decoded, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .divide, + .global_application = .multiply, + } }); + + var map: std.AutoHashMapUnmanaged(Value, Value) = .empty; + defer map.deinit(allocator); + const dst = try func.cloneBlock(allocator, src, &map); + const copied = func.blockInsts(dst); + const dequantize = func.opcode(copied[0]).dequantize_nvfp4; + const quantize = func.opcode(copied[1]).quantize_nvfp4; + try std.testing.expectEqual(map.get(payload).?, dequantize.value); + try std.testing.expectEqual(map.get(block_scale).?, dequantize.block_scale); + try std.testing.expectEqual(map.get(global_scale).?, dequantize.global_scale); + try std.testing.expectEqual(nvfp4.ScaleApplication.multiply, dequantize.block_application); + try std.testing.expectEqual(nvfp4.ScaleApplication.divide, dequantize.global_application); + try std.testing.expectEqual(map.get(decoded).?, quantize.value); + try std.testing.expectEqual(map.get(block_scale).?, quantize.block_scale); + try std.testing.expectEqual(map.get(global_scale).?, quantize.global_scale); + try std.testing.expectEqual(nvfp4.ScaleApplication.divide, quantize.block_application); + try std.testing.expectEqual(nvfp4.ScaleApplication.multiply, quantize.global_application); + var inst_attrs = func.attributesOf(.{ .inst = copied[0] }); + try std.testing.expectEqualStrings("nvfp4_inst", inst_attrs.next().?.custom.key); + var result_attrs = func.attributesOf(.{ .value = map.get(decoded).? }); + try std.testing.expectEqualStrings("nvfp4_result", result_attrs.next().?.custom.key); + + const fresh_payload = try func.appendBlockParam(dst, u8_t); + const fresh_block = try func.appendBlockParam(dst, u8_t); + const fresh_global = try func.appendBlockParam(dst, f32_t); + const fresh_value = try func.appendBlockParam(dst, f32_t); + func.replaceAllUses(map.get(payload).?, fresh_payload); + func.replaceAllUses(map.get(block_scale).?, fresh_block); + func.replaceAllUses(map.get(global_scale).?, fresh_global); + func.replaceAllUses(map.get(decoded).?, fresh_value); + const replaced = func.opcode(copied[0]).dequantize_nvfp4; + try std.testing.expectEqual(fresh_payload, replaced.value); + try std.testing.expectEqual(fresh_block, replaced.block_scale); + try std.testing.expectEqual(fresh_global, replaced.global_scale); + const replaced_quantize = func.opcode(copied[1]).quantize_nvfp4; + try std.testing.expectEqual(fresh_value, replaced_quantize.value); + try std.testing.expectEqual(fresh_block, replaced_quantize.block_scale); + try std.testing.expectEqual(fresh_global, replaced_quantize.global_scale); +} + test "cloneBlock remaps a ret terminator's value" { var func = Function.init(std.testing.allocator); defer func.deinit(); diff --git a/libs/vulcan-ir/legalize.zig b/libs/vulcan-ir/legalize.zig index 6d06f95..f447032 100644 --- a/libs/vulcan-ir/legalize.zig +++ b/libs/vulcan-ir/legalize.zig @@ -319,7 +319,7 @@ fn foldArith(op: function.BinOp, lhs: i64, rhs: i64) ?i64 { /// Whether an instruction has no side effects, so it may be dropped when unused. fn isPure(op: function.Opcode) bool { return switch (op) { - .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .struct_new, .extract, .convert, .unary, .alloca, .global_addr, .dot => true, + .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .struct_new, .extract, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .alloca, .global_addr, .dot => true, // Loads are kept conservatively. Stores, `if`, and calls have effects. // A prefetch hint behaves like store here (effectful, not droppable). // A matmul writes the `c` memory, likewise effectful. @@ -405,6 +405,12 @@ fn applySubst(func: *Function, subst: *const Subst) void { }, .extract => |*ex| ex.aggregate = sub(subst, ex.aggregate), .convert => |*cv| cv.value = sub(subst, cv.value), + .decode_low_float, .encode_low_float => |*cv| cv.value = sub(subst, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |*cv| { + cv.value = sub(subst, cv.value); + cv.block_scale = sub(subst, cv.block_scale); + cv.global_scale = sub(subst, cv.global_scale); + }, .unary => |*u| u.value = sub(subst, u.value), .load => |*ld| ld.ptr = sub(subst, ld.ptr), .store => |*st| { @@ -517,6 +523,12 @@ fn countUses(func: *const Function, uses: []u32) void { }, .extract => |ex| uses[@intFromEnum(ex.aggregate)] += 1, .convert => |cv| uses[@intFromEnum(cv.value)] += 1, + .decode_low_float, .encode_low_float => |cv| uses[@intFromEnum(cv.value)] += 1, + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + uses[@intFromEnum(cv.value)] += 1; + uses[@intFromEnum(cv.block_scale)] += 1; + uses[@intFromEnum(cv.global_scale)] += 1; + }, .unary => |u| uses[@intFromEnum(u.value)] += 1, .load => |ld| uses[@intFromEnum(ld.ptr)] += 1, .store => |st| { diff --git a/libs/vulcan-ir/low_float.zig b/libs/vulcan-ir/low_float.zig new file mode 100644 index 0000000..637c6dc --- /dev/null +++ b/libs/vulcan-ir/low_float.zig @@ -0,0 +1,283 @@ +const std = @import("std"); + +pub const Format = enum(u8) { + bf16 = 0, + f8_e4m3 = 1, + f8_e5m2 = 2, + + pub fn payloadBits(self: Format) u16 { + return switch (self) { + .bf16 => 16, + .f8_e4m3, .f8_e5m2 => 8, + }; + } +}; + +pub const Error = error{InvalidPayload}; + +pub fn decode(format: Format, payload: u16) Error!f32 { + return switch (format) { + .bf16 => @bitCast(@as(u32, payload) << 16), + .f8_e4m3 => decodeE4M3(try payload8(payload)), + .f8_e5m2 => decodeE5M2(try payload8(payload)), + }; +} + +pub fn encode(format: Format, value: f32) u16 { + return switch (format) { + .bf16 => encodeBf16(value), + .f8_e4m3 => encodeE4M3(value), + .f8_e5m2 => encodeE5M2(value), + }; +} + +fn payload8(payload: u16) Error!u8 { + if (payload > std.math.maxInt(u8)) return error.InvalidPayload; + return @intCast(payload); +} + +fn encodeBf16(value: f32) u16 { + const source: u32 = @bitCast(value); + const exponent = source & 0x7f80_0000; + const fraction = source & 0x007f_ffff; + var retained: u16 = @truncate(source >> 16); + if (exponent == 0x7f80_0000) { + if (fraction != 0) retained |= 0x0040; + return retained; + } + return @truncate((source + 0x7fff + (retained & 1)) >> 16); +} + +fn encodeE4M3(value: f32) u8 { + const source: u32 = @bitCast(value); + const sign: u8 = @truncate((source >> 24) & 0x80); + const source_exponent: u8 = @truncate((source >> 23) & 0xff); + const source_fraction = source & 0x007f_ffff; + if (source_exponent == 0xff) { + if (source_fraction != 0) return sign | 0x7f; + return sign | 0x7e; + } + if (source_exponent == 0) return sign; + + const exponent: i16 = @as(i16, source_exponent) - 127; + const significand = 0x0080_0000 | source_fraction; + var magnitude: u32 = undefined; + if (exponent < -6) { + const shift: u8 = @intCast(-9 - exponent + 23); + magnitude = roundRight(significand, shift); + } else { + var rounded = roundRight(significand, 20); + var target_exponent = exponent; + if (rounded == 16) { + rounded = 8; + target_exponent += 1; + } + magnitude = @as(u32, @intCast(target_exponent + 7)) * 8 + (rounded - 8); + } + return sign | @as(u8, @intCast(@min(magnitude, 0x7e))); +} + +fn decodeE4M3(payload: u8) f32 { + const sign = @as(u32, payload & 0x80) << 24; + const exponent = (payload >> 3) & 0x0f; + const fraction = payload & 0x07; + if (exponent == 0x0f and fraction == 0x07) return @bitCast(sign | 0x7fc0_0000); + if (exponent == 0) { + if (fraction == 0) return @bitCast(sign); + const leading: u5 = @intCast(7 - @clz(fraction)); + const f32_exponent = @as(u32, 118) + leading; + const remainder = @as(u32, fraction) - (@as(u32, 1) << leading); + return @bitCast(sign | (f32_exponent << 23) | (remainder << @intCast(23 - leading))); + } + const f32_exponent = @as(u32, exponent) + 120; + return @bitCast(sign | (f32_exponent << 23) | (@as(u32, fraction) << 20)); +} + +fn encodeE5M2(value: f32) u8 { + const source: u32 = @bitCast(value); + const sign: u8 = @truncate((source >> 24) & 0x80); + const source_exponent: u8 = @truncate((source >> 23) & 0xff); + const source_fraction = source & 0x007f_ffff; + if (source_exponent == 0xff) { + if (source_fraction == 0) return sign | 0x7c; + const payload: u8 = @truncate(source_fraction >> 21); + return sign | 0x7c | payload | 0x02; + } + if (source_exponent == 0) return sign; + + const exponent: i16 = @as(i16, source_exponent) - 127; + const significand = 0x0080_0000 | source_fraction; + var magnitude: u32 = undefined; + if (exponent < -14) { + const shift: u8 = @intCast(-16 - exponent + 23); + magnitude = roundRight(significand, shift); + } else { + var rounded = roundRight(significand, 21); + var target_exponent = exponent; + if (rounded == 8) { + rounded = 4; + target_exponent += 1; + } + magnitude = @as(u32, @intCast(target_exponent + 15)) * 4 + (rounded - 4); + } + return sign | @as(u8, @intCast(@min(magnitude, 0x7b))); +} + +fn decodeE5M2(payload: u8) f32 { + const sign = @as(u32, payload & 0x80) << 24; + const exponent = (payload >> 2) & 0x1f; + const fraction = payload & 0x03; + if (exponent == 0x1f) { + if (fraction == 0) return @bitCast(sign | 0x7f80_0000); + return @bitCast(sign | 0x7f80_0000 | (@as(u32, fraction | 0x02) << 21)); + } + if (exponent == 0) { + if (fraction == 0) return @bitCast(sign); + const leading: u5 = @intCast(7 - @clz(fraction)); + const f32_exponent = @as(u32, 111) + leading; + const remainder = @as(u32, fraction) - (@as(u32, 1) << leading); + return @bitCast(sign | (f32_exponent << 23) | (remainder << @intCast(23 - leading))); + } + const f32_exponent = @as(u32, exponent) + 112; + return @bitCast(sign | (f32_exponent << 23) | (@as(u32, fraction) << 21)); +} + +fn roundRight(value: u32, shift: u8) u32 { + if (shift > 24) return 0; + const retained = value >> @intCast(shift); + const mask = (@as(u32, 1) << @intCast(shift)) - 1; + const discarded = value & mask; + const halfway = @as(u32, 1) << @intCast(shift - 1); + return retained + @intFromBool(discarded > halfway or + (discarded == halfway and retained & 1 != 0)); +} + +pub const bf16_encode_boundary_cases = [_]struct { source: u32, expected: u16 }{ + .{ .source = 0x0000_0000, .expected = 0x0000 }, + .{ .source = 0x8000_0000, .expected = 0x8000 }, + .{ .source = 0x3f80_0000, .expected = 0x3f80 }, + .{ .source = 0x3f80_7fff, .expected = 0x3f80 }, + .{ .source = 0x3f80_8000, .expected = 0x3f80 }, + .{ .source = 0x3f80_8001, .expected = 0x3f81 }, + .{ .source = 0x3f81_7fff, .expected = 0x3f81 }, + .{ .source = 0x3f81_8000, .expected = 0x3f82 }, + .{ .source = 0x3f81_8001, .expected = 0x3f82 }, + .{ .source = 0x7f7f_ffff, .expected = 0x7f80 }, + .{ .source = 0xff7f_ffff, .expected = 0xff80 }, + .{ .source = 0x7f80_0000, .expected = 0x7f80 }, + .{ .source = 0xff80_0000, .expected = 0xff80 }, + .{ .source = 0x7f81_2345, .expected = 0x7fc1 }, + .{ .source = 0xffa5_4321, .expected = 0xffe5 }, +}; + +pub const e4_encode_boundary_cases = [_]struct { source: u32, expected: u16 }{ + .{ .source = 0x3f87_ffff, .expected = 0x38 }, + .{ .source = 0x3f88_0000, .expected = 0x38 }, + .{ .source = 0x3f88_0001, .expected = 0x39 }, + .{ .source = 0x3f97_ffff, .expected = 0x39 }, + .{ .source = 0x3f98_0000, .expected = 0x3a }, + .{ .source = 0x3f98_0001, .expected = 0x3a }, + .{ .source = 0x3a7f_ffff, .expected = 0x00 }, + .{ .source = 0x3a80_0000, .expected = 0x00 }, + .{ .source = 0x3a80_0001, .expected = 0x01 }, + .{ .source = 0x3b3f_ffff, .expected = 0x01 }, + .{ .source = 0x3b40_0000, .expected = 0x02 }, + .{ .source = 0x3b40_0001, .expected = 0x02 }, + .{ .source = 0x3c6f_ffff, .expected = 0x07 }, + .{ .source = 0x3c70_0000, .expected = 0x08 }, + .{ .source = 0x3c70_0001, .expected = 0x08 }, +}; + +pub const e5_encode_boundary_cases = [_]struct { source: u32, expected: u16 }{ + .{ .source = 0x3f8f_ffff, .expected = 0x3c }, + .{ .source = 0x3f90_0000, .expected = 0x3c }, + .{ .source = 0x3f90_0001, .expected = 0x3d }, + .{ .source = 0x3faf_ffff, .expected = 0x3d }, + .{ .source = 0x3fb0_0000, .expected = 0x3e }, + .{ .source = 0x3fb0_0001, .expected = 0x3e }, + .{ .source = 0x36ff_ffff, .expected = 0x00 }, + .{ .source = 0x3700_0000, .expected = 0x00 }, + .{ .source = 0x3700_0001, .expected = 0x01 }, + .{ .source = 0x37bf_ffff, .expected = 0x01 }, + .{ .source = 0x37c0_0000, .expected = 0x02 }, + .{ .source = 0x37c0_0001, .expected = 0x02 }, + .{ .source = 0x385f_ffff, .expected = 0x03 }, + .{ .source = 0x3860_0000, .expected = 0x04 }, + .{ .source = 0x3860_0001, .expected = 0x04 }, +}; + +pub const fp8_encode_special_cases = [_]struct { source: u32, e4: u16, e5: u16 }{ + .{ .source = 0x0000_0000, .e4 = 0x00, .e5 = 0x00 }, + .{ .source = 0x8000_0000, .e4 = 0x80, .e5 = 0x80 }, + .{ .source = 0x3f88_0000, .e4 = 0x38, .e5 = 0x3c }, + .{ .source = 0x3f88_0001, .e4 = 0x39, .e5 = 0x3c }, + .{ .source = 0x3a80_0000, .e4 = 0x00, .e5 = 0x14 }, + .{ .source = 0x3a80_0001, .e4 = 0x01, .e5 = 0x14 }, + .{ .source = 0x3700_0000, .e4 = 0x00, .e5 = 0x00 }, + .{ .source = 0x3700_0001, .e4 = 0x00, .e5 = 0x01 }, + .{ .source = 0x7f7f_ffff, .e4 = 0x7e, .e5 = 0x7b }, + .{ .source = 0xff7f_ffff, .e4 = 0xfe, .e5 = 0xfb }, + .{ .source = 0x7f80_0000, .e4 = 0x7e, .e5 = 0x7c }, + .{ .source = 0xff80_0000, .e4 = 0xfe, .e5 = 0xfc }, + .{ .source = 0x7f80_0001, .e4 = 0x7f, .e5 = 0x7e }, + .{ .source = 0xffa0_0001, .e4 = 0xff, .e5 = 0xff }, +}; + +test "BF16 payloads round trip with NaNs quieted" { + for (0..@as(usize, std.math.maxInt(u16)) + 1) |raw| { + const payload: u16 = @intCast(raw); + const expected = if (payload & 0x7f80 == 0x7f80 and payload & 0x007f != 0) + payload | 0x0040 + else + payload; + try std.testing.expectEqual(expected, encode(.bf16, try decode(.bf16, payload))); + } +} + +test "BF16 encode uses ties to even and preserves special signs" { + for (bf16_encode_boundary_cases) |case| try std.testing.expectEqual(case.expected, encode(.bf16, @bitCast(case.source))); +} + +test "FP8 payloads round trip with format-specific NaNs" { + for (0..256) |raw| { + const payload: u8 = @intCast(raw); + try std.testing.expectEqual(@as(u16, payload), encode(.f8_e4m3, try decode(.f8_e4m3, payload))); + const e5_expected = if (payload & 0x7c == 0x7c and payload & 0x03 != 0) + payload | 0x02 + else + payload; + try std.testing.expectEqual(e5_expected, encode(.f8_e5m2, try decode(.f8_e5m2, payload))); + } +} + +test "FP8 normal and subnormal boundaries round nearest ties to even" { + for (e4_encode_boundary_cases) |case| { + try std.testing.expectEqual(case.expected, encode(.f8_e4m3, @bitCast(case.source))); + try std.testing.expectEqual(case.expected | 0x80, encode(.f8_e4m3, @bitCast(case.source | 0x8000_0000))); + } + + for (e5_encode_boundary_cases) |case| { + try std.testing.expectEqual(case.expected, encode(.f8_e5m2, @bitCast(case.source))); + try std.testing.expectEqual(case.expected | 0x80, encode(.f8_e5m2, @bitCast(case.source | 0x8000_0000))); + } +} + +test "FP8 encode covers rounding saturation infinity and NaN" { + for (fp8_encode_special_cases) |case| { + const value: f32 = @bitCast(case.source); + try std.testing.expectEqual(case.e4, encode(.f8_e4m3, value)); + try std.testing.expectEqual(case.e5, encode(.f8_e5m2, value)); + } + try std.testing.expect(@as(u32, @bitCast(try decode(.f8_e4m3, 0x7c))) != + @as(u32, @bitCast(try decode(.f8_e5m2, 0x7c)))); + try std.testing.expect(@as(u32, @bitCast(try decode(.f8_e4m3, 0x7e))) != + @as(u32, @bitCast(try decode(.f8_e5m2, 0x7e)))); +} + +test "FP8 rejects upper payload bits while BF16 accepts all payloads" { + try std.testing.expectError(error.InvalidPayload, decode(.f8_e4m3, 0x0100)); + try std.testing.expectError(error.InvalidPayload, decode(.f8_e5m2, 0xffff)); + for (0..@as(usize, std.math.maxInt(u16)) + 1) |raw| { + _ = try decode(.bf16, @intCast(raw)); + } +} diff --git a/libs/vulcan-ir/nvfp4.zig b/libs/vulcan-ir/nvfp4.zig new file mode 100644 index 0000000..1b83a4c --- /dev/null +++ b/libs/vulcan-ir/nvfp4.zig @@ -0,0 +1,269 @@ +const std = @import("std"); +const low_float = @import("low_float.zig"); + +pub const ScaleApplication = enum(u8) { + multiply = 0, + divide = 1, +}; + +comptime { + if (@intFromEnum(ScaleApplication.multiply) != 0) @compileError("multiply tag changed"); + if (@intFromEnum(ScaleApplication.divide) != 1) @compileError("divide tag changed"); +} + +pub fn decodeE2M1(payload: u8) f32 { + const nibble = payload & 0x0f; + const sign = @as(u32, nibble & 0x08) << 28; + const magnitude = [_]u32{ + 0x0000_0000, + 0x3f00_0000, + 0x3f80_0000, + 0x3fc0_0000, + 0x4000_0000, + 0x4040_0000, + 0x4080_0000, + 0x40c0_0000, + }; + return @bitCast(sign | magnitude[nibble & 0x07]); +} + +pub fn encodeE2M1(value: f32) u8 { + const source: u32 = @bitCast(value); + const sign: u8 = @truncate((source >> 28) & 0x08); + const magnitude = source & 0x7fff_ffff; + if (magnitude > 0x7f80_0000) return 0x07; + if (magnitude == 0x7f80_0000) return sign | 0x07; + + const result: u8 = if (magnitude <= 0x3e80_0000) + 0x00 + else if (magnitude < 0x3f40_0000) + 0x01 + else if (magnitude <= 0x3fa0_0000) + 0x02 + else if (magnitude < 0x3fe0_0000) + 0x03 + else if (magnitude <= 0x4020_0000) + 0x04 + else if (magnitude < 0x4060_0000) + 0x05 + else if (magnitude <= 0x40a0_0000) + 0x06 + else + 0x07; + return sign | result; +} + +fn apply(value: f32, scale: f32, application: ScaleApplication) f32 { + return switch (application) { + .multiply => value * scale, + .divide => value / scale, + }; +} + +fn undo(value: f32, scale: f32, application: ScaleApplication) f32 { + return switch (application) { + .multiply => value / scale, + .divide => value * scale, + }; +} + +pub fn dequantize( + payload: u8, + block_scale: u8, + global_scale: f32, + block_application: ScaleApplication, + global_application: ScaleApplication, +) f32 { + const decoded_scale = low_float.decode(.f8_e4m3, block_scale) catch unreachable; + const local: f32 = apply(decodeE2M1(payload), decoded_scale, block_application); + return apply(local, global_scale, global_application); +} + +pub fn quantize( + value: f32, + block_scale: u8, + global_scale: f32, + block_application: ScaleApplication, + global_application: ScaleApplication, +) u8 { + const decoded_scale = low_float.decode(.f8_e4m3, block_scale) catch unreachable; + const global_unscaled: f32 = undo(value, global_scale, global_application); + const local_unscaled: f32 = undo(global_unscaled, decoded_scale, block_application); + return encodeE2M1(local_unscaled); +} + +test "all E2M1 payloads decode exactly and round trip" { + const positive = [_]u32{ 0, 0x3f00_0000, 0x3f80_0000, 0x3fc0_0000, 0x4000_0000, 0x4040_0000, 0x4080_0000, 0x40c0_0000 }; + for (0..16) |payload| { + const expected = positive[payload & 7] | @as(u32, @intFromBool(payload & 8 != 0)) << 31; + try std.testing.expectEqual(expected, @as(u32, @bitCast(decodeE2M1(@intCast(payload))))); + try std.testing.expectEqual(@as(u8, @intCast(payload)), encodeE2M1(decodeE2M1(@intCast(payload)))); + } +} + +test "E2M1 midpoint neighbors round to the adjacent even payload" { + const cases = [_]struct { midpoint: u32, lower: u8, tie: u8, upper: u8 }{ + .{ .midpoint = 0x3e80_0000, .lower = 0, .tie = 0, .upper = 1 }, + .{ .midpoint = 0x3f40_0000, .lower = 1, .tie = 2, .upper = 2 }, + .{ .midpoint = 0x3fa0_0000, .lower = 2, .tie = 2, .upper = 3 }, + .{ .midpoint = 0x3fe0_0000, .lower = 3, .tie = 4, .upper = 4 }, + .{ .midpoint = 0x4020_0000, .lower = 4, .tie = 4, .upper = 5 }, + .{ .midpoint = 0x4060_0000, .lower = 5, .tie = 6, .upper = 6 }, + .{ .midpoint = 0x40a0_0000, .lower = 6, .tie = 6, .upper = 7 }, + }; + for (cases) |case| { + try std.testing.expectEqual(case.lower, encodeE2M1(@bitCast(case.midpoint - 1))); + try std.testing.expectEqual(case.tie, encodeE2M1(@bitCast(case.midpoint))); + try std.testing.expectEqual(case.upper, encodeE2M1(@bitCast(case.midpoint + 1))); + try std.testing.expectEqual(case.lower | 8, encodeE2M1(@bitCast((case.midpoint - 1) | 0x8000_0000))); + try std.testing.expectEqual(case.tie | 8, encodeE2M1(@bitCast(case.midpoint | 0x8000_0000))); + try std.testing.expectEqual(case.upper | 8, encodeE2M1(@bitCast((case.midpoint + 1) | 0x8000_0000))); + } +} + +test "E2M1 special values preserve signs and canonicalize NaNs" { + try std.testing.expectEqual(@as(u8, 0), encodeE2M1(@bitCast(@as(u32, 1)))); + try std.testing.expectEqual(@as(u8, 8), encodeE2M1(@bitCast(@as(u32, 0x8000_0001)))); + try std.testing.expectEqual(@as(u8, 7), encodeE2M1(@bitCast(@as(u32, 0x7f7f_ffff)))); + try std.testing.expectEqual(@as(u8, 15), encodeE2M1(@bitCast(@as(u32, 0xff7f_ffff)))); + try std.testing.expectEqual(@as(u8, 7), encodeE2M1(std.math.inf(f32))); + try std.testing.expectEqual(@as(u8, 15), encodeE2M1(-std.math.inf(f32))); + for ([_]u32{ 0x7f80_0001, 0x7fc0_0000, 0xff80_0001, 0xffc0_0000 }) |bits| { + try std.testing.expectEqual(@as(u8, 7), encodeE2M1(@bitCast(bits))); + } +} + +test "E2M1 carrier masks and clears its high nibble" { + for (0..256) |carrier| { + try std.testing.expectEqual(@as(u32, @bitCast(decodeE2M1(@intCast(carrier & 0x0f)))), @as(u32, @bitCast(decodeE2M1(@intCast(carrier))))); + try std.testing.expectEqual(@as(u8, 0), encodeE2M1(@bitCast(@as(u32, @intCast(carrier)) << 23)) & 0xf0); + } +} + +test "packed bytes decode both ordered nibbles" { + for (0..256) |packed_value| { + try std.testing.expectEqual(@as(u32, @bitCast(decodeE2M1(@intCast(packed_value & 0x0f)))), @as(u32, @bitCast(decodeE2M1(@intCast(packed_value))))); + try std.testing.expectEqual(@as(u32, @bitCast(decodeE2M1(@intCast((packed_value >> 4) & 0x0f)))), @as(u32, @bitCast(decodeE2M1(@intCast(packed_value >> 4))))); + } +} + +test "scale policies use local then global order and reverse undo order" { + const cases = [_]struct { + block: ScaleApplication, + global: ScaleApplication, + expected: f32, + }{ + .{ .block = .multiply, .global = .multiply, .expected = 24 }, + .{ .block = .multiply, .global = .divide, .expected = 1.5 }, + .{ .block = .divide, .global = .multiply, .expected = 6 }, + .{ .block = .divide, .global = .divide, .expected = 0.375 }, + }; + for (cases) |case| { + const decoded = dequantize(5, 0x40, 4, case.block, case.global); + try std.testing.expectEqual(@as(u32, @bitCast(case.expected)), @as(u32, @bitCast(decoded))); + try std.testing.expectEqual(@as(u8, 5), quantize(decoded, 0x40, 4, case.block, case.global)); + } + + const dequantize_order_cases = [_]struct { + payload: u8, + block_scale: u8, + global_scale_bits: u32, + block_application: ScaleApplication, + global_application: ScaleApplication, + expected_bits: u32, + reordered_bits: u32, + }{ + .{ .payload = 0x03, .block_scale = 0x03, .global_scale_bits = 0x3e00_0001, .block_application = .multiply, .global_application = .multiply, .expected_bits = 0x3a90_0001, .reordered_bits = 0x3a90_0002 }, + .{ .payload = 0x01, .block_scale = 0x03, .global_scale_bits = 0x3e00_0001, .block_application = .multiply, .global_application = .divide, .expected_bits = 0x3cbf_ffff, .reordered_bits = 0x3cbf_fffe }, + .{ .payload = 0x01, .block_scale = 0x05, .global_scale_bits = 0x3e00_0001, .block_application = .divide, .global_application = .multiply, .expected_bits = 0x40cc_cccf, .reordered_bits = 0x40cc_ccce }, + .{ .payload = 0x01, .block_scale = 0x03, .global_scale_bits = 0x3e00_0001, .block_application = .divide, .global_application = .divide, .expected_bits = 0x442a_aaaa, .reordered_bits = 0x442a_aaa9 }, + }; + for (dequantize_order_cases) |case| { + const global_scale: f32 = @bitCast(case.global_scale_bits); + const block_scale = low_float.decode(.f8_e4m3, case.block_scale) catch unreachable; + const ordered = dequantize(case.payload, case.block_scale, global_scale, case.block_application, case.global_application); + const reordered = apply( + apply(decodeE2M1(case.payload), global_scale, case.global_application), + block_scale, + case.block_application, + ); + try std.testing.expectEqual(case.expected_bits, @as(u32, @bitCast(ordered))); + try std.testing.expectEqual(case.reordered_bits, @as(u32, @bitCast(reordered))); + try std.testing.expect(case.expected_bits != case.reordered_bits); + } + + const quantize_order_cases = [_]struct { + value_bits: u32, + block_scale: u8, + global_scale_bits: u32, + block_application: ScaleApplication, + global_application: ScaleApplication, + expected: u8, + reordered: u8, + }{ + .{ .value_bits = 0x3e00_0522, .block_scale = 0x23, .global_scale_bits = 0x3e54_d004, .block_application = .multiply, .global_application = .multiply, .expected = 0x06, .reordered = 0x05 }, + .{ .value_bits = 0x3e00_154f, .block_scale = 0x0f, .global_scale_bits = 0x3ed1_dd10, .block_application = .multiply, .global_application = .divide, .expected = 0x04, .reordered = 0x03 }, + .{ .value_bits = 0x3e00_154f, .block_scale = 0x47, .global_scale_bits = 0x3f20_1aa3, .block_application = .divide, .global_application = .multiply, .expected = 0x01, .reordered = 0x02 }, + .{ .value_bits = 0x3e00_399b, .block_scale = 0x55, .global_scale_bits = 0x3eeb_e49b, .block_application = .divide, .global_application = .divide, .expected = 0x01, .reordered = 0x02 }, + }; + for (quantize_order_cases) |case| { + const value: f32 = @bitCast(case.value_bits); + const global_scale: f32 = @bitCast(case.global_scale_bits); + const block_scale = low_float.decode(.f8_e4m3, case.block_scale) catch unreachable; + const ordered = quantize(value, case.block_scale, global_scale, case.block_application, case.global_application); + const reordered = encodeE2M1(undo( + undo(value, block_scale, case.block_application), + global_scale, + case.global_application, + )); + try std.testing.expectEqual(case.expected, ordered); + try std.testing.expectEqual(case.reordered, reordered); + try std.testing.expect(case.expected != case.reordered); + } +} + +test "scale special values follow IEEE arithmetic" { + try std.testing.expectEqual(@as(u32, 0), @as(u32, @bitCast(dequantize(4, 0x00, 2, .multiply, .multiply)))); + try std.testing.expectEqual(@as(u32, 0xc000_0000), @as(u32, @bitCast(dequantize(2, 0xb8, 2, .multiply, .multiply)))); + try std.testing.expectEqual(@as(u32, 0x43e0_0000), @as(u32, @bitCast(dequantize(2, 0x7e, 1, .multiply, .multiply)))); + try std.testing.expect(std.math.isNan(dequantize(2, 0x7f, 1, .multiply, .multiply))); + try std.testing.expectEqual(@as(u8, 7), quantize(1, 0x00, 1, .multiply, .multiply)); + try std.testing.expectEqual(@as(u8, 10), quantize(1, 0xb8, 1, .multiply, .multiply)); + try std.testing.expectEqual(@as(u8, 0), quantize(1, 0x7e, 1, .multiply, .multiply)); + try std.testing.expectEqual(@as(u8, 7), quantize(1, 0x7f, 1, .multiply, .multiply)); + + try std.testing.expectEqual(@as(u32, 0), @as(u32, @bitCast(dequantize(2, 0x38, 0, .multiply, .multiply)))); + try std.testing.expectEqual(@as(u32, 0xbf80_0000), @as(u32, @bitCast(dequantize(2, 0x38, -1, .multiply, .multiply)))); + try std.testing.expect(std.math.isInf(dequantize(2, 0x38, std.math.inf(f32), .multiply, .multiply))); + try std.testing.expect(std.math.isNan(dequantize(2, 0x38, std.math.nan(f32), .multiply, .multiply))); + try std.testing.expectEqual(@as(u8, 7), quantize(1, 0x38, 0, .multiply, .multiply)); + try std.testing.expectEqual(@as(u8, 10), quantize(1, 0x38, -1, .multiply, .multiply)); + try std.testing.expectEqual(@as(u8, 0), quantize(1, 0x38, std.math.inf(f32), .multiply, .multiply)); + try std.testing.expectEqual(@as(u8, 7), quantize(1, 0x38, std.math.nan(f32), .multiply, .multiply)); +} + +test "canonical blocks and padded row and column layouts use scalar formulas" { + var payloads: [16]u8 = undefined; + for (&payloads, 0..) |*payload, i| payload.* = @intCast(i); + for (payloads) |payload| { + const expected = decodeE2M1(payload) * 2 * 3; + try std.testing.expectEqual(@as(u32, @bitCast(expected)), @as(u32, @bitCast(dequantize(payload, 0x40, 3, .multiply, .multiply)))); + } + + const rows: usize = 3; + const columns: usize = 17; + const padded_columns = std.mem.alignForward(usize, columns, 16); + const padded_rows = std.mem.alignForward(usize, rows, 16); + const row_scales: [6]u8 = .{ 0x38, 0x40, 0x39, 0x38, 0x40, 0x39 }; + var column_scales: [34]u8 = undefined; + for (&column_scales, 0..) |*scale, i| scale.* = if (i % 2 == 0) 0x38 else 0x40; + for (0..rows) |row| for (0..columns) |column| { + const row_physical = row * padded_columns + column; + const column_physical = column * padded_rows + row; + const payload: u8 = @intCast((row + column) & 0x0f); + const row_expected = decodeE2M1(payload) * (low_float.decode(.f8_e4m3, row_scales[row_physical / 16]) catch unreachable); + const column_expected = decodeE2M1(payload) * (low_float.decode(.f8_e4m3, column_scales[column_physical / 16]) catch unreachable); + try std.testing.expectEqual(@as(u32, @bitCast(row_expected)), @as(u32, @bitCast(dequantize(payload, row_scales[row_physical / 16], 1, .multiply, .multiply)))); + try std.testing.expectEqual(@as(u32, @bitCast(column_expected)), @as(u32, @bitCast(dequantize(payload, column_scales[column_physical / 16], 1, .multiply, .multiply)))); + }; +} diff --git a/libs/vulcan-ir/parser.zig b/libs/vulcan-ir/parser.zig index 76f6733..6ad16ee 100644 --- a/libs/vulcan-ir/parser.zig +++ b/libs/vulcan-ir/parser.zig @@ -7,6 +7,7 @@ const std = @import("std"); const function = @import("function.zig"); const types = @import("types.zig"); const attribute = @import("attribute.zig"); +const nvfp4 = @import("nvfp4.zig"); const Function = function.Function; const Block = function.Block; @@ -734,6 +735,69 @@ const FunctionParser = struct { try self.recordValue(result); return result; } + if (std.mem.eql(u8, op, "decode_low_float") or + std.mem.eql(u8, op, "encode_low_float")) + { + self.skipWs(); + const format = std.meta.stringToEnum(function.LowFloatFormat, self.readWord()) orelse + return error.InvalidSyntax; + self.skipWs(); + try self.eat(','); + self.skipWs(); + const value = try self.parseValueRef(); + const decode_direction = std.mem.eql(u8, op, "decode_low_float"); + const ty = if (decode_direction) + try self.func.types.intern(.{ .float = .f32 }) + else + try self.func.types.intern(.{ .int = .{ + .signedness = .unsigned, + .bits = format.payloadBits(), + } }); + const conversion: function.LowFloatConvert = .{ .value = value, .format = format }; + const result = if (decode_direction) + try self.func.appendInst(block, ty, .{ .decode_low_float = conversion }) + else + try self.func.appendInst(block, ty, .{ .encode_low_float = conversion }); + try self.recordValue(result); + return result; + } + if (std.mem.eql(u8, op, "dequantize_nvfp4") or + std.mem.eql(u8, op, "quantize_nvfp4")) + { + self.skipWs(); + const block_application = std.meta.stringToEnum(nvfp4.ScaleApplication, self.readWord()) orelse + return error.InvalidSyntax; + self.skipWs(); + try self.eat(','); + self.skipWs(); + const global_application = std.meta.stringToEnum(nvfp4.ScaleApplication, self.readWord()) orelse + return error.InvalidSyntax; + var operands: [3]Value = undefined; + for (&operands) |*operand| { + self.skipWs(); + try self.eat(','); + self.skipWs(); + operand.* = try self.parseValueRef(); + } + const dequantize_direction = std.mem.eql(u8, op, "dequantize_nvfp4"); + const ty = if (dequantize_direction) + try self.func.types.intern(.{ .float = .f32 }) + else + try self.func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const conversion: function.NvFp4Convert = .{ + .value = operands[0], + .block_scale = operands[1], + .global_scale = operands[2], + .block_application = block_application, + .global_application = global_application, + }; + const result = if (dequantize_direction) + try self.func.appendInst(block, ty, .{ .dequantize_nvfp4 = conversion }) + else + try self.func.appendInst(block, ty, .{ .quantize_nvfp4 = conversion }); + try self.recordValue(result); + return result; + } if (std.mem.eql(u8, op, "struct")) { self.skipWs(); try self.eat('{'); @@ -1858,3 +1922,90 @@ test "verify rejects each broken atomic operand shape" { } } } + +test "low float conversions print parse and verify in every format and direction" { + const verify = @import("verify.zig"); + const text = + \\fn { + \\ block0(v0: u16, v1: u8, v2: f32): + \\ let v3 = decode_low_float bf16, v0 + \\ let v4 = decode_low_float f8_e4m3, v1 + \\ let v5 = decode_low_float f8_e5m2, v1 + \\ let v6 = encode_low_float bf16, v2 + \\ let v7 = encode_low_float f8_e4m3, v2 + \\ let v8 = encode_low_float f8_e5m2, v2 + \\ ret v3 + \\} + ; + var func = try parse(std.testing.allocator, text); + defer func.deinit(); + try std.testing.expectFmt(text, "{f}", .{func}); + var diags = try verify.verify(std.testing.allocator, &func, .high); + defer diags.deinit(); + try std.testing.expect(diags.ok()); + + const insts = func.blockInsts(@enumFromInt(0)); + inline for (.{ + function.LowFloatFormat.bf16, + function.LowFloatFormat.f8_e4m3, + function.LowFloatFormat.f8_e5m2, + }, 0..) |format, index| { + try std.testing.expectEqual(format, func.opcode(insts[index]).decode_low_float.format); + try std.testing.expectEqual(format, func.opcode(insts[index + 3]).encode_low_float.format); + } + + const bad = "fn {\n block0(v0: u8):\n let v1 = decode_low_float e4m3, v0\n ret v1\n}"; + try std.testing.expectError(error.InvalidSyntax, parse(std.testing.allocator, bad)); +} + +test "NVFP4 conversions print parse and verify every policy pair" { + const verify = @import("verify.zig"); + const text = + \\fn { + \\ block0(v0: u8, v1: u8, v2: f32): + \\ #![test.inst] + \\ #[test.result] + \\ let v3 = dequantize_nvfp4 multiply, multiply, v0, v1, v2 + \\ let v4 = dequantize_nvfp4 multiply, divide, v0, v1, v2 + \\ let v5 = quantize_nvfp4 divide, multiply, v3, v1, v2 + \\ let v6 = quantize_nvfp4 divide, divide, v4, v1, v2 + \\ ret v6 + \\} + ; + var func = try parse(std.testing.allocator, text); + defer func.deinit(); + try std.testing.expectFmt(text, "{f}", .{func}); + var diags = try verify.verify(std.testing.allocator, &func, .high); + defer diags.deinit(); + try std.testing.expect(diags.ok()); + + const insts = func.blockInsts(@enumFromInt(0)); + var inst_attrs = func.attributesOf(.{ .inst = insts[0] }); + try std.testing.expectEqualStrings("inst", inst_attrs.next().?.custom.key); + var result_attrs = func.attributesOf(.{ .value = func.instResult(insts[0]).? }); + try std.testing.expectEqualStrings("result", result_attrs.next().?.custom.key); + try std.testing.expectEqual(nvfp4.ScaleApplication.multiply, func.opcode(insts[0]).dequantize_nvfp4.block_application); + try std.testing.expectEqual(nvfp4.ScaleApplication.divide, func.opcode(insts[1]).dequantize_nvfp4.global_application); + try std.testing.expectEqual(nvfp4.ScaleApplication.divide, func.opcode(insts[2]).quantize_nvfp4.block_application); + try std.testing.expectEqual(nvfp4.ScaleApplication.divide, func.opcode(insts[3]).quantize_nvfp4.global_application); +} + +test "NVFP4 parser rejects malformed policies and operand counts" { + const prefix = "fn {\n block0(v0: u8, v1: u8, v2: f32):\n let v3 = "; + const suffix = "\n ret v0\n}"; + const bodies = [_][]const u8{ + "dequantize_nvfp4 multiply, v0, v1, v2", + "dequantize_nvfp4 multiply, divide, multiply, v0, v1, v2", + "dequantize_nvfp4 unknown, divide, v0, v1, v2", + "dequantize_nvfp4 multiply, unknown, v0, v1, v2", + "dequantize_nvfp4 multiply, divide, v0, v1", + "dequantize_nvfp4 multiply, divide, v0, v1, v2, v0", + "quantize_nvfp4 multiply, divide, v2, v1", + "quantize_nvfp4 multiply, divide, v2, v1, v2, v0", + }; + for (bodies) |body| { + const text = try std.mem.concat(std.testing.allocator, u8, &.{ prefix, body, suffix }); + defer std.testing.allocator.free(text); + try std.testing.expectError(error.InvalidSyntax, parse(std.testing.allocator, text)); + } +} diff --git a/libs/vulcan-ir/verify.zig b/libs/vulcan-ir/verify.zig index 3a0f76d..678ec45 100644 --- a/libs/vulcan-ir/verify.zig +++ b/libs/vulcan-ir/verify.zig @@ -124,6 +124,18 @@ fn checkOperandTypes(func: *const Function, diags: *Diagnostics) std.mem.Allocat .dot => |d| if (dotOperandsMismatch(func, inst, d)) { if (func.instResult(inst)) |result| try diags.add(.{ .operand_type_mismatch = result }); }, + .decode_low_float => |cv| if (lowFloatOperandsMismatch(func, inst, cv, true)) { + try diags.add(.{ .operand_type_mismatch = func.instResult(inst) orelse cv.value }); + }, + .encode_low_float => |cv| if (lowFloatOperandsMismatch(func, inst, cv, false)) { + try diags.add(.{ .operand_type_mismatch = func.instResult(inst) orelse cv.value }); + }, + .dequantize_nvfp4 => |cv| if (nvFp4OperandsMismatch(func, inst, cv, true)) { + try diags.add(.{ .operand_type_mismatch = func.instResult(inst) orelse cv.value }); + }, + .quantize_nvfp4 => |cv| if (nvFp4OperandsMismatch(func, inst, cv, false)) { + try diags.add(.{ .operand_type_mismatch = func.instResult(inst) orelse cv.value }); + }, // matmul has no result value, so a mismatch is reported against // `c` (the pointer the tile is written to). .matmul => |mm| if (matmulOperandsMismatch(func, mm)) { @@ -202,6 +214,48 @@ fn isPtrValue(func: *const Function, v: Value) bool { return func.types.type_kind(func.valueType(v)) == .ptr; } +fn lowFloatOperandsMismatch( + func: *const Function, + inst: function.Inst, + conversion: function.LowFloatConvert, + decode_direction: bool, +) bool { + const result = func.instResult(inst) orelse return true; + const payload_bits = conversion.format.payloadBits(); + if (decode_direction) { + return !isUnsignedIntOfBits(func, func.valueType(conversion.value), payload_bits) or + !isF32(func, func.valueType(result)); + } + return !isF32(func, func.valueType(conversion.value)) or + !isUnsignedIntOfBits(func, func.valueType(result), payload_bits); +} + +fn nvFp4OperandsMismatch(func: *const Function, inst: function.Inst, conversion: function.NvFp4Convert, dequantize_direction: bool) bool { + const result = func.instResult(inst) orelse return true; + if (!isUnsignedIntOfBits(func, func.valueType(conversion.block_scale), 8) or + !isF32(func, func.valueType(conversion.global_scale))) return true; + if (dequantize_direction) { + return !isUnsignedIntOfBits(func, func.valueType(conversion.value), 8) or + !isF32(func, func.valueType(result)); + } + return !isF32(func, func.valueType(conversion.value)) or + !isUnsignedIntOfBits(func, func.valueType(result), 8); +} + +fn isUnsignedIntOfBits(func: *const Function, ty: Type, bits: u16) bool { + return switch (func.types.type_kind(ty)) { + .int => |int| int.signedness == .unsigned and int.bits == bits, + else => false, + }; +} + +fn isF32(func: *const Function, ty: Type) bool { + return switch (func.types.type_kind(ty)) { + .float => |float| float == .f32, + else => false, + }; +} + /// Whether an `atomic_rmw`'s operands break one of the four rules `AtomicRmw` states. /// /// The width limits are NOT checked here. A backend accepts the widths its atomic @@ -460,6 +514,12 @@ fn checkDominance(func: *const Function, diags: *Diagnostics) std.mem.Allocator. }, .extract => |ex| try checkUse(&dominance, def_block, diags, ex.aggregate, bi), .convert => |cv| try checkUse(&dominance, def_block, diags, cv.value, bi), + .decode_low_float, .encode_low_float => |cv| try checkUse(&dominance, def_block, diags, cv.value, bi), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + try checkUse(&dominance, def_block, diags, cv.value, bi); + try checkUse(&dominance, def_block, diags, cv.block_scale, bi); + try checkUse(&dominance, def_block, diags, cv.global_scale, bi); + }, .unary => |u| try checkUse(&dominance, def_block, diags, u.value, bi), .load => |ld| try checkUse(&dominance, def_block, diags, ld.ptr, bi), .store => |st| { @@ -1238,3 +1298,241 @@ test "an endian tag on an atomic is reported as misplaced" { try std.testing.expectEqual(@as(usize, 1), d.count()); // the store's tag is fine try std.testing.expectEqual(Diagnostic{ .misplaced_endian = .{ .inst = atomic_inst } }, d.items()[0]); } + +test "low float verifier rejects every wrong scalar and composite type class" { + const allocator = std.testing.allocator; + const Wrong = enum { bool, ptr, f16, f64, f128, signed8, signed16, u8, u16, u32, vector, array, aggregate }; + const Side = enum { source, result }; + const cases = [_]struct { + format: function.LowFloatFormat, + decode_direction: bool, + side: Side, + wrong: Wrong, + }{ + .{ .format = .bf16, .decode_direction = true, .side = .source, .wrong = .signed16 }, + .{ .format = .bf16, .decode_direction = true, .side = .source, .wrong = .u8 }, + .{ .format = .bf16, .decode_direction = true, .side = .source, .wrong = .u32 }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .signed8 }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .u16 }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .bool }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .ptr }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .f16 }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .f64 }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .f128 }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .vector }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .array }, + .{ .format = .f8_e4m3, .decode_direction = true, .side = .source, .wrong = .aggregate }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .bool }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .ptr }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .f16 }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .f64 }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .f128 }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .u8 }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .vector }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .array }, + .{ .format = .f8_e5m2, .decode_direction = true, .side = .result, .wrong = .aggregate }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .bool }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .ptr }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .f16 }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .f64 }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .f128 }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .u8 }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .vector }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .array }, + .{ .format = .f8_e4m3, .decode_direction = false, .side = .source, .wrong = .aggregate }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .bool }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .ptr }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .f16 }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .f64 }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .f128 }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .signed8 }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .u16 }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .vector }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .array }, + .{ .format = .f8_e5m2, .decode_direction = false, .side = .result, .wrong = .aggregate }, + .{ .format = .bf16, .decode_direction = false, .side = .result, .wrong = .signed16 }, + .{ .format = .bf16, .decode_direction = false, .side = .result, .wrong = .u8 }, + .{ .format = .bf16, .decode_direction = false, .side = .result, .wrong = .u32 }, + }; + for (cases) |case| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const wrong_type = try switch (case.wrong) { + .bool => func.types.intern(.bool), + .ptr => func.types.intern(.{ .ptr = .global }), + .f16 => func.types.intern(.{ .float = .f16 }), + .f64 => func.types.intern(.{ .float = .f64 }), + .f128 => func.types.intern(.{ .float = .f128 }), + .signed8 => func.types.intern(.{ .int = .{ .signedness = .signed, .bits = 8 } }), + .signed16 => func.types.intern(.{ .int = .{ .signedness = .signed, .bits = 16 } }), + .u8 => func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }), + .u16 => func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }), + .u32 => func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }), + .vector => func.types.intern(.{ .vector = .{ .len = 4, .elem = u8_t } }), + .array => func.types.intern(.{ .array = .{ .len = 4, .elem = u8_t } }), + .aggregate => func.types.intern(.{ .@"struct" = &.{u8_t} }), + }; + const payload_type = try func.types.intern(.{ .int = .{ + .signedness = .unsigned, + .bits = case.format.payloadBits(), + } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const source_type = if (case.side == .source) wrong_type else if (case.decode_direction) payload_type else f32_t; + const result_type = if (case.side == .result) wrong_type else if (case.decode_direction) f32_t else payload_type; + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, source_type); + const conversion: function.LowFloatConvert = .{ .value = source, .format = case.format }; + const result = if (case.decode_direction) + try func.appendInst(block, result_type, .{ .decode_low_float = conversion }) + else + try func.appendInst(block, result_type, .{ .encode_low_float = conversion }); + func.setTerminator(block, .{ .ret = function.Ret.one(result) }); + var diags = try verify(allocator, &func, .high); + defer diags.deinit(); + try std.testing.expect(!diags.ok()); + try std.testing.expectEqual(Diagnostic{ .operand_type_mismatch = result }, diags.items()[0]); + } +} + +test "low float verifier checks dominance in both directions" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const entry = try func.appendBlock(); + const left = try func.appendBlock(); + const merge = try func.appendBlock(); + const source = try func.appendBlockParam(left, if (decode_direction) u8_t else f32_t); + try func.setJump(entry, merge, &.{}); + try func.setJump(left, merge, &.{}); + const result = try func.appendInst( + merge, + if (decode_direction) f32_t else u8_t, + if (decode_direction) + .{ .decode_low_float = .{ .value = source, .format = .f8_e5m2 } } + else + .{ .encode_low_float = .{ .value = source, .format = .f8_e5m2 } }, + ); + func.setTerminator(merge, .{ .ret = function.Ret.one(result) }); + var diags = try verify(allocator, &func, .high); + defer diags.deinit(); + try std.testing.expect(!diags.ok()); + try std.testing.expectEqual(Diagnostic{ .not_dominated = .{ .value = source, .block = merge } }, diags.items()[0]); + } +} + +test "NVFP4 verifier checks every operand and result independently" { + const allocator = std.testing.allocator; + const Field = enum { value, block_scale, global_scale, result, missing_result }; + inline for (.{ true, false }) |dequantize_direction| { + inline for (.{ Field.value, Field.block_scale, Field.global_scale, Field.result, Field.missing_result }) |field| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const i8_t = try func.types.intern(.{ .int = .{ .signedness = .signed, .bits = 8 } }); + const f64_t = try func.types.intern(.{ .float = .f64 }); + const block = try func.appendBlock(); + const value = try func.appendBlockParam(block, if (field == .value) + (if (dequantize_direction) f32_t else u8_t) + else if (dequantize_direction) u8_t else f32_t); + const block_scale = try func.appendBlockParam(block, if (field == .block_scale) i8_t else u8_t); + const global_scale = try func.appendBlockParam(block, if (field == .global_scale) u8_t else f32_t); + const conversion: function.NvFp4Convert = .{ + .value = value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + }; + const op: function.Opcode = if (dequantize_direction) + .{ .dequantize_nvfp4 = conversion } + else + .{ .quantize_nvfp4 = conversion }; + const result: ?Value = if (field == .missing_result) blk: { + _ = try func.appendStmtRaw(block, op); + break :blk null; + } else try func.appendInst(block, if (field == .result) + (if (dequantize_direction) u8_t else f64_t) + else if (dequantize_direction) f32_t else u8_t, op); + func.setTerminator(block, .{ .ret = if (result) |r| function.Ret.one(r) else function.Ret.none() }); + var diags = try verify(allocator, &func, .high); + defer diags.deinit(); + try std.testing.expect(!diags.ok()); + } + } + + inline for (.{ true, false }) |dequantize_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const value = try func.appendBlockParam(block, if (dequantize_direction) u8_t else f32_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: function.NvFp4Convert = .{ + .value = value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + }; + const result = try func.appendInst(block, if (dequantize_direction) f32_t else u8_t, if (dequantize_direction) + .{ .dequantize_nvfp4 = conversion } + else + .{ .quantize_nvfp4 = conversion }); + func.setTerminator(block, .{ .ret = function.Ret.one(result) }); + var diags = try verify(allocator, &func, .high); + defer diags.deinit(); + try std.testing.expect(diags.ok()); + } +} + +test "NVFP4 verifier checks dominance of all three operands" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |dequantize_direction| { + inline for (0..3) |bad_operand| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const entry = try func.appendBlock(); + const sibling = try func.appendBlock(); + const merge = try func.appendBlock(); + const good_value = try func.appendBlockParam(entry, if (dequantize_direction) u8_t else f32_t); + const good_block = try func.appendBlockParam(entry, u8_t); + const good_global = try func.appendBlockParam(entry, f32_t); + const bad_value = try func.appendBlockParam(sibling, if (dequantize_direction) u8_t else f32_t); + const bad_block = try func.appendBlockParam(sibling, u8_t); + const bad_global = try func.appendBlockParam(sibling, f32_t); + try func.setJump(entry, merge, &.{}); + try func.setJump(sibling, merge, &.{}); + const conversion: function.NvFp4Convert = .{ + .value = if (bad_operand == 0) bad_value else good_value, + .block_scale = if (bad_operand == 1) bad_block else good_block, + .global_scale = if (bad_operand == 2) bad_global else good_global, + .block_application = .multiply, + .global_application = .divide, + }; + const result = try func.appendInst(merge, if (dequantize_direction) f32_t else u8_t, if (dequantize_direction) + .{ .dequantize_nvfp4 = conversion } + else + .{ .quantize_nvfp4 = conversion }); + func.setTerminator(merge, .{ .ret = function.Ret.one(result) }); + var diags = try verify(allocator, &func, .high); + defer diags.deinit(); + try std.testing.expect(!diags.ok()); + const bad = switch (bad_operand) { + 0 => bad_value, + 1 => bad_block, + 2 => bad_global, + else => unreachable, + }; + try std.testing.expectEqual(Diagnostic{ .not_dominated = .{ .value = bad, .block = merge } }, diags.items()[0]); + } + } +} diff --git a/libs/vulcan-opt/dce.zig b/libs/vulcan-opt/dce.zig index f41128a..ea22a08 100644 --- a/libs/vulcan-opt/dce.zig +++ b/libs/vulcan-opt/dce.zig @@ -13,7 +13,7 @@ pub const pass_def = pass.Pass{ .name = "dce", .run = run }; /// Whether an instruction has no side effects, so it may be dropped when unused. fn isPure(op: ir.function.Opcode) bool { return switch (op) { - .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .struct_new, .extract, .convert, .unary, .alloca, .global_addr, .dot => true, + .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .struct_new, .extract, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .alloca, .global_addr, .dot => true, // A prefetch hint has no result but must be kept, like a store. A // matmul writes the `c` memory, likewise kept. .load, .store, .prefetch, .matmul, .@"if", .call, .call_indirect => false, @@ -59,6 +59,12 @@ pub fn countUses(func: *const Function, uses: []u32) void { }, .extract => |e| uses[@intFromEnum(e.aggregate)] += 1, .convert => |cv| uses[@intFromEnum(cv.value)] += 1, + .decode_low_float, .encode_low_float => |cv| uses[@intFromEnum(cv.value)] += 1, + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + uses[@intFromEnum(cv.value)] += 1; + uses[@intFromEnum(cv.block_scale)] += 1; + uses[@intFromEnum(cv.global_scale)] += 1; + }, .unary => |u| uses[@intFromEnum(u.value)] += 1, .load => |l| uses[@intFromEnum(l.ptr)] += 1, .store => |st| { @@ -254,3 +260,53 @@ test "keeps both forms of an atomic, read result or not" { try std.testing.expect(func.opcode(insts[1]) == .atomic_rmw); try std.testing.expect(func.opcode(insts[2]) == .atomic_rmw); } + +test "low float conversion liveness controls dead code elimination" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const live = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .value = payload, .format = .f8_e4m3 } }); + _ = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .value = payload, .format = .f8_e5m2 } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(live) }); + var analyses = pass.Analyses{ .allocator = allocator, .func = &func }; + defer analyses.deinit(); + try std.testing.expect(try run(allocator, &func, &analyses)); + try std.testing.expectEqual(@as(usize, 1), func.blockInsts(block).len); + try std.testing.expectEqual(ir.function.LowFloatFormat.f8_e4m3, func.opcode(func.blockInsts(block)[0]).decode_low_float.format); +} + +test "nvfp4 conversion liveness counts every operand and removes an unused conversion" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendInst(block, u8_t, .{ .iconst = 1 }); + const block_scale = try func.appendInst(block, u8_t, .{ .iconst = 2 }); + const global_scale = try func.appendInst(block, f32_t, .{ .fconst = 3.0 }); + const conversion: ir.function.NvFp4Convert = .{ + .value = payload, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + }; + const live = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = conversion }); + _ = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(live) }); + var analyses = pass.Analyses{ .allocator = allocator, .func = &func }; + defer analyses.deinit(); + try std.testing.expect(try run(allocator, &func, &analyses)); + // Each defining constant is live only through a different NVFP4 operand. Omitting any + // operand from the use count lets DCE erase its definition and makes this assertion fail. + try std.testing.expectEqual(@as(usize, 4), func.blockInsts(block).len); + const kept = func.opcode(func.blockInsts(block)[3]).dequantize_nvfp4; + try std.testing.expectEqual(payload, kept.value); + try std.testing.expectEqual(block_scale, kept.block_scale); + try std.testing.expectEqual(global_scale, kept.global_scale); +} diff --git a/libs/vulcan-opt/gvn.zig b/libs/vulcan-opt/gvn.zig index 95ff574..53d6984 100644 --- a/libs/vulcan-opt/gvn.zig +++ b/libs/vulcan-opt/gvn.zig @@ -125,6 +125,7 @@ fn keyOf(func: *const Function, canon: []const Value, inst: Inst, result: Value) }, .select => |x| .{ .kind = .select, .a = vn(canon, x.cond), .b = vn(canon, x.then), .c = vn(canon, x.@"else") }, .convert => |x| .{ .kind = .convert, .sub = @intFromEnum(func.valueType(result)), .a = vn(canon, x.value) }, + .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4 => null, .unary => |x| .{ .kind = .unary, .sub = @intFromEnum(func.valueType(result)), .a = vn(canon, x.value), .b = @intFromEnum(x.op) }, .extract => |x| .{ .kind = .extract, .sub = x.index, .a = vn(canon, x.aggregate) }, // `via_got` must be part of the key, not just `symbol`: a direct global_addr and a @@ -187,6 +188,12 @@ fn rewriteOperands(func: *Function, canon: []const Value) void { }, .extract => |*e| e.aggregate = sub(canon, e.aggregate), .convert => |*cv| cv.value = sub(canon, cv.value), + .decode_low_float, .encode_low_float => |*cv| cv.value = sub(canon, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |*cv| { + cv.value = sub(canon, cv.value); + cv.block_scale = sub(canon, cv.block_scale); + cv.global_scale = sub(canon, cv.global_scale); + }, .unary => |*u| u.value = sub(canon, u.value), .load => |*l| l.ptr = sub(canon, l.ptr), .store => |*st| { @@ -290,6 +297,78 @@ test "cse does not reuse across a non-dominating block" { try std.testing.expectEqual(e2, func.terminator(b2).?.ret.values[0]); } +test "gvn deliberately leaves low float conversions distinct" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + + const f32_t = try func.types.intern(.{ .float = .f32 }); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const block = try func.appendBlock(); + const input = try func.appendBlockParam(block, f32_t); + const encoded_a = try func.appendInst(block, u8_t, .{ .encode_low_float = .{ .value = input, .format = .f8_e4m3 } }); + const encoded_b = try func.appendInst(block, u8_t, .{ .encode_low_float = .{ .value = input, .format = .f8_e4m3 } }); + const encoded_other = try func.appendInst(block, u8_t, .{ .encode_low_float = .{ .value = input, .format = .f8_e5m2 } }); + const decoded_a = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .value = encoded_a, .format = .f8_e4m3 } }); + const decoded_b = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .value = encoded_a, .format = .f8_e4m3 } }); + const decoded_other = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .value = encoded_other, .format = .f8_e5m2 } }); + const same_sum = try func.appendInst(block, f32_t, .{ .arith = .{ .op = .add, .lhs = decoded_a, .rhs = decoded_b } }); + const total = try func.appendInst(block, f32_t, .{ .arith = .{ .op = .add, .lhs = same_sum, .rhs = decoded_other } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(total) }); + + var analyses = pass.Analyses{ .allocator = allocator, .func = &func }; + defer analyses.deinit(); + try std.testing.expect(!try run(allocator, &func, &analyses)); + + const sum = func.opcode(func.definingInst(same_sum).?).arith; + try std.testing.expectEqual(decoded_a, sum.lhs); + try std.testing.expectEqual(decoded_b, sum.rhs); + try std.testing.expect(sum.lhs != sum.rhs); + try std.testing.expect(encoded_a != encoded_b); + try std.testing.expectEqual(ir.function.LowFloatFormat.f8_e4m3, func.opcode(func.definingInst(encoded_b).?).encode_low_float.format); + try std.testing.expectEqual(ir.function.LowFloatFormat.f8_e5m2, func.opcode(func.definingInst(encoded_other).?).encode_low_float.format); + try std.testing.expectEqual(ir.function.LowFloatFormat.f8_e4m3, func.opcode(func.definingInst(decoded_b).?).decode_low_float.format); + try std.testing.expectEqual(ir.function.LowFloatFormat.f8_e5m2, func.opcode(func.definingInst(decoded_other).?).decode_low_float.format); +} + +test "gvn deliberately leaves nvfp4 conversions distinct" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: ir.function.NvFp4Convert = .{ + .value = payload, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + }; + const decoded_a = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = conversion }); + const decoded_b = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = conversion }); + const encode_conversion: ir.function.NvFp4Convert = .{ + .value = decoded_a, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .divide, + .global_application = .multiply, + }; + const encoded_a = try func.appendInst(block, u8_t, .{ .quantize_nvfp4 = encode_conversion }); + const encoded_b = try func.appendInst(block, u8_t, .{ .quantize_nvfp4 = encode_conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(encoded_b) }); + + var analyses = pass.Analyses{ .allocator = allocator, .func = &func }; + defer analyses.deinit(); + try std.testing.expect(!try run(allocator, &func, &analyses)); + try std.testing.expect(decoded_a != decoded_b); + try std.testing.expect(encoded_a != encoded_b); + try std.testing.expectEqual(encoded_b, func.terminator(block).?.ret.values[0]); +} + test "gvn does not merge a direct and a via_got global_addr of the same symbol" { const allocator = std.testing.allocator; var func = Function.init(allocator); diff --git a/libs/vulcan-opt/inline.zig b/libs/vulcan-opt/inline.zig index e4a1618..a25d621 100644 --- a/libs/vulcan-opt/inline.zig +++ b/libs/vulcan-opt/inline.zig @@ -215,6 +215,10 @@ fn mapOpcode(caller: *Function, callee: *const Function, vmap: std.AutoHashMapUn .icmp => |c| .{ .icmp = .{ .op = c.op, .lhs = m(vmap, c.lhs), .rhs = m(vmap, c.rhs) } }, .select => |s| .{ .select = .{ .cond = m(vmap, s.cond), .then = m(vmap, s.then), .@"else" = m(vmap, s.@"else") } }, .convert => |cv| .{ .convert = .{ .value = m(vmap, cv.value) } }, + .decode_low_float => |cv| .{ .decode_low_float = .{ .value = m(vmap, cv.value), .format = cv.format } }, + .encode_low_float => |cv| .{ .encode_low_float = .{ .value = m(vmap, cv.value), .format = cv.format } }, + .dequantize_nvfp4 => |cv| .{ .dequantize_nvfp4 = .{ .value = m(vmap, cv.value), .block_scale = m(vmap, cv.block_scale), .global_scale = m(vmap, cv.global_scale), .block_application = cv.block_application, .global_application = cv.global_application } }, + .quantize_nvfp4 => |cv| .{ .quantize_nvfp4 = .{ .value = m(vmap, cv.value), .block_scale = m(vmap, cv.block_scale), .global_scale = m(vmap, cv.global_scale), .block_application = cv.block_application, .global_application = cv.global_application } }, .unary => |u| .{ .unary = .{ .op = u.op, .value = m(vmap, u.value) } }, // `volatile` is an observable side effect, not a hint. A callee that reads an MMIO // register keeps that read observable after it is inlined, so carry the flag. @@ -280,6 +284,12 @@ fn substituteValue(func: *Function, from: Value, to: Value) void { }, .extract => |*e| e.aggregate = r(from, to, e.aggregate), .convert => |*cv| cv.value = r(from, to, cv.value), + .decode_low_float, .encode_low_float => |*cv| cv.value = r(from, to, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |*cv| { + cv.value = r(from, to, cv.value); + cv.block_scale = r(from, to, cv.block_scale); + cv.global_scale = r(from, to, cv.global_scale); + }, .unary => |*u| u.value = r(from, to, u.value), .load => |*l| l.ptr = r(from, to, l.ptr), .store => |*st| { @@ -660,6 +670,116 @@ test "inlines a leaf helper and replaces the call result" { for (caller.blockInsts(b)) |inst| try std.testing.expect(caller.opcode(inst) != .call); } +test "inlining preserves low float directions formats and remapped operands" { + const allocator = std.testing.allocator; + const f32_kind = ir.types.TypeKind{ .float = .f32 }; + const u8_kind = ir.types.TypeKind{ .int = .{ .signedness = .unsigned, .bits = 8 } }; + + var callee = Function.init(allocator); + defer callee.deinit(); + { + const f32_t = try callee.types.intern(f32_kind); + const u8_t = try callee.types.intern(u8_kind); + const block = try callee.appendBlock(); + const input = try callee.appendBlockParam(block, f32_t); + const payload = try callee.appendInst(block, u8_t, .{ .encode_low_float = .{ + .value = input, + .format = .f8_e4m3, + } }); + const output = try callee.appendInst(block, f32_t, .{ .decode_low_float = .{ + .value = payload, + .format = .f8_e5m2, + } }); + callee.setTerminator(block, .{ .ret = ir.function.Ret.one(output) }); + } + + var caller = Function.init(allocator); + defer caller.deinit(); + const f32_t = try caller.types.intern(f32_kind); + const block = try caller.appendBlock(); + const input = try caller.appendBlockParam(block, f32_t); + const call = try caller.appendCall(block, f32_t, "convert", &.{input}); + caller.setTerminator(block, .{ .ret = ir.function.Ret.one(call) }); + + var lookup_context = TestLookup{ .callee = &callee, .name = "convert" }; + try std.testing.expect(try run(allocator, &caller, .{ + .context = &lookup_context, + .func = TestLookup.get, + })); + + var encoded: ?ir.function.LowFloatConvert = null; + var decoded: ?ir.function.LowFloatConvert = null; + for (0..caller.blockCount()) |block_index| { + for (caller.blockInsts(@enumFromInt(block_index))) |inst| switch (caller.opcode(inst)) { + .encode_low_float => |conversion| encoded = conversion, + .decode_low_float => |conversion| decoded = conversion, + .call => return error.TestUnexpectedResult, + else => {}, + }; + } + try std.testing.expectEqual(ir.function.LowFloatFormat.f8_e4m3, encoded.?.format); + try std.testing.expectEqual(ir.function.LowFloatFormat.f8_e5m2, decoded.?.format); + try std.testing.expectEqual(input, encoded.?.value); + const encoded_inst = caller.definingInst(decoded.?.value).?; + try std.testing.expect(caller.opcode(encoded_inst) == .encode_low_float); +} + +test "inlining remaps all nvfp4 operands and preserves policies" { + const allocator = std.testing.allocator; + var callee = Function.init(allocator); + defer callee.deinit(); + const cu8 = try callee.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const cf32 = try callee.types.intern(.{ .float = .f32 }); + const cb = try callee.appendBlock(); + const payload = try callee.appendBlockParam(cb, cu8); + const block_scale = try callee.appendBlockParam(cb, cu8); + const global_scale = try callee.appendBlockParam(cb, cf32); + const decoded = try callee.appendInst(cb, cf32, .{ .dequantize_nvfp4 = .{ + .value = payload, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + } }); + const encoded = try callee.appendInst(cb, cu8, .{ .quantize_nvfp4 = .{ + .value = decoded, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .divide, + .global_application = .multiply, + } }); + callee.setTerminator(cb, .{ .ret = ir.function.Ret.one(encoded) }); + + var caller = Function.init(allocator); + defer caller.deinit(); + const u8_t = try caller.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try caller.types.intern(.{ .float = .f32 }); + const block = try caller.appendBlock(); + const p = try caller.appendBlockParam(block, u8_t); + const bs = try caller.appendBlockParam(block, u8_t); + const gs = try caller.appendBlockParam(block, f32_t); + const call = try caller.appendCall(block, u8_t, "nvfp4", &.{ p, bs, gs }); + caller.setTerminator(block, .{ .ret = ir.function.Ret.one(call) }); + var lookup_context = TestLookup{ .callee = &callee, .name = "nvfp4" }; + try std.testing.expect(try run(allocator, &caller, .{ .context = &lookup_context, .func = TestLookup.get })); + + var saw_decode = false; + var saw_encode = false; + var remapped_decode: ?Value = null; + for (0..caller.blockCount()) |bi| for (caller.blockInsts(@enumFromInt(bi))) |inst| switch (caller.opcode(inst)) { + .dequantize_nvfp4 => |cv| { + saw_decode = cv.value == p and cv.block_scale == bs and cv.global_scale == gs and cv.block_application == .multiply and cv.global_application == .divide; + remapped_decode = caller.instResult(inst).?; + }, + .quantize_nvfp4 => |cv| { + saw_encode = remapped_decode != null and cv.value == remapped_decode.? and cv.block_scale == bs and cv.global_scale == gs and cv.block_application == .divide and cv.global_application == .multiply; + }, + .call => return error.TestUnexpectedResult, + else => {}, + }; + try std.testing.expect(saw_decode and saw_encode); +} + test "inlines a multi-block, two-return callee (the call is replaced by cloned control flow)" { const allocator = std.testing.allocator; const i32k = ir.types.TypeKind{ .int = .{ .signedness = .signed, .bits = 32 } }; diff --git a/libs/vulcan-opt/ivsr.zig b/libs/vulcan-opt/ivsr.zig index 792d0d5..5d5c71f 100644 --- a/libs/vulcan-opt/ivsr.zig +++ b/libs/vulcan-opt/ivsr.zig @@ -585,6 +585,10 @@ fn stepOf(ctx: *const LoopCtx, hp: Value, next: Value) ?Step { .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .call, @@ -624,6 +628,10 @@ fn affineOf(allocator: std.mem.Allocator, ctx: *LoopCtx, pool: *Pool, inst: Inst .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .call, @@ -804,6 +812,10 @@ fn countUses(allocator: std.mem.Allocator, ctx: *LoopCtx) pass.Error!void { .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .call, @@ -846,6 +858,12 @@ fn appendOperands(allocator: std.mem.Allocator, func: *const Function, inst: Ins .struct_new => |s| try out.appendSlice(allocator, func.valueList(s.fields)), .extract => |e| try out.append(allocator, e.aggregate), .convert => |c| try out.append(allocator, c.value), + .decode_low_float, .encode_low_float => |c| try out.append(allocator, c.value), + .dequantize_nvfp4, .quantize_nvfp4 => |c| { + try out.append(allocator, c.value); + try out.append(allocator, c.block_scale); + try out.append(allocator, c.global_scale); + }, .unary => |u| try out.append(allocator, u.value), .call => |c| { try out.appendSlice(allocator, func.valueList(c.args)); diff --git a/libs/vulcan-opt/jumpthread.zig b/libs/vulcan-opt/jumpthread.zig index c698175..3164072 100644 --- a/libs/vulcan-opt/jumpthread.zig +++ b/libs/vulcan-opt/jumpthread.zig @@ -430,6 +430,8 @@ fn blockUsesValueFromBlock(func: *const Function, block: Block, def_bi: u32, def }, .extract => |ex| if (usesB(def_block, ex.aggregate, def_bi)) return true, .convert => |cv| if (usesB(def_block, cv.value, def_bi)) return true, + .decode_low_float, .encode_low_float => |cv| if (usesB(def_block, cv.value, def_bi)) return true, + .dequantize_nvfp4, .quantize_nvfp4 => |cv| if (usesB(def_block, cv.value, def_bi) or usesB(def_block, cv.block_scale, def_bi) or usesB(def_block, cv.global_scale, def_bi)) return true, .unary => |u| if (usesB(def_block, u.value, def_bi)) return true, .load => |ld| if (usesB(def_block, ld.ptr, def_bi)) return true, .store => |st| if (usesB(def_block, st.value, def_bi) or usesB(def_block, st.ptr, def_bi)) return true, @@ -497,6 +499,10 @@ fn hasSideEffect(func: *const Function, block: Block) bool { .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .global_addr, diff --git a/libs/vulcan-opt/licm.zig b/libs/vulcan-opt/licm.zig index f1b92c2..0df0149 100644 --- a/libs/vulcan-opt/licm.zig +++ b/libs/vulcan-opt/licm.zig @@ -29,7 +29,7 @@ fn nonTrapping(op: BinOp) bool { /// Whether an instruction may be hoisted out of a loop if it is invariant. fn hoistable(opcode: ir.function.Opcode) bool { return switch (opcode) { - .iconst, .fconst, .fconst128, .icmp, .select, .convert, .unary, .extract, .global_addr, .dot => true, + .iconst, .fconst, .fconst128, .icmp, .select, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .extract, .global_addr, .dot => true, .arith => |a| nonTrapping(a.op), .arith_imm => |a| nonTrapping(a.op), .alloca, .struct_new, .load, .store, .prefetch, .matmul, .call, .call_indirect, .@"if" => false, @@ -61,6 +61,8 @@ fn operandsInvariant(func: *const Function, inst: Inst, invariant: []const bool) .icmp => |c| inv(invariant, c.lhs) and inv(invariant, c.rhs), .select => |s| inv(invariant, s.cond) and inv(invariant, s.then) and inv(invariant, s.@"else"), .convert => |cv| inv(invariant, cv.value), + .decode_low_float, .encode_low_float => |cv| inv(invariant, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| inv(invariant, cv.value) and inv(invariant, cv.block_scale) and inv(invariant, cv.global_scale), .unary => |u| inv(invariant, u.value), .extract => |e| inv(invariant, e.aggregate), .dot => |d| inv(invariant, d.acc) and inv(invariant, d.a) and inv(invariant, d.b), @@ -172,6 +174,37 @@ fn isHoisted(list: []const Hoist, inst: Inst) bool { return false; } +test "nvfp4 is hoistable only when all three operands are invariant" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const result = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = payload, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + } }); + const inst = func.definingInst(result).?; + try std.testing.expect(hoistable(func.opcode(inst))); + + const invariant = try allocator.alloc(bool, func.valueCount()); + defer allocator.free(invariant); + @memset(invariant, true); + try std.testing.expect(operandsInvariant(&func, inst, invariant)); + inline for (.{ payload, block_scale, global_scale }) |operand| { + invariant[@intFromEnum(operand)] = false; + try std.testing.expect(!operandsInvariant(&func, inst, invariant)); + invariant[@intFromEnum(operand)] = true; + } +} + test "hoists a loop-invariant product to the preheader" { const allocator = std.testing.allocator; var func = Function.init(allocator); diff --git a/libs/vulcan-opt/loadfwd.zig b/libs/vulcan-opt/loadfwd.zig index 5d04ea1..cc21390 100644 --- a/libs/vulcan-opt/loadfwd.zig +++ b/libs/vulcan-opt/loadfwd.zig @@ -102,6 +102,10 @@ pub fn run(allocator: std.mem.Allocator, func: *Function, analyses: *pass.Analys .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .global_addr, diff --git a/libs/vulcan-opt/lowerdiv.zig b/libs/vulcan-opt/lowerdiv.zig index 6bc1cb0..4863909 100644 --- a/libs/vulcan-opt/lowerdiv.zig +++ b/libs/vulcan-opt/lowerdiv.zig @@ -189,6 +189,12 @@ fn rewriteUses(func: *Function, subst: std.AutoHashMapUnmanaged(Value, Value)) v }, .extract => |*e| e.aggregate = sub(subst, e.aggregate), .convert => |*cv| cv.value = sub(subst, cv.value), + .decode_low_float, .encode_low_float => |*cv| cv.value = sub(subst, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |*cv| { + cv.value = sub(subst, cv.value); + cv.block_scale = sub(subst, cv.block_scale); + cv.global_scale = sub(subst, cv.global_scale); + }, .unary => |*u| u.value = sub(subst, u.value), .load => |*l| l.ptr = sub(subst, l.ptr), .store => |*st| { diff --git a/libs/vulcan-opt/mem2reg.zig b/libs/vulcan-opt/mem2reg.zig index 6bd7063..3ab6720 100644 --- a/libs/vulcan-opt/mem2reg.zig +++ b/libs/vulcan-opt/mem2reg.zig @@ -544,6 +544,12 @@ fn markEscapes(func: *const Function, promotable: []bool) void { }, .extract => |e| esc(promotable, e.aggregate), .convert => |cv| esc(promotable, cv.value), + .decode_low_float, .encode_low_float => |cv| esc(promotable, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + esc(promotable, cv.value); + esc(promotable, cv.block_scale); + esc(promotable, cv.global_scale); + }, .unary => |u| esc(promotable, u.value), .prefetch => |pf| esc(promotable, pf.ptr), // SM12 T3: unlike `load`/`store`'s `ptr` (a sanctioned dereference), `list` is the diff --git a/libs/vulcan-opt/microarch/loopvec.zig b/libs/vulcan-opt/microarch/loopvec.zig index 333acd7..62f7491 100644 --- a/libs/vulcan-opt/microarch/loopvec.zig +++ b/libs/vulcan-opt/microarch/loopvec.zig @@ -439,6 +439,8 @@ fn usesValue(func: *const Function, inst: Inst, v: Value) bool { .icmp => |c| c.lhs == v or c.rhs == v, .select => |s| s.cond == v or s.then == v or s.@"else" == v, .convert => |c| c.value == v, + .decode_low_float, .encode_low_float => |c| c.value == v, + .dequantize_nvfp4, .quantize_nvfp4 => |c| c.value == v or c.block_scale == v or c.global_scale == v, .unary => |u| u.value == v, .load => |l| l.ptr == v, .store => |s| s.value == v or s.ptr == v, @@ -545,6 +547,10 @@ fn recognizeReduction(func: *const Function, model: *const mm.Model, loop: *cons // times fewer read-modify-writes, and `applyReduction` does not clone this body at // all, so the write would happen only in the scalar remainder. .atomic_rmw, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, // A barrier joins them: vectorizing by V divides the trip count by V, so the loop // would meet V times fewer than the source says. // diff --git a/libs/vulcan-opt/microarch/prefetch.zig b/libs/vulcan-opt/microarch/prefetch.zig index e5ee830..254ce72 100644 --- a/libs/vulcan-opt/microarch/prefetch.zig +++ b/libs/vulcan-opt/microarch/prefetch.zig @@ -113,7 +113,7 @@ fn loopShape(func: *const Function, all_loops: []const loops.Loop, loop: *const if (idx != h_insts.len - 1) return null; // the `if` must end the block if_inst = inst; }, - .iconst, .fconst, .arith, .arith_imm, .icmp, .select, .convert, .unary, .extract, .struct_new => {}, + .iconst, .fconst, .arith, .arith_imm, .icmp, .select, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .extract, .struct_new => {}, // load/store/prefetch/call/call_indirect/alloca/global_addr are impure or memory ops. else => return null, } diff --git a/libs/vulcan-opt/microarch/registry.zig b/libs/vulcan-opt/microarch/registry.zig index 948a340..309ab17 100644 --- a/libs/vulcan-opt/microarch/registry.zig +++ b/libs/vulcan-opt/microarch/registry.zig @@ -31,7 +31,7 @@ fn altraLatency(op: ir.function.Opcode) u32 { .arith => |a| altraArith(a.op), .arith_imm => |a| altraArith(a.op), .load => 4, - .convert, .unary => 3, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 3, // A dot is a multiply-class op (4-way multiply-accumulate), grouped with `mul`. .dot => 4, // A matmul (et-soc tensor tile) is a big multicycle op, priced well above a @@ -84,7 +84,7 @@ fn altraThroughput(op: ir.function.Opcode, elem_float: bool) u32 { // Pipelined load-to-use: latency 4, but one independent load issues per mem port per cycle. .load => 1, // Pipelined FP/SIMD converts and unary FP ops. - .convert, .unary => 1, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 1, // NEON dotprod is a pipelined multiply-accumulate: one issues per cycle. .dot => 1, // A matmul is not native here; a placeholder, non-pipelined (== latency). @@ -118,7 +118,7 @@ fn cascadelakeLatency(op: ir.function.Opcode) u32 { .arith => |a| cascadelakeArith(a.op), .arith_imm => |a| cascadelakeArith(a.op), .load => 5, // L1 ~4.7 corrected - .convert, .unary => 4, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 4, .dot => 3, // mul-class .matmul => 64, // non-native placeholder .iconst, .fconst, .fconst128, .icmp, .select, .struct_new, .extract, .alloca, .call, .call_indirect, .global_addr, .store, .prefetch, .@"if" => 1, @@ -155,7 +155,7 @@ fn cascadelakeThroughput(op: ir.function.Opcode, elem_float: bool) u32 { .arith => |a| cascadelakeArithThroughput(a.op, elem_float), .arith_imm => |a| cascadelakeArithThroughput(a.op, elem_float), .load => 1, - .convert, .unary => 1, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 1, .dot => 1, .matmul => 64, .iconst, .fconst, .fconst128, .icmp, .select, .struct_new, .extract, .alloca, .call, .call_indirect, .global_addr, .store, .prefetch, .@"if" => 1, @@ -187,7 +187,7 @@ fn etsocLatency(op: ir.function.Opcode) u32 { .arith => |a| etsocArith(a.op), .arith_imm => |a| etsocArith(a.op), .load => 4, - .convert, .unary => 7, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 7, // A dot is a multiply-class op (4-way multiply-accumulate), grouped with `mul`. .dot => 8, // The et-soc fixed-tile matmul: the real tensor CSR-write sequence (load, wait, @@ -238,7 +238,7 @@ fn etsocThroughput(op: ir.function.Opcode, elem_float: bool) u32 { // Pipelined load-to-use (latency 4, throughput 1). .load => 1, // Pipelined VPU converts / unary FP. - .convert, .unary => 1, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 1, // The dot macro is an async multicycle MulDiv-class op: throughput == latency (never used by SLP). .dot => 8, // Matmul is the async CSR-write tensor sequence: non-pipelined, throughput == latency. @@ -274,7 +274,7 @@ fn riverInorderLatency(op: ir.function.Opcode) u32 { .arith => |a| riverInorderArith(a.op), .arith_imm => |a| riverInorderArith(a.op), .load => 2, - .convert, .unary => 2, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 2, // A dot is a multiply-class op (4-way multiply-accumulate), grouped with `mul`. // River carries no dotprod feature today; this is a placeholder in case one is added. .dot => 3, @@ -313,7 +313,7 @@ fn riverMacroLatency(op: ir.function.Opcode) u32 { .arith => |a| riverMacroArith(a.op), .arith_imm => |a| riverMacroArith(a.op), .load => 3, - .convert, .unary => 2, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 2, // A dot is a multiply-class op (4-way multiply-accumulate), grouped with `mul`. // River carries no dotprod feature today; this is a placeholder in case one is added. .dot => 3, @@ -355,7 +355,7 @@ fn riverInorderThroughput(op: ir.function.Opcode, elem_float: bool) u32 { .arith => |a| riverInorderArithThroughput(a.op, elem_float), .arith_imm => |a| riverInorderArithThroughput(a.op, elem_float), .load => 1, // pipelined load-to-use (latency 2) - .convert, .unary => 1, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 1, .dot => 3, // mul-class placeholder, non-pipelined here .matmul => 64, // no tensor unit here; non-pipelined placeholder .iconst, .fconst, .fconst128, .icmp, .select, .struct_new, .extract, .alloca, .call, .call_indirect, .global_addr, .store, .prefetch, .@"if" => 1, @@ -395,7 +395,7 @@ fn riverPipelinedThroughput(op: ir.function.Opcode, elem_float: bool) u32 { .arith => |a| riverPipelinedArithThroughput(a.op, elem_float), .arith_imm => |a| riverPipelinedArithThroughput(a.op, elem_float), .load => 1, - .convert, .unary => 1, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 1, .dot => 1, // pipelined mul-accumulate on the wider profile .matmul => 64, // no tensor unit here; non-pipelined placeholder .iconst, .fconst, .fconst128, .icmp, .select, .struct_new, .extract, .alloca, .call, .call_indirect, .global_addr, .store, .prefetch, .@"if" => 1, @@ -457,7 +457,7 @@ fn sm120Latency(op: ir.function.Opcode) u32 { // nvidia/schedule.zig), which means there IS no fixed number: the result lands an unknown // number of cycles after issue and the hardware needs a scoreboard, not a stall count. 8 is // a placeholder that says "more than an ALU op" and nothing more. - .convert, .unary => 8, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 8, // ISETP and SEL are ordinary fixed-pipe ALU instructions, unlike the bookkeeping group. .icmp, .select => 4, // A BAR.SYNC waits for the slowest warp of the workgroup to arrive. That is a property of @@ -501,7 +501,7 @@ fn sm120Throughput(op: ir.function.Opcode, elem_float: bool) u32 { // The load/store unit accepts a new request while earlier ones are outstanding. THIS is the // number the unroll rule is built on: independent loads cost issue slots, not latencies. .load, .atomic_rmw => 1, - .convert, .unary => 1, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 1, .icmp, .select => 1, // A barrier is not pipelined: a second one cannot start before the first releases. .barrier => 32, @@ -525,7 +525,7 @@ fn unitOfShared(op: ir.function.Opcode) UnitClass { .arith => |a| sharedArithUnit(a.op), .arith_imm => |a| sharedArithUnit(a.op), .icmp, .select, .iconst, .fconst, .fconst128, .global_addr => .alu, - .convert => .fpsimd, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4 => .fpsimd, .unary => |u| switch (u.op) { .reinterpret => .alu, .sqrt, .ceil, .floor, .trunc, .nearest => .fpsimd, diff --git a/libs/vulcan-opt/microarch/schedule.zig b/libs/vulcan-opt/microarch/schedule.zig index f16a1a7..a0baf45 100644 --- a/libs/vulcan-opt/microarch/schedule.zig +++ b/libs/vulcan-opt/microarch/schedule.zig @@ -28,7 +28,7 @@ const UnitClass = mm.UnitClass; /// Model. fn movable(op: ir.function.Opcode) bool { return switch (op) { - .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .struct_new, .extract, .convert, .unary, .alloca, .global_addr, .dot => true, + .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .struct_new, .extract, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .alloca, .global_addr, .dot => true, // matmul writes the `c` memory: a barrier, like store/prefetch, not reordered. .load, .store, .prefetch, .matmul, .@"if", .call, .call_indirect => false, // SM12 T3: `va_start`/`va_arg`/`va_end` all read/mutate the `va_list` object at @@ -77,6 +77,12 @@ fn collectOperands( }, .extract => |e| try buf.append(allocator, e.aggregate), .convert => |cv| try buf.append(allocator, cv.value), + .decode_low_float, .encode_low_float => |cv| try buf.append(allocator, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + try buf.append(allocator, cv.value); + try buf.append(allocator, cv.block_scale); + try buf.append(allocator, cv.global_scale); + }, .unary => |u| try buf.append(allocator, u.value), .struct_new => |sn| for (func.valueList(sn.fields)) |f| try buf.append(allocator, f), .call => |c| for (func.valueList(c.args)) |a| try buf.append(allocator, a), @@ -111,6 +117,29 @@ fn collectOperands( } } +test "scheduler records every nvfp4 dependency in operand order" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const result = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = payload, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + } }); + var operands: std.ArrayList(Value) = .empty; + defer operands.deinit(allocator); + try collectOperands(allocator, &func, func.definingInst(result).?, &operands); + try std.testing.expectEqualSlices(Value, &.{ payload, block_scale, global_scale }, operands.items); +} + /// The per-cycle port capacity for a unit class. A class with a 0 count (or `.none`) is treated as /// unmodeled and does not constrain issue, so a model that omits a class never deadlocks the loop. /// Public so the profitability cost model (cost.zig) reads the same port data the scheduler does. @@ -593,7 +622,7 @@ fn windowTestLatency(op: ir.function.Opcode) u32 { .mul, .mulh => 5, .div, .rem, .add, .sub, .bit_and, .bit_or, .bit_xor, .shl, .shr => 1, }, - .arith_imm, .iconst, .fconst, .fconst128, .icmp, .select, .struct_new, .extract, .convert, .unary, .alloca, .global_addr, .load, .store, .prefetch, .dot, .matmul, .@"if", .call, .call_indirect, .va_start, .va_arg, .va_end, .barrier, .atomic_rmw => 1, + .arith_imm, .iconst, .fconst, .fconst128, .icmp, .select, .struct_new, .extract, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .alloca, .global_addr, .load, .store, .prefetch, .dot, .matmul, .@"if", .call, .call_indirect, .va_start, .va_arg, .va_end, .barrier, .atomic_rmw => 1, }; } fn windowTestUnit(op: ir.function.Opcode) UnitClass { diff --git a/libs/vulcan-opt/microarch/splitunroll.zig b/libs/vulcan-opt/microarch/splitunroll.zig index a505b76..cf1183b 100644 --- a/libs/vulcan-opt/microarch/splitunroll.zig +++ b/libs/vulcan-opt/microarch/splitunroll.zig @@ -147,6 +147,10 @@ fn recognize(allocator: std.mem.Allocator, func: *Function, model: *const mm.Mod .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .global_addr, @@ -161,7 +165,7 @@ fn recognize(allocator: std.mem.Allocator, func: *Function, model: *const mm.Mod if (if_inst != null) return null; if_inst = inst; }, - .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .convert, .unary => {}, + .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => {}, else => return null, // impure/memory op in the header }; const iff = if_inst orelse return null; @@ -516,6 +520,10 @@ fn remapOp(func: *Function, op: Opcode, vmap: *const std.AutoHashMapUnmanaged(Va .icmp => |c| .{ .icmp = .{ .op = c.op, .lhs = rv(vmap, c.lhs), .rhs = rv(vmap, c.rhs) } }, .select => |s| .{ .select = .{ .cond = rv(vmap, s.cond), .then = rv(vmap, s.then), .@"else" = rv(vmap, s.@"else") } }, .convert => |c| .{ .convert = .{ .value = rv(vmap, c.value) } }, + .decode_low_float => |c| .{ .decode_low_float = .{ .value = rv(vmap, c.value), .format = c.format } }, + .encode_low_float => |c| .{ .encode_low_float = .{ .value = rv(vmap, c.value), .format = c.format } }, + .dequantize_nvfp4 => |c| .{ .dequantize_nvfp4 = .{ .value = rv(vmap, c.value), .block_scale = rv(vmap, c.block_scale), .global_scale = rv(vmap, c.global_scale), .block_application = c.block_application, .global_application = c.global_application } }, + .quantize_nvfp4 => |c| .{ .quantize_nvfp4 = .{ .value = rv(vmap, c.value), .block_scale = rv(vmap, c.block_scale), .global_scale = rv(vmap, c.global_scale), .block_application = c.block_application, .global_application = c.global_application } }, .unary => |u| .{ .unary = .{ .op = u.op, .value = rv(vmap, u.value) } }, .extract => |e| .{ .extract = .{ .aggregate = rv(vmap, e.aggregate), .index = e.index } }, .load => |l| .{ .load = .{ .ptr = rv(vmap, l.ptr), .@"volatile" = l.@"volatile" } }, @@ -592,6 +600,12 @@ fn useCounts(allocator: std.mem.Allocator, func: *const Function) Error![]u32 { }, .extract => |x| bump(counts, x.aggregate), .convert => |x| bump(counts, x.value), + .decode_low_float, .encode_low_float => |x| bump(counts, x.value), + .dequantize_nvfp4, .quantize_nvfp4 => |x| { + bump(counts, x.value); + bump(counts, x.block_scale); + bump(counts, x.global_scale); + }, .unary => |x| bump(counts, x.value), .load => |x| bump(counts, x.ptr), .store => |x| { diff --git a/libs/vulcan-opt/microarch/unroll.zig b/libs/vulcan-opt/microarch/unroll.zig index a2b8d7a..257373d 100644 --- a/libs/vulcan-opt/microarch/unroll.zig +++ b/libs/vulcan-opt/microarch/unroll.zig @@ -109,6 +109,10 @@ fn rebuildOpcode( .index = e.index, } }, .convert => |cv| .{ .convert = .{ .value = remapValue(value_map, cv.value) } }, + .decode_low_float => |cv| .{ .decode_low_float = .{ .value = remapValue(value_map, cv.value), .format = cv.format } }, + .encode_low_float => |cv| .{ .encode_low_float = .{ .value = remapValue(value_map, cv.value), .format = cv.format } }, + .dequantize_nvfp4 => |cv| .{ .dequantize_nvfp4 = .{ .value = remapValue(value_map, cv.value), .block_scale = remapValue(value_map, cv.block_scale), .global_scale = remapValue(value_map, cv.global_scale), .block_application = cv.block_application, .global_application = cv.global_application } }, + .quantize_nvfp4 => |cv| .{ .quantize_nvfp4 = .{ .value = remapValue(value_map, cv.value), .block_scale = remapValue(value_map, cv.block_scale), .global_scale = remapValue(value_map, cv.global_scale), .block_application = cv.block_application, .global_application = cv.global_application } }, .unary => |u| .{ .unary = .{ .op = u.op, .value = remapValue(value_map, u.value) } }, .alloca => |a| .{ .alloca = .{ .elem = a.elem } }, // Only `args` (and `target`/`ret_dest`) are Values. `is_variadic`/`num_fixed` and @@ -207,6 +211,47 @@ fn rebuildOpcode( }; } +test "unroll rebuild remaps every nvfp4 operand and preserves policies" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const old_value = try func.appendBlockParam(block, f32_t); + const old_block = try func.appendBlockParam(block, u8_t); + const old_global = try func.appendBlockParam(block, f32_t); + const new_value = try func.appendBlockParam(block, f32_t); + const new_block = try func.appendBlockParam(block, u8_t); + const new_global = try func.appendBlockParam(block, f32_t); + var values: ValueMap = .empty; + defer values.deinit(allocator); + try values.put(allocator, old_value, new_value); + try values.put(allocator, old_block, new_block); + try values.put(allocator, old_global, new_global); + var blocks: BlockMap = .empty; + defer blocks.deinit(allocator); + var scratch: std.ArrayList(Value) = .empty; + defer scratch.deinit(allocator); + + inline for (.{ false, true }) |quantize| { + const conversion: ir.function.NvFp4Convert = .{ + .value = old_value, + .block_scale = old_block, + .global_scale = old_global, + .block_application = .divide, + .global_application = .multiply, + }; + const rebuilt = try rebuildOpcode(&func, if (quantize) .{ .quantize_nvfp4 = conversion } else .{ .dequantize_nvfp4 = conversion }, &values, &blocks, &scratch, allocator); + const got = if (quantize) rebuilt.quantize_nvfp4 else rebuilt.dequantize_nvfp4; + try std.testing.expectEqual(new_value, got.value); + try std.testing.expectEqual(new_block, got.block_scale); + try std.testing.expectEqual(new_global, got.global_scale); + try std.testing.expectEqual(.divide, got.block_application); + try std.testing.expectEqual(.multiply, got.global_application); + } +} + /// Deep-copies `blocks` into `func`, remapping every Value operand through /// `value_map` and every Block target through `block_map`. Two passes: the /// first creates every cloned block and its params (so forward branches and @@ -681,7 +726,7 @@ fn eligible( if (idx != h_insts.len - 1) return null; // the `if` must end the block if_inst = inst; }, - .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .convert, .unary, .extract, .struct_new, .dot => {}, + .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .extract, .struct_new, .dot => {}, // load/store/call/call_indirect/alloca/global_addr are impure or memory ops. else => return null, } @@ -1359,6 +1404,8 @@ fn instUsesValue(func: *const Function, inst: Inst, value: Value) bool { .select => |x| x.cond == value or x.then == value or x.@"else" == value, .extract => |x| x.aggregate == value, .convert => |x| x.value == value, + .decode_low_float, .encode_low_float => |x| x.value == value, + .dequantize_nvfp4, .quantize_nvfp4 => |x| x.value == value or x.block_scale == value or x.global_scale == value, .unary => |x| x.value == value, .load => |x| x.ptr == value, .store => |x| x.value == value or x.ptr == value, @@ -1535,6 +1582,12 @@ fn collectOperands( }, .extract => |x| try set.put(a, x.aggregate, {}), .convert => |x| try set.put(a, x.value, {}), + .decode_low_float, .encode_low_float => |x| try set.put(a, x.value, {}), + .dequantize_nvfp4, .quantize_nvfp4 => |x| { + try set.put(a, x.value, {}); + try set.put(a, x.block_scale, {}); + try set.put(a, x.global_scale, {}); + }, .unary => |x| try set.put(a, x.value, {}), .load => |x| try set.put(a, x.ptr, {}), .store => |x| { @@ -1608,6 +1661,12 @@ fn replaceInBlock(func: *Function, block: Block, from: Value, to: Value) void { }, .extract => |*x| x.aggregate = rep(from, to, x.aggregate), .convert => |*x| x.value = rep(from, to, x.value), + .decode_low_float, .encode_low_float => |*x| x.value = rep(from, to, x.value), + .dequantize_nvfp4, .quantize_nvfp4 => |*x| { + x.value = rep(from, to, x.value); + x.block_scale = rep(from, to, x.block_scale); + x.global_scale = rep(from, to, x.global_scale); + }, .unary => |*x| x.value = rep(from, to, x.value), .load => |*x| x.ptr = rep(from, to, x.ptr), .store => |*x| { diff --git a/libs/vulcan-opt/uniform.zig b/libs/vulcan-opt/uniform.zig index 63a768c..d1fb30b 100644 --- a/libs/vulcan-opt/uniform.zig +++ b/libs/vulcan-opt/uniform.zig @@ -101,7 +101,7 @@ pub fn twoWayIf(func: *const Function, bi: usize) ?ir.function.If { return cf; }, .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select => {}, - .struct_new, .extract, .convert, .unary, .alloca => {}, + .struct_new, .extract, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .alloca => {}, .call, .call_indirect, .global_addr, .load, .store, .prefetch => {}, .va_start, .va_arg, .va_end, .dot, .matmul, .barrier, .atomic_rmw => {}, } @@ -129,6 +129,8 @@ fn operandsDiverge(func: *const Function, divergent: []const bool, op: ir.functi .icmp => |c| d(divergent, c.lhs) or d(divergent, c.rhs), .select => |s| d(divergent, s.cond) or d(divergent, s.then) or d(divergent, s.@"else"), .convert => |c| d(divergent, c.value), + .decode_low_float, .encode_low_float => |c| d(divergent, c.value), + .dequantize_nvfp4, .quantize_nvfp4 => |c| d(divergent, c.value) or d(divergent, c.block_scale) or d(divergent, c.global_scale), .unary => |u| d(divergent, u.value), .extract => |e| d(divergent, e.aggregate), .dot => |x| d(divergent, x.acc) or d(divergent, x.a) or d(divergent, x.b), @@ -236,7 +238,7 @@ pub fn analyze(allocator: std.mem.Allocator, func: *const Function) Error!Unifor propagateEdge(func, divergent, cf.@"else", &changed); }, .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select => {}, - .struct_new, .extract, .convert, .unary, .alloca => {}, + .struct_new, .extract, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .alloca => {}, .call, .call_indirect, .global_addr, .load, .store, .prefetch => {}, .va_start, .va_arg, .va_end, .dot, .matmul, .barrier, .atomic_rmw => {}, } diff --git a/libs/vulcan-opt/vectorize.zig b/libs/vulcan-opt/vectorize.zig index 3602d28..146e5e5 100644 --- a/libs/vulcan-opt/vectorize.zig +++ b/libs/vulcan-opt/vectorize.zig @@ -456,6 +456,10 @@ fn hasWriteBetween(func: *const Function, block: Block, lo: usize, hi: usize) bo .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .global_addr, @@ -591,6 +595,10 @@ fn resultsAreCoalesceableStores(func: *const Function, block: Block, results: [] .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .global_addr, @@ -646,6 +654,14 @@ fn valueUseCount(func: *const Function, v: Value) usize { .convert => |x| if (x.value == v) { n += 1; }, + .decode_low_float, .encode_low_float => |x| if (x.value == v) { + n += 1; + }, + .dequantize_nvfp4, .quantize_nvfp4 => |x| { + if (x.value == v) n += 1; + if (x.block_scale == v) n += 1; + if (x.global_scale == v) n += 1; + }, .unary => |x| if (x.value == v) { n += 1; }, @@ -850,6 +866,10 @@ fn coalesceStoreRun(allocator: std.mem.Allocator, func: *Function, block: Block, .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .global_addr, @@ -950,7 +970,7 @@ fn cleanup(allocator: std.mem.Allocator, func: *Function, coalesced_loads: *cons /// loads/stores/prefetch/calls/`if` are impure and kept (coalesced loads are handled separately). fn isPure(op: ir.function.Opcode) bool { return switch (op) { - .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .struct_new, .extract, .convert, .unary, .alloca, .global_addr, .dot => true, + .iconst, .fconst, .fconst128, .arith, .arith_imm, .icmp, .select, .struct_new, .extract, .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary, .alloca, .global_addr, .dot => true, .load, .store, .prefetch, .matmul, .@"if", .call, .call_indirect => false, // SM12 T3: mutate/read the `va_list` object at `list`, like `load`/`store` above. .va_start, .va_arg, .va_end => false, @@ -995,6 +1015,12 @@ fn countUses(func: *const Function, uses: []u32) void { }, .extract => |x| uses[@intFromEnum(x.aggregate)] += 1, .convert => |x| uses[@intFromEnum(x.value)] += 1, + .decode_low_float, .encode_low_float => |x| uses[@intFromEnum(x.value)] += 1, + .dequantize_nvfp4, .quantize_nvfp4 => |x| { + uses[@intFromEnum(x.value)] += 1; + uses[@intFromEnum(x.block_scale)] += 1; + uses[@intFromEnum(x.global_scale)] += 1; + }, .unary => |x| uses[@intFromEnum(x.value)] += 1, .load => |x| uses[@intFromEnum(x.ptr)] += 1, .store => |x| { diff --git a/libs/vulcan-spirv/emit.zig b/libs/vulcan-spirv/emit.zig index da15ad2..68a37fa 100644 --- a/libs/vulcan-spirv/emit.zig +++ b/libs/vulcan-spirv/emit.zig @@ -940,6 +940,10 @@ const Emitter = struct { .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .global_addr, @@ -974,6 +978,7 @@ const Emitter = struct { const val = self.idFor(c.value); try self.emit(&self.body, opcode, &.{ ty, self.idFor(result), val }); }, + .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4 => return error.UnsupportedConstruct, .icmp => |c| { const operand_kind = self.func.types.type_kind(self.func.valueType(c.lhs)); const opcode = cmpOpcode(operand_kind, c.op); @@ -1548,6 +1553,46 @@ test "a barrier is refused, not silently dropped" { try testing.expectError(error.UnsupportedConstruct, emitModule(allocator, &func, "sync")); } +test "both low float directions are refused while their results are live" { + const allocator = testing.allocator; + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) u16_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = .bf16 }; + const result = try func.appendInst(block, if (decode_direction) f32_t else u16_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + try testing.expectError(error.UnsupportedConstruct, emitModule(allocator, &func, "low_float")); + } +} + +test "both nvfp4 directions are refused live without mutating caller IR" { + const allocator = testing.allocator; + inline for (.{ true, false }) |dequantize| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const value = try func.appendBlockParam(block, f32_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: ir.function.NvFp4Convert = .{ .value = if (dequantize) payload else value, .block_scale = block_scale, .global_scale = global_scale, .block_application = .multiply, .global_application = .divide }; + const result = try func.appendInst(block, if (dequantize) f32_t else u8_t, if (dequantize) .{ .dequantize_nvfp4 = conversion } else .{ .quantize_nvfp4 = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + try testing.expectError(error.UnsupportedConstruct, emitModule(allocator, &func, "nvfp4")); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try testing.expectEqualSlices(u8, before, after); + } +} + test "a store is refused, not silently dropped" { // A store that vanishes from the emitted module is a miscompile: the module computes a // value and then writes it nowhere. This emitter cannot spell `OpStore` (see diff --git a/libs/vulcan-spirv/widen.zig b/libs/vulcan-spirv/widen.zig index e882b7f..b7d183e 100644 --- a/libs/vulcan-spirv/widen.zig +++ b/libs/vulcan-spirv/widen.zig @@ -362,6 +362,10 @@ fn emitArm(func: *Function, out: *std.ArrayListUnmanaged(Inst), visited: []bool, // today they cannot reach the emitted function either way. These arms are the // FIRST refusal, not the only one. Do not read them as the guard. .atomic_rmw, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .matmul, .va_start, .va_arg, @@ -434,6 +438,7 @@ fn widenFlattened(func: *Function) Error!void { .iconst => try new_insts.append(func.allocator, inst), // SPIR-V has no 128-bit float, so a shader holding one cannot widen. .fconst128 => return error.NotWidenable, + .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4 => return error.NotWidenable, .fconst => { const cval = func.instResult(inst).?; try new_insts.append(func.allocator, inst); diff --git a/libs/vulcan-target.zig b/libs/vulcan-target.zig index fd53f5b..52aa525 100644 --- a/libs/vulcan-target.zig +++ b/libs/vulcan-target.zig @@ -38,6 +38,7 @@ pub const object_emit = @import("vulcan-target/object_emit.zig"); /// Shared, target-independent Wimmer-Franz register allocator (target abstraction + algorithm). pub const wimmer = @import("vulcan-target/wimmer.zig"); +pub const f32_division_cases = @import("vulcan-target/tests/f32_division_cases.zig"); /// Shared, target-independent address-mode-folding analysis (recognizer + dead-add analysis). pub const addrfold = @import("vulcan-target/addrfold.zig"); diff --git a/libs/vulcan-target/aarch64/isel.zig b/libs/vulcan-target/aarch64/isel.zig index 1cd84e8..f958318 100644 --- a/libs/vulcan-target/aarch64/isel.zig +++ b/libs/vulcan-target/aarch64/isel.zig @@ -615,6 +615,9 @@ pub fn compileFunction(allocator: std.mem.Allocator, func: *const Function, caps var work = try func.clone(allocator); defer work.deinit(); + _ = try ir.expand.expandNvFp4(allocator, &work); + _ = try ir.expand.expandLowFloat(allocator, &work); + // Lower binary128 arithmetic, compares, conversions, and sqrt to soft-fp libcalls before any // numbering is built, so the call clobbers and the f128 argument placement are visible to the // register allocator. f128 DATA MOVEMENT (constant, move, spill, memory load and store, call @@ -1398,6 +1401,8 @@ fn emitFromAllocation(allocator: std.mem.Allocator, func: *const Function, caps: try storeResult(allocator, &code, ctx, result, rd); } }, + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, .convert => |cv| { const result = func.instResult(inst).?; // Any conversion touching f128 has no aarch64 form; the softfp pass rewrites it @@ -3446,6 +3451,12 @@ fn forEachOperand( }, .extract => |e| f(ctx, e.aggregate, false), .convert => |cv| f(ctx, cv.value, false), + .decode_low_float, .encode_low_float => |cv| f(ctx, cv.value, false), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + f(ctx, cv.value, false); + f(ctx, cv.block_scale, false); + f(ctx, cv.global_scale, false); + }, .unary => |u| f(ctx, u.value, false), // A folded load/store attributes its POINTER use to the fold BASE (the add's lhs), not the // raw ptr, so the base's live range reaches the mem op (including cross-block) and the dead @@ -3893,6 +3904,14 @@ fn usesOfInInst(func: *const Function, inst: ir.function.Inst, v: Value) usize { .convert => |cv| { if (cv.value == v) c += 1; }, + .decode_low_float, .encode_low_float => |cv| { + if (cv.value == v) c += 1; + }, + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + if (cv.value == v) c += 1; + if (cv.block_scale == v) c += 1; + if (cv.global_scale == v) c += 1; + }, .unary => |u| { if (u.value == v) c += 1; }, @@ -3992,6 +4011,12 @@ fn markUsedBitset(func: *const Function, inst: ir.function.Inst, fold: *const ad }, .extract => |e| setUsed(row, e.aggregate), .convert => |cv| setUsed(row, cv.value), + .decode_low_float, .encode_low_float => |cv| setUsed(row, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + setUsed(row, cv.value); + setUsed(row, cv.block_scale); + setUsed(row, cv.global_scale); + }, .unary => |u| setUsed(row, u.value), // Same fold reroute as `forEachOperand`: a folded mem op's pointer use is the fold base, so // the base's cross-block liveness reaches the mem op. `baseOf` is the raw ptr when unfolded. @@ -4713,6 +4738,36 @@ test "a barrier is rejected, not dropped like a prefetch" { try std.testing.expectError(error.Unsupported, selectFunction(allocator, &func)); } +test "all low float formats and directions compile without mutating caller IR" { + const allocator = std.testing.allocator; + inline for ([_]ir.low_float.Format{ .bf16, .f8_e4m3, .f8_e5m2 }) |format| { + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) payload_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = format }; + const result = try func.appendInst(block, if (decode_direction) f32_t else payload_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ .namespace = "test", .key = "preserve", .value = .flag } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + const before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + const code = try selectFunction(allocator, &func); + allocator.free(code); + const after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try std.testing.expectEqualStrings(before, after); + try std.testing.expectEqualSlices(u8, bits_before, bits_after); + } + } +} + test "an atomic is rejected, not dropped like a prefetch" { // This backend has no LSE or load-exclusive lowering, so there is no honest form. Both // IR forms are refused, and refusing must not be a panic: the reduction form has no diff --git a/libs/vulcan-target/aarch64/tests/native.zig b/libs/vulcan-target/aarch64/tests/native.zig index e982cf1..1b02b42 100644 --- a/libs/vulcan-target/aarch64/tests/native.zig +++ b/libs/vulcan-target/aarch64/tests/native.zig @@ -12,9 +12,148 @@ const disasm = @import("../disasm.zig"); const link = @import("../link.zig"); const jit = @import("../jit.zig"); const encode = @import("../encode.zig"); +const division_cases = @import("../../tests/f32_division_cases.zig"); const Function = ir.function.Function; +fn lowFloatWrapper(allocator: std.mem.Allocator, format: ir.low_float.Format, decode: bool) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const input = try func.appendBlockParam(block, if (decode) payload_t else u32_t); + const result = if (decode) blk: { + const value = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .format = format, .value = input } }); + break :blk try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = value } }); + } else blk: { + const value = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = input } }); + const payload = try func.appendInst(block, payload_t, .{ .encode_low_float = .{ .format = format, .value = value } }); + break :blk try func.appendInst(block, u32_t, .{ .convert = .{ .value = payload } }); + }; + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +test "native low float conversions match exhaustive and structured references" { + if (builtin.cpu.arch != .aarch64) return error.SkipZigTest; + const allocator = std.testing.allocator; + const mantissas = [_]u32{ 0, 1, 0x003f_ffff, 0x0040_0000, 0x0040_0001, 0x007f_7fff, 0x007f_8000, 0x007f_8001, 0x007f_ffff, 0x0000_7fff, 0x0000_8000, 0x0000_8001 }; + var distinguished_decode: [3][2]u32 = undefined; + for (std.enums.values(ir.low_float.Format)) |format| { + var decode_func = try lowFloatWrapper(allocator, format, true); + defer decode_func.deinit(); + const decode_words = try isel.selectFunction(allocator, &decode_func); + defer allocator.free(decode_words); + var decode_buffer = try jit.CodeBuffer.map(std.mem.sliceAsBytes(decode_words)); + defer decode_buffer.deinit(); + const decode = @as(*const fn (u32) callconv(.c) u32, @ptrCast(decode_buffer.memory.ptr)); + + var encode_func = try lowFloatWrapper(allocator, format, false); + defer encode_func.deinit(); + const encode_words = try isel.selectFunction(allocator, &encode_func); + defer allocator.free(encode_words); + var encode_buffer = try jit.CodeBuffer.map(std.mem.sliceAsBytes(encode_words)); + defer encode_buffer.deinit(); + const encode_fn = @as(*const fn (u32) callconv(.c) u32, @ptrCast(encode_buffer.memory.ptr)); + + const limit: usize = if (format == .bf16) 65536 else 256; + for (0..limit) |raw| { + const payload: u16 = @intCast(raw); + const expected_bits: u32 = @bitCast(try ir.low_float.decode(format, payload)); + const actual_bits = decode(payload); + try std.testing.expectEqual(expected_bits, actual_bits); + try std.testing.expectEqual(@as(u32, ir.low_float.encode(format, @bitCast(expected_bits))), encode_fn(actual_bits)); + } + distinguished_decode[@intFromEnum(format)] = .{ decode(0x7c), decode(0x7e) }; + switch (format) { + .bf16 => for (ir.low_float.bf16_encode_boundary_cases) |case| { + try std.testing.expectEqual(@as(u32, case.expected), encode_fn(case.source)); + }, + .f8_e4m3 => for (ir.low_float.e4_encode_boundary_cases) |case| { + try std.testing.expectEqual(@as(u32, case.expected), encode_fn(case.source)); + try std.testing.expectEqual(@as(u32, case.expected | 0x80), encode_fn(case.source | 0x8000_0000)); + }, + .f8_e5m2 => for (ir.low_float.e5_encode_boundary_cases) |case| { + try std.testing.expectEqual(@as(u32, case.expected), encode_fn(case.source)); + try std.testing.expectEqual(@as(u32, case.expected | 0x80), encode_fn(case.source | 0x8000_0000)); + }, + } + if (format != .bf16) for (ir.low_float.fp8_encode_special_cases) |case| { + const expected = if (format == .f8_e4m3) case.e4 else case.e5; + try std.testing.expectEqual(@as(u32, expected), encode_fn(case.source)); + }; + for (0..256) |exponent| { + for ([_]u32{ 0, 0x8000_0000 }) |sign| { + for (mantissas) |mantissa| { + const bits = sign | (@as(u32, @intCast(exponent)) << 23) | mantissa; + try std.testing.expectEqual(@as(u32, ir.low_float.encode(format, @bitCast(bits))), encode_fn(bits)); + } + } + } + } + try std.testing.expect(distinguished_decode[@intFromEnum(ir.low_float.Format.f8_e4m3)][0] != distinguished_decode[@intFromEnum(ir.low_float.Format.f8_e5m2)][0]); + try std.testing.expect(distinguished_decode[@intFromEnum(ir.low_float.Format.f8_e4m3)][1] != distinguished_decode[@intFromEnum(ir.low_float.Format.f8_e5m2)][1]); +} + +fn f32DivisionWrapper(allocator: std.mem.Allocator) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const numerator_bits = try func.appendBlockParam(block, u32_t); + const denominator_bits = try func.appendBlockParam(block, u32_t); + const numerator = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = numerator_bits } }); + const denominator = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = denominator_bits } }); + const quotient = try func.appendInst(block, f32_t, .{ .arith = .{ .op = .div, .lhs = numerator, .rhs = denominator } }); + const result = try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = quotient } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +const F32DivisionBitsFn = *const fn (u32, u32) callconv(.c) u32; + +fn expectF32Division(divide: F32DivisionBitsFn, numerator_bits: u32, denominator_bits: u32) !void { + const numerator: f32 = @bitCast(numerator_bits); + const denominator: f32 = @bitCast(denominator_bits); + const expected = numerator / denominator; + const actual_bits = divide(numerator_bits, denominator_bits); + const actual: f32 = @bitCast(actual_bits); + if (std.math.isNan(expected)) { + try std.testing.expect(std.math.isNan(actual)); + } else { + const Observation = struct { numerator: u32, denominator: u32, result: u32 }; + try std.testing.expectEqual( + Observation{ .numerator = numerator_bits, .denominator = denominator_bits, .result = @bitCast(expected) }, + Observation{ .numerator = numerator_bits, .denominator = denominator_bits, .result = actual_bits }, + ); + } +} + +test "native expanded f32 division matches hardware across boundaries and random bits" { + if (builtin.cpu.arch != .aarch64) return error.SkipZigTest; + const allocator = std.testing.allocator; + var func = try f32DivisionWrapper(allocator); + defer func.deinit(); + try std.testing.expect(try ir.expand.expandF32Div(allocator, &func)); + try std.testing.expect(!(try ir.expand.expandF32Div(allocator, &func))); + var diagnostics = try ir.verify.verify(allocator, &func, .low); + defer diagnostics.deinit(); + try std.testing.expect(diagnostics.ok()); + + const words = try isel.selectFunction(allocator, &func); + defer allocator.free(words); + var buffer = try jit.CodeBuffer.map(std.mem.sliceAsBytes(words)); + defer buffer.deinit(); + const divide: F32DivisionBitsFn = @ptrCast(buffer.memory.ptr); + + var corpus = try division_cases.build(allocator); + defer corpus.deinit(); + for (corpus.pairs) |pair| try expectF32Division(divide, pair.numerator, pair.denominator); +} + /// Compile `func` to A64 and assert its disassembled listing equals `expected`. This round-trips /// codegen through the disassembler. So it checks the actual instructions and register allocation, /// not just the run result. It works on any host, because it never executes the code. diff --git a/libs/vulcan-target/addrfold.zig b/libs/vulcan-target/addrfold.zig index fc4d74a..79ab4eb 100644 --- a/libs/vulcan-target/addrfold.zig +++ b/libs/vulcan-target/addrfold.zig @@ -180,6 +180,12 @@ fn countUses(func: *const Function, uses: []u32) void { }, .extract => |e| uses[@intFromEnum(e.aggregate)] += 1, .convert => |cv| uses[@intFromEnum(cv.value)] += 1, + .decode_low_float, .encode_low_float => |cv| uses[@intFromEnum(cv.value)] += 1, + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + uses[@intFromEnum(cv.value)] += 1; + uses[@intFromEnum(cv.block_scale)] += 1; + uses[@intFromEnum(cv.global_scale)] += 1; + }, .unary => |u| uses[@intFromEnum(u.value)] += 1, .load => |l| uses[@intFromEnum(l.ptr)] += 1, .store => |st| { diff --git a/libs/vulcan-target/c.zig b/libs/vulcan-target/c.zig index a6e5c65..821fb21 100644 --- a/libs/vulcan-target/c.zig +++ b/libs/vulcan-target/c.zig @@ -361,6 +361,8 @@ const Emitter = struct { const op = func.opcode(inst); const res = func.instResult(inst); switch (op) { + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, .iconst => |val| try self.print(" v{d} = {d};\n", .{ self.name(res.?), val }), .fconst => |val| try self.emitFconst(res.?, val), // The u128 payload is the binary128 bit pattern; decode it and print the @@ -905,6 +907,46 @@ test "a barrier is rejected, not emitted as a silent no-op" { try std.testing.expectError(error.Unsupported, emitFunction(std.testing.allocator, &func, "sync")); } +test "both low float directions are rejected while their results are live" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) u16_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = .bf16 }; + const result = try func.appendInst(block, if (decode_direction) f32_t else u16_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + try std.testing.expectError(error.Unsupported, emitFunction(allocator, &func, "low_float")); + } +} + +test "both nvfp4 directions are rejected live without mutating caller IR" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |dequantize| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const value = try func.appendBlockParam(block, f32_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: ir.function.NvFp4Convert = .{ .value = if (dequantize) payload else value, .block_scale = block_scale, .global_scale = global_scale, .block_application = .multiply, .global_application = .divide }; + const result = try func.appendInst(block, if (dequantize) f32_t else u8_t, if (dequantize) .{ .dequantize_nvfp4 = conversion } else .{ .quantize_nvfp4 = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + try std.testing.expectError(error.Unsupported, emitFunction(allocator, &func, "nvfp4")); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try std.testing.expectEqualSlices(u8, before, after); + } +} + test "an atomic is rejected, not emitted as a plain read-modify-write" { // Plain C has no guaranteed atomic for a pointer that is not `_Atomic`-qualified. An // ordinary read-modify-write would compile and would NOT be atomic, which is worse than diff --git a/libs/vulcan-target/js.zig b/libs/vulcan-target/js.zig index f5461eb..8b74583 100644 --- a/libs/vulcan-target/js.zig +++ b/libs/vulcan-target/js.zig @@ -238,6 +238,8 @@ const Emitter = struct { const op = func.opcode(inst); const res = func.instResult(inst); switch (op) { + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, .iconst => |val| try self.emitIconst(res.?, val, indent), // JS numbers are f64; there is no f128. emitFunction's gate rejects any // function using f128 first, and this arm is the belt to its suspenders. @@ -757,6 +759,46 @@ test "a barrier is rejected, not emitted as a silent no-op" { try std.testing.expectError(error.Unsupported, emitFunction(std.testing.allocator, &func, "sync")); } +test "both low float directions are rejected while their results are live" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) u16_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = .bf16 }; + const result = try func.appendInst(block, if (decode_direction) f32_t else u16_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + try std.testing.expectError(error.Unsupported, emitFunction(allocator, &func, "low_float")); + } +} + +test "both nvfp4 directions are rejected live without mutating caller IR" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |dequantize| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const value = try func.appendBlockParam(block, f32_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: ir.function.NvFp4Convert = .{ .value = if (dequantize) payload else value, .block_scale = block_scale, .global_scale = global_scale, .block_application = .multiply, .global_application = .divide }; + const result = try func.appendInst(block, if (dequantize) f32_t else u8_t, if (dequantize) .{ .dequantize_nvfp4 = conversion } else .{ .quantize_nvfp4 = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + try std.testing.expectError(error.Unsupported, emitFunction(allocator, &func, "nvfp4")); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try std.testing.expectEqualSlices(u8, before, after); + } +} + test "an atomic is rejected, not emitted as a plain read-modify-write" { // This backend emits single-threaded JS over plain numbers, not a SharedArrayBuffer, so // `Atomics.add` has nothing to apply to and an ordinary read-modify-write would not be diff --git a/libs/vulcan-target/nvidia/isel.zig b/libs/vulcan-target/nvidia/isel.zig index b0bd58e..e3633e6 100644 --- a/libs/vulcan-target/nvidia/isel.zig +++ b/libs/vulcan-target/nvidia/isel.zig @@ -606,11 +606,34 @@ pub fn compileShader(allocator: std.mem.Allocator, func: *Function, stage: Stage /// `compileShader` with explicit code-generation options. See `Options`. pub fn compileShaderOpts(allocator: std.mem.Allocator, func: *Function, stage: Stage, a: gpu.Abi, options: Options) Error!Kernel { + var work = try func.clone(allocator); + defer work.deinit(); + if (stage == .compute) { + _ = try ir.expand.expandNvFp4(allocator, &work); + _ = try ir.expand.expandF32Div(allocator, &work); + _ = try ir.expand.expandLowFloat(allocator, &work); + } + return compileShaderOwned(allocator, &work, stage, a, options); +} + +fn compileShaderOwned(allocator: std.mem.Allocator, func: *Function, stage: Stage, a: gpu.Abi, options: Options) Error!Kernel { // This backend does not lower f16 yet. Reject it cleanly instead of // silently treating it as f64. This check covers both this direct entry // and compileKernel, which calls this function. if (ir.function.functionUsesF16(func)) return error.Unsupported; if (ir.function.functionUsesF128(func)) return error.Unsupported; + for (0..func.blockCount()) |bi| { + for (func.blockInsts(@enumFromInt(bi))) |inst| switch (func.opcode(inst)) { + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, + .arith => |arithmetic| if (stage == .compute and arithmetic.op == .div and + std.meta.activeTag(func.types.type_kind(func.valueType(func.instResult(inst).?))) == .float) + { + return error.Unsupported; + }, + else => {}, + }; + } const nblocks = func.blockCount(); if (nblocks == 0) return error.Unsupported; @@ -4110,14 +4133,34 @@ fn lowerInst(allocator: std.mem.Allocator, func: *const Function, loc: *std.Auto const result = func.instResult(inst).?; const rd = gprOf(loc.*, result); const rs = gprOf(loc.*, cv.value); - const dst_float = isFloat(func, result); - const src_float = isFloat(func, cv.value); - if (src_float and !dst_float) { - try code.append(allocator, encode.f2i(rd, rs, isSignedRaw(func, result), .{})); // f32 -> i32 - } else if (!src_float and dst_float) { - try code.append(allocator, encode.i2f(rd, rs, isSignedRaw(func, cv.value), .{})); // i32 -> f32 - } else { - return error.Unsupported; // int-to-int width change, or f32-to-f64, is not modeled yet + const dst_kind = func.types.type_kind(func.valueType(result)); + const src_kind = func.types.type_kind(func.valueType(cv.value)); + switch (src_kind) { + .float => |src_float| switch (dst_kind) { + .int => |dst_int| { + if (src_float != .f32 or dst_int.bits != 32) return error.Unsupported; + try code.append(allocator, encode.f2i(rd, rs, dst_int.signedness == .signed, .{})); + }, + else => return error.Unsupported, + }, + .int => |src_int| switch (dst_kind) { + .float => |dst_float| { + if (dst_float != .f32 or src_int.bits != 32) return error.Unsupported; + try code.append(allocator, encode.i2f(rd, rs, src_int.signedness == .signed, .{})); + }, + .int => |dst_int| { + if (src_int.signedness != .unsigned or dst_int.signedness != .unsigned) return error.Unsupported; + const mask: u32 = if ((src_int.bits == 8 and dst_int.bits == 32) or (src_int.bits == 32 and dst_int.bits == 8)) + 0xff + else if ((src_int.bits == 16 and dst_int.bits == 32) or (src_int.bits == 32 and dst_int.bits == 16)) + 0xffff + else + return error.Unsupported; + try code.append(allocator, encode.lop3Imm(rd, rs, mask, encode.LUT_AND, .{})); + }, + else => return error.Unsupported, + }, + else => return error.Unsupported, } }, .alloca => { @@ -4688,6 +4731,12 @@ fn forEachUse(func: *const Function, inst: ir.function.Inst, last_use: []u32, po }, .extract => |e| markUse(last_use, e.aggregate, pos), .convert => |cv| markUse(last_use, cv.value, pos), + .decode_low_float, .encode_low_float => |cv| markUse(last_use, cv.value, pos), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + markUse(last_use, cv.value, pos); + markUse(last_use, cv.block_scale, pos); + markUse(last_use, cv.global_scale, pos); + }, .unary => |u| markUse(last_use, u.value, pos), .load => |ld| markUse(last_use, ld.ptr, pos), .store => |st| { @@ -4758,6 +4807,12 @@ fn markUsedBitset(func: *const Function, inst: ir.function.Inst, row: []bool) vo }, .extract => |e| setUsed(row, e.aggregate), .convert => |cv| setUsed(row, cv.value), + .decode_low_float, .encode_low_float => |cv| setUsed(row, cv.value), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + setUsed(row, cv.value); + setUsed(row, cv.block_scale); + setUsed(row, cv.global_scale); + }, .unary => |u| setUsed(row, u.value), .load => |ld| setUsed(row, ld.ptr), .store => |st| { @@ -5801,6 +5856,409 @@ test "an f16 function is rejected cleanly, not miscompiled as f64" { try testing.expectError(error.Unsupported, compileKernel(allocator, &func, nvidia_abi)); } +fn buildLowFloatFixture(allocator: std.mem.Allocator, format: ir.low_float.Format, decode_direction: bool, later_failure: bool) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) payload_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = format }; + const result = try func.appendInst(block, if (decode_direction) f32_t else payload_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + try func.addAttr(.{ .inst = func.definingInst(result).? }, .{ .custom = .{ .namespace = "debug", .key = "line", .value = .{ .int = 44 } } }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ .namespace = "test", .key = "preserve", .value = .flag } }); + if (later_failure) try func.appendBarrier(block, .subgroup); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +const ComputeFunnel = enum { kernel, kernel_opts, shader, shader_opts }; + +fn compileComputeFunnel(allocator: std.mem.Allocator, func: *Function, funnel: ComputeFunnel) Error!Kernel { + return switch (funnel) { + .kernel => compileKernel(allocator, func, nvidia_abi), + .kernel_opts => compileKernelOpts(allocator, func, nvidia_abi, .{}), + .shader => compileShader(allocator, func, .compute, nvidia_abi), + .shader_opts => compileShaderOpts(allocator, func, .compute, nvidia_abi, .{}), + }; +} + +test "all NVIDIA compute funnels expand every low float conversion without mutating callers" { + const allocator = testing.allocator; + inline for (std.enums.values(ir.low_float.Format)) |format| inline for (.{ true, false }) |decode_direction| { + var baseline_code: ?[]u32 = null; + defer if (baseline_code) |code| allocator.free(code); + inline for (std.enums.values(ComputeFunnel)) |funnel| { + var func = try buildLowFloatFixture(allocator, format, decode_direction, false); + defer func.deinit(); + const text_before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + var kernel = try compileComputeFunnel(allocator, &func, funnel); + defer kernel.deinit(allocator); + try testing.expect(kernel.code.len != 0); + if (baseline_code) |code| { + try testing.expectEqualSlices(u32, code, kernel.code); + } else { + baseline_code = try allocator.dupe(u32, kernel.code); + } + const text_after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try testing.expectEqualStrings(text_before, text_after); + try testing.expectEqualSlices(u8, bits_before, bits_after); + + var expanded = try func.clone(allocator); + defer expanded.deinit(); + try testing.expect(try ir.expand.expandLowFloat(allocator, &expanded)); + var diags = try ir.verify.verify(allocator, &expanded, .low); + defer diags.deinit(); + try testing.expect(diags.ok()); + for (0..expanded.blockCount()) |block_index| for (expanded.blockInsts(@enumFromInt(block_index))) |inst| switch (expanded.opcode(inst)) { + .decode_low_float, .encode_low_float => return error.TestUnexpectedResult, + else => {}, + }; + } + }; +} + +test "NVIDIA compute low float expansion reaches later failure without mutating callers" { + const allocator = testing.allocator; + inline for (std.enums.values(ir.low_float.Format)) |format| inline for (.{ true, false }) |decode_direction| { + var func = try buildLowFloatFixture(allocator, format, decode_direction, true); + defer func.deinit(); + const text_before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + try testing.expectError(error.Unsupported, compileKernel(allocator, &func, nvidia_abi)); + const text_after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try testing.expectEqualStrings(text_before, text_after); + try testing.expectEqualSlices(u8, bits_before, bits_after); + }; +} + +test "NVIDIA graphics keeps every low float conversion unsupported" { + const allocator = testing.allocator; + inline for (std.enums.values(ir.low_float.Format)) |format| inline for (.{ true, false }) |decode_direction| { + var func = try buildLowFloatFixture(allocator, format, decode_direction, false); + defer func.deinit(); + try testing.expectError(error.Unsupported, compileShader(allocator, &func, .fragment, nvidia_abi)); + try testing.expectError(error.Unsupported, compileShaderOpts(allocator, &func, .vertex, nvidia_abi, .{})); + }; +} + +fn buildConvertFixture(allocator: std.mem.Allocator, source_kind: ir.types.TypeKind, result_kind: ir.types.TypeKind, dirty_upper: bool) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const source_t = try func.types.intern(source_kind); + const result_t = try func.types.intern(result_kind); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, source_t); + const converted_source = if (dirty_upper) dirty: { + const bits = source_kind.int.bits; + break :dirty try func.appendArithImm(block, source_t, .add, source, @as(i64, 1) << @intCast(@min(bits, 31))); + } else source; + const result = try func.appendInst(block, result_t, .{ .convert = .{ .value = converted_source } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +test "NVIDIA unsigned low width conversions emit exact integer masks" { + const allocator = testing.allocator; + const cases = [_]struct { source_bits: u16, result_bits: u16, mask: u32 }{ + .{ .source_bits = 8, .result_bits = 32, .mask = 0xff }, + .{ .source_bits = 16, .result_bits = 32, .mask = 0xffff }, + .{ .source_bits = 32, .result_bits = 8, .mask = 0xff }, + .{ .source_bits = 32, .result_bits = 16, .mask = 0xffff }, + }; + for (cases) |case| { + var func = try buildConvertFixture( + allocator, + .{ .int = .{ .signedness = .unsigned, .bits = case.source_bits } }, + .{ .int = .{ .signedness = .unsigned, .bits = case.result_bits } }, + true, + ); + defer func.deinit(); + var kernel = try compileKernel(allocator, &func, nvidia_abi); + defer kernel.deinit(allocator); + var saw_mask = false; + var index: usize = 0; + while (index * 4 < kernel.code.len) : (index += 1) { + const opcode = opAt(kernel.code, index); + try testing.expect(opcode != encode.I2F_OPCODE); + try testing.expect(opcode != encode.F2I_OPCODE); + if (opcode == 0x812 and immAt(kernel.code, index) == case.mask) saw_mask = true; + } + try testing.expect(saw_mask); + } +} + +test "NVIDIA conversion selector keeps its scalar whitelist narrow" { + const allocator = testing.allocator; + const int_cases = [_]struct { source: ir.types.Int, result: ir.types.Int }{ + .{ .source = .{ .signedness = .unsigned, .bits = 8 }, .result = .{ .signedness = .unsigned, .bits = 16 } }, + .{ .source = .{ .signedness = .unsigned, .bits = 16 }, .result = .{ .signedness = .unsigned, .bits = 8 } }, + .{ .source = .{ .signedness = .unsigned, .bits = 32 }, .result = .{ .signedness = .unsigned, .bits = 32 } }, + .{ .source = .{ .signedness = .signed, .bits = 8 }, .result = .{ .signedness = .signed, .bits = 32 } }, + .{ .source = .{ .signedness = .signed, .bits = 32 }, .result = .{ .signedness = .signed, .bits = 8 } }, + .{ .source = .{ .signedness = .unsigned, .bits = 64 }, .result = .{ .signedness = .unsigned, .bits = 32 } }, + .{ .source = .{ .signedness = .unsigned, .bits = 32 }, .result = .{ .signedness = .unsigned, .bits = 64 } }, + }; + for (int_cases) |case| { + var func = try buildConvertFixture(allocator, .{ .int = case.source }, .{ .int = case.result }, false); + defer func.deinit(); + try testing.expectError(error.Unsupported, compileKernel(allocator, &func, nvidia_abi)); + } + + const mixed_cases = [_]struct { source: ir.types.TypeKind, result: ir.types.TypeKind }{ + .{ .source = .{ .int = .{ .signedness = .unsigned, .bits = 16 } }, .result = .{ .float = .f32 } }, + .{ .source = .{ .float = .f32 }, .result = .{ .int = .{ .signedness = .unsigned, .bits = 16 } } }, + .{ .source = .{ .float = .f64 }, .result = .{ .int = .{ .signedness = .signed, .bits = 32 } } }, + .{ .source = .{ .float = .f32 }, .result = .{ .float = .f64 } }, + }; + for (mixed_cases) |case| { + var func = try buildConvertFixture(allocator, case.source, case.result, false); + defer func.deinit(); + try testing.expectError(error.Unsupported, compileKernel(allocator, &func, nvidia_abi)); + } + + inline for (.{ std.builtin.Signedness.signed, std.builtin.Signedness.unsigned }) |signedness| { + var to_float = try buildConvertFixture(allocator, .{ .int = .{ .signedness = signedness, .bits = 32 } }, .{ .float = .f32 }, false); + defer to_float.deinit(); + var to_float_kernel = try compileKernel(allocator, &to_float, nvidia_abi); + defer to_float_kernel.deinit(allocator); + var from_float = try buildConvertFixture(allocator, .{ .float = .f32 }, .{ .int = .{ .signedness = signedness, .bits = 32 } }, false); + defer from_float.deinit(); + var from_float_kernel = try compileKernel(allocator, &from_float, nvidia_abi); + defer from_float_kernel.deinit(allocator); + try testing.expect(to_float_kernel.code.len != 0); + try testing.expect(from_float_kernel.code.len != 0); + } + + var vector_func = Function.init(allocator); + defer vector_func.deinit(); + const u8_t = try vector_func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try vector_func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const v8_t = try vector_func.types.intern(.{ .vector = .{ .len = 4, .elem = u8_t } }); + const v32_t = try vector_func.types.intern(.{ .vector = .{ .len = 4, .elem = u32_t } }); + const block = try vector_func.appendBlock(); + const source = try vector_func.appendBlockParam(block, v8_t); + const result = try vector_func.appendInst(block, v32_t, .{ .convert = .{ .value = source } }); + vector_func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + if (compileKernel(allocator, &vector_func, nvidia_abi)) |kernel_value| { + var kernel = kernel_value; + defer kernel.deinit(allocator); + return error.TestUnexpectedResult; + } else |err| { + try testing.expect(err == error.Unsupported or err == error.UnsupportedParamType); + } +} + +fn buildF32DivisionFixture(allocator: std.mem.Allocator, later_failure: bool) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const numerator = try func.appendBlockParam(block, f32_t); + const denominator = try func.appendBlockParam(block, f32_t); + const result = try func.appendInst(block, f32_t, .{ .arith = .{ .op = .div, .lhs = numerator, .rhs = denominator } }); + try func.addAttr(.{ .inst = func.definingInst(result).? }, .{ .custom = .{ .namespace = "debug", .key = "line", .value = .{ .int = 45 } } }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ .namespace = "test", .key = "division", .value = .flag } }); + if (later_failure) try func.appendBarrier(block, .subgroup); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +fn kernelHasOpcode(kernel: *const Kernel, opcode: u32) bool { + for (0..kernel.code.len / 4) |index| if (opAt(kernel.code, index) == opcode) return true; + return false; +} + +test "all NVIDIA compute funnels expand exact f32 division without mutating callers" { + const allocator = testing.allocator; + var baseline_code: ?[]u32 = null; + defer if (baseline_code) |code| allocator.free(code); + inline for (std.enums.values(ComputeFunnel)) |funnel| { + var func = try buildF32DivisionFixture(allocator, false); + defer func.deinit(); + const text_before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + var kernel = try compileComputeFunnel(allocator, &func, funnel); + defer kernel.deinit(allocator); + try testing.expect(!kernelHasOpcode(&kernel, encode.MUFU_OPCODE)); + if (baseline_code) |code| { + try testing.expectEqualSlices(u32, code, kernel.code); + } else { + baseline_code = try allocator.dupe(u32, kernel.code); + } + const text_after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try testing.expectEqualStrings(text_before, text_after); + try testing.expectEqualSlices(u8, bits_before, bits_after); + } +} + +test "NVIDIA exact division preserves callers on later failure and rejects an unexpanded compute preflight" { + const allocator = testing.allocator; + var func = try buildF32DivisionFixture(allocator, true); + defer func.deinit(); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + try testing.expectError(error.Unsupported, compileKernel(allocator, &func, nvidia_abi)); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try testing.expectEqualSlices(u8, before, after); + + var raw = try buildF32DivisionFixture(allocator, false); + defer raw.deinit(); + try testing.expectError(error.Unsupported, compileShaderOwned(allocator, &raw, .compute, nvidia_abi, .{})); +} + +test "NVIDIA graphics retains reciprocal multiplication for ordinary f32 division" { + const allocator = testing.allocator; + inline for (.{ Stage.vertex, Stage.fragment }) |stage| { + var func = try buildF32DivisionFixture(allocator, false); + defer func.deinit(); + var kernel = try compileShader(allocator, &func, stage, nvidia_abi); + defer kernel.deinit(allocator); + try testing.expect(kernelHasOpcode(&kernel, encode.MUFU_OPCODE)); + try testing.expect(kernelHasOpcode(&kernel, FMUL_REG)); + } +} + +fn buildNvFp4Fixture( + allocator: std.mem.Allocator, + dequantize: bool, + block_application: ir.nvfp4.ScaleApplication, + global_application: ir.nvfp4.ScaleApplication, + later_failure: bool, +) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const value = try func.appendBlockParam(block, f32_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: ir.function.NvFp4Convert = .{ + .value = if (dequantize) payload else value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = block_application, + .global_application = global_application, + }; + const result = try func.appendInst(block, if (dequantize) f32_t else u8_t, if (dequantize) + .{ .dequantize_nvfp4 = conversion } + else + .{ .quantize_nvfp4 = conversion }); + try func.addAttr(.{ .inst = func.definingInst(result).? }, .{ .custom = .{ .namespace = "debug", .key = "line", .value = .{ .int = 45 } } }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ .namespace = "test", .key = "nvfp4", .value = .flag } }); + if (later_failure) try func.appendBarrier(block, .subgroup); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +test "all NVIDIA compute funnels expand every nvfp4 policy and direction" { + const allocator = testing.allocator; + inline for (.{ true, false }) |dequantize| + inline for (.{ ir.nvfp4.ScaleApplication.multiply, .divide }) |block_application| + inline for (.{ ir.nvfp4.ScaleApplication.multiply, .divide }) |global_application| + inline for (std.enums.values(ComputeFunnel)) |funnel| { + var func = try buildNvFp4Fixture(allocator, dequantize, block_application, global_application, false); + defer func.deinit(); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + var kernel = try compileComputeFunnel(allocator, &func, funnel); + defer kernel.deinit(allocator); + try testing.expect(kernel.code.len != 0); + try testing.expect(!kernelHasOpcode(&kernel, encode.MUFU_OPCODE)); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try testing.expectEqualSlices(u8, before, after); + }; +} + +test "NVIDIA graphics rejects every nvfp4 policy and later compute failures preserve callers" { + const allocator = testing.allocator; + inline for (.{ true, false }) |dequantize| + inline for (.{ ir.nvfp4.ScaleApplication.multiply, .divide }) |block_application| + inline for (.{ ir.nvfp4.ScaleApplication.multiply, .divide }) |global_application| { + var func = try buildNvFp4Fixture(allocator, dequantize, block_application, global_application, false); + defer func.deinit(); + try testing.expectError(error.Unsupported, compileShader(allocator, &func, .vertex, nvidia_abi)); + try testing.expectError(error.Unsupported, compileShaderOpts(allocator, &func, .fragment, nvidia_abi, .{})); + }; + inline for (.{ true, false }) |dequantize| { + var func = try buildNvFp4Fixture(allocator, dequantize, .divide, .divide, true); + defer func.deinit(); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + try testing.expectError(error.Unsupported, compileKernel(allocator, &func, nvidia_abi)); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try testing.expectEqualSlices(u8, before, after); + } +} + +test "NVIDIA production allocation handles interleaved division and nvfp4 pressure" { + const allocator = testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const bool_t = try func.types.intern(.bool); + const ptr_t = try func.types.ptrGlobal(); + const block = try func.appendBlock(); + const output = try func.appendBlockParam(block, ptr_t); + const numerator = try func.appendBlockParam(block, f32_t); + const denominator = try func.appendBlockParam(block, f32_t); + const carrier = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const value = try func.appendBlockParam(block, f32_t); + const live_predicate = try func.appendInst(block, bool_t, .{ .icmp = .{ .op = .lt, .lhs = numerator, .rhs = denominator } }); + const quotient = try func.appendInst(block, f32_t, .{ .arith = .{ .op = .div, .lhs = numerator, .rhs = denominator } }); + const dequantized = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = carrier, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .divide, + .global_application = .divide, + } }); + const quantized = try func.appendInst(block, u8_t, .{ .quantize_nvfp4 = .{ + .value = value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .multiply, + } }); + const selected = try func.appendInst(block, f32_t, .{ .select = .{ .cond = live_predicate, .then = quotient, .@"else" = dequantized } }); + const second_quotient = try func.appendInst(block, f32_t, .{ .arith = .{ .op = .div, .lhs = selected, .rhs = value } }); + try func.appendStore(block, selected, output); + try func.appendStore(block, quantized, try func.appendArithImm(block, ptr_t, .add, output, 4)); + try func.appendStore(block, second_quotient, try func.appendArithImm(block, ptr_t, .add, output, 8)); + func.setTerminator(block, .{ .ret = ir.function.Ret.none() }); + + var kernel = try compileKernel(allocator, &func, nvidia_abi); + defer kernel.deinit(allocator); + try testing.expect(kernel.code.len != 0); + try testing.expect(kernel.reg_count > 0); + try testing.expect(kernel.reg_count <= 255); + try testing.expect(!kernelHasOpcode(&kernel, encode.MUFU_OPCODE)); +} + test "compiles control flow: a max via if and a merge block" { const allocator = testing.allocator; var func = Function.init(allocator); diff --git a/libs/vulcan-target/nvidia/schedule.zig b/libs/vulcan-target/nvidia/schedule.zig index 23f9a40..4732086 100644 --- a/libs/vulcan-target/nvidia/schedule.zig +++ b/libs/vulcan-target/nvidia/schedule.zig @@ -1183,23 +1183,28 @@ fn isControlFlow(opcode: u32) bool { }; } -/// The latency of a COUPLED producer, in cycles, for the stall model. Two classes cover -/// every coupled op this backend emits: the FMA pipe and everything else. +/// The latency of a COUPLED producer, in cycles, for the stall model. /// /// THE VALUES ARE CALIBRATED AGAINST PTXAS OUTPUT, NOT DERIVED FROM A MANUAL. The FMA -/// anchor is ptxas's own dependent chains on sm_120: a back-to-back pair carries stall 4 -/// and a four-accumulator chain carries stall 1, and both give a latency of 5. The ALU -/// anchor is the integer half of the same streams: an IMAD whose result a LEA reads one -/// instruction later carries stall 5, and ptxas k2n's IMAD.WIDE whose store reads it two -/// instructions later carries stall 4, and both give a latency of 6. A first version of -/// this model took 4 for the ALU class, and every hardware test with an integer address -/// chain came back reading a stale register, which the RTX 5070 confirmed directly. +/// FADD and FFMA use ptxas's dependent chains on sm_120: a back-to-back pair carries +/// stall 4 and a four-accumulator chain carries stall 1, and both give a latency of 5. +/// FMUL is one cycle longer on the RTX 5070. A back-to-back pair with stall 4 returned +/// a stale first product under occupancy, while stall 5 made the exhaustive NVFP4 chain +/// exact. Keep that measured distinction: folding FMUL back into the FMA class silently +/// corrupts two ordered scale multiplications. The ALU anchor is the integer half of the +/// same ptxas streams: an IMAD whose result a LEA reads one instruction later carries +/// stall 5, and ptxas k2n's IMAD.WIDE whose store reads it two instructions later carries +/// stall 4, and both give a latency of 6. A first version of this model took 4 for the ALU +/// class, and every hardware test with an integer address chain came back reading a stale +/// register, which the RTX 5070 confirmed directly. +const coupled_fmul_latency: u32 = 6; const coupled_fma_latency: u32 = 5; const coupled_alu_latency: u32 = 6; fn coupledLatency(opcode: u32) u32 { return switch (opcode & 0x1ff) { - 0x020, 0x021, 0x023 => coupled_fma_latency, // FMUL, FADD, FFMA + 0x020 => coupled_fmul_latency, // FMUL + 0x021, 0x023 => coupled_fma_latency, // FADD, FFMA else => coupled_alu_latency, }; } @@ -3507,6 +3512,26 @@ test "a tight dependent pair stalls more than an independent pair" { try std.testing.expectEqual(@as(u32, 1), getField(indep[1], 105, 4)); } +test "a dependent FMUL pair uses the measured six cycle latency" { + var dep = [_]Inst{ + encode.fmul(8, 4, 5, .{}), + encode.fmul(9, 8, 6, .{}), + encode.exit(.{}), + }; + scheduleBlocks(&dep, &.{0}); + try std.testing.expectEqual(@as(u32, 5), getField(dep[0], 105, 4)); + try std.testing.expectEqual(@as(u32, 1), getField(dep[1], 105, 4)); + + var indep = [_]Inst{ + encode.fmul(8, 4, 5, .{}), + encode.fmul(9, 10, 11, .{}), + encode.exit(.{}), + }; + scheduleBlocks(&indep, &.{0}); + try std.testing.expectEqual(@as(u32, 1), getField(indep[0], 105, 4)); + try std.testing.expectEqual(@as(u32, 1), getField(indep[1], 105, 4)); +} + test "a variable-latency producer contributes no stall: the scoreboard covers it" { // The LDG is decoupled, so the stall model sets no ready time for its result and its // consumer stalls zero. The consumer still waits on the load's scoreboard, which the diff --git a/libs/vulcan-target/nvidia/tests/execute.zig b/libs/vulcan-target/nvidia/tests/execute.zig index 2c42337..b0ddc00 100644 --- a/libs/vulcan-target/nvidia/tests/execute.zig +++ b/libs/vulcan-target/nvidia/tests/execute.zig @@ -28,6 +28,7 @@ const ir = @import("vulcan-ir"); const gpu_abi = @import("vulcan-gpu"); const target = @import("vulcan-target"); const nvidia = @import("nvidia"); +const division_cases = target.f32_division_cases; const isel = target.nvidia.isel; const compute = nvidia.compute; @@ -111,6 +112,11 @@ const Harness = struct { return self.runner.alloc(.system, size); } + fn ensureCodeCapacity(self: *Harness, bytes: usize) !void { + if (self.runner.code.bytes.len >= bytes) return; + self.runner.code = try self.runner.alloc(.system_wc, @intCast(bytes)); + } + /// Allocate a zeroed buffer at the GPU virtual address `va`. The runner hands /// addresses out of a bump pointer, so moving the pointer first places the mapping. /// The read-barrier test needs this: a clobbered address must land in MAPPED memory, @@ -220,6 +226,1151 @@ fn bin(func: *Function, b: Block, ty: ir.types.Type, op: ir.function.BinOp, lhs: return func.appendInst(b, ty, .{ .arith = .{ .op = op, .lhs = lhs, .rhs = rhs } }); } +const low_float_guard: u8 = 0xa5; + +fn buildLowFloatBufferKernel(allocator: std.mem.Allocator, format: ir.low_float.Format, decode: bool) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const ptr_t = try func.types.ptrGlobal(); + const b = try func.appendBlock(); + const out = try func.appendBlockParam(b, ptr_t); + const input = try func.appendBlockParam(b, ptr_t); + const gid = try func.appendBlockParam(b, u32_t); + try gpu_abi.attrs.setBuiltin(&func, gid, .global_id_x); + + const payload_bytes: i64 = @intCast(format.payloadBits() / 8); + const input_stride: i64 = if (decode) payload_bytes else 4; + const output_stride: i64 = if (decode) 4 else payload_bytes; + const input_byte = try binImm(&func, b, u32_t, .mul, gid, input_stride); + const output_byte = try binImm(&func, b, u32_t, .mul, gid, output_stride); + const source = try func.appendInst(b, if (decode) payload_t else f32_t, .{ .load = .{ + .ptr = try ptrAddVal(&func, b, ptr_t, input, input_byte), + } }); + const conversion: ir.function.LowFloatConvert = .{ .format = format, .value = source }; + const result = try func.appendInst(b, if (decode) f32_t else payload_t, if (decode) + .{ .decode_low_float = conversion } + else + .{ .encode_low_float = conversion }); + try func.appendStore(b, result, try ptrAddVal(&func, b, ptr_t, out, output_byte)); + func.setTerminator(b, .{ .ret = ir.function.Ret.none() }); + return func; +} + +fn guardedBuffer(h: *Harness, active_offset: usize, active_bytes: usize) !compute.Buffer { + const buf = try h.alloc(@intCast(active_offset + active_bytes + 29)); + @memset(buf.bytes, low_float_guard); + return buf; +} + +fn expectGuard(buf: compute.Buffer, active_offset: usize, active_bytes: usize) !void { + const bytes: [*]const volatile u8 = @ptrCast(buf.bytes.ptr); + for (0..active_offset) |i| try testing.expectEqual(low_float_guard, bytes[i]); + for (active_offset + active_bytes..buf.bytes.len) |i| try testing.expectEqual(low_float_guard, bytes[i]); +} + +const MultiplicationPair = struct { + lhs: u32, + rhs: u32, +}; + +fn buildF32BinaryKernel(allocator: std.mem.Allocator, op: ir.function.BinOp) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const ptr_t = try func.types.ptrGlobal(); + const b = try func.appendBlock(); + const out = try func.appendBlockParam(b, ptr_t); + const lhs_base = try func.appendBlockParam(b, ptr_t); + const rhs_base = try func.appendBlockParam(b, ptr_t); + const gid = try func.appendBlockParam(b, u32_t); + try gpu_abi.attrs.setBuiltin(&func, gid, .global_id_x); + + const byte = try binImm(&func, b, u32_t, .mul, gid, 4); + const lhs = try func.appendInst(b, f32_t, .{ .load = .{ .ptr = try ptrAddVal(&func, b, ptr_t, lhs_base, byte) } }); + const rhs = try func.appendInst(b, f32_t, .{ .load = .{ .ptr = try ptrAddVal(&func, b, ptr_t, rhs_base, byte) } }); + const result = try func.appendInst(b, f32_t, .{ .arith = .{ .op = op, .lhs = lhs, .rhs = rhs } }); + const bits = try func.appendInst(b, u32_t, .{ .unary = .{ .op = .reinterpret, .value = result } }); + try func.appendStore(b, bits, try ptrAddVal(&func, b, ptr_t, out, byte)); + func.setTerminator(b, .{ .ret = ir.function.Ret.none() }); + return func; +} + +fn appendSignedMultiplicationPair(pairs: *std.ArrayList(MultiplicationPair), allocator: std.mem.Allocator, pair: MultiplicationPair) !void { + try pairs.append(allocator, pair); + try pairs.append(allocator, .{ .lhs = pair.lhs ^ 0x8000_0000, .rhs = pair.rhs }); + try pairs.append(allocator, .{ .lhs = pair.lhs, .rhs = pair.rhs ^ 0x8000_0000 }); + try pairs.append(allocator, .{ .lhs = pair.lhs ^ 0x8000_0000, .rhs = pair.rhs ^ 0x8000_0000 }); +} + +fn containsMultiplicationPair(pairs: []const MultiplicationPair, expected: MultiplicationPair) bool { + for (pairs) |pair| if (pair.lhs == expected.lhs and pair.rhs == expected.rhs) return true; + return false; +} + +fn appendNvFp4MultiplicationPairs(pairs: *std.ArrayList(MultiplicationPair), allocator: std.mem.Allocator) ![8]MultiplicationPair { + const dequantize = [_]struct { payload: u8, block: u8, global: u32, block_op: ir.nvfp4.ScaleApplication, global_op: ir.nvfp4.ScaleApplication }{ + .{ .payload = 0x03, .block = 0x03, .global = 0x3e00_0001, .block_op = .multiply, .global_op = .multiply }, + .{ .payload = 0x01, .block = 0x03, .global = 0x3e00_0001, .block_op = .multiply, .global_op = .divide }, + .{ .payload = 0x01, .block = 0x05, .global = 0x3e00_0001, .block_op = .divide, .global_op = .multiply }, + .{ .payload = 0x01, .block = 0x03, .global = 0x3e00_0001, .block_op = .divide, .global_op = .divide }, + }; + const quantize = [_]struct { value: u32, block: u8, global: u32, block_op: ir.nvfp4.ScaleApplication, global_op: ir.nvfp4.ScaleApplication }{ + .{ .value = 0x3e00_0522, .block = 0x23, .global = 0x3e54_d004, .block_op = .multiply, .global_op = .multiply }, + .{ .value = 0x3e00_154f, .block = 0x0f, .global = 0x3ed1_dd10, .block_op = .multiply, .global_op = .divide }, + .{ .value = 0x3e00_154f, .block = 0x47, .global = 0x3f20_1aa3, .block_op = .divide, .global_op = .multiply }, + .{ .value = 0x3e00_399b, .block = 0x55, .global = 0x3eeb_e49b, .block_op = .divide, .global_op = .divide }, + }; + + var required: [8]MultiplicationPair = undefined; + var count: usize = 0; + for (dequantize) |case| { + const decoded = ir.nvfp4.decodeE2M1(case.payload); + const block = try ir.low_float.decode(.f8_e4m3, case.block); + const global: f32 = @bitCast(case.global); + const local = switch (case.block_op) { + .multiply => value: { + required[count] = .{ .lhs = @bitCast(decoded), .rhs = @bitCast(block) }; + count += 1; + break :value decoded * block; + }, + .divide => decoded / block, + }; + if (case.global_op == .multiply) { + required[count] = .{ .lhs = @bitCast(local), .rhs = @bitCast(global) }; + count += 1; + } + } + for (quantize) |case| { + const value: f32 = @bitCast(case.value); + const block = try ir.low_float.decode(.f8_e4m3, case.block); + const global: f32 = @bitCast(case.global); + const global_unscaled = switch (case.global_op) { + .multiply => value / global, + .divide => product: { + required[count] = .{ .lhs = case.value, .rhs = case.global }; + count += 1; + break :product value * global; + }, + }; + if (case.block_op == .divide) { + required[count] = .{ .lhs = @bitCast(global_unscaled), .rhs = @bitCast(block) }; + count += 1; + } + } + try testing.expectEqual(required.len, count); + for (required) |pair| { + try testing.expect(std.math.isFinite(@as(f32, @bitCast(pair.lhs)))); + try testing.expect(std.math.isFinite(@as(f32, @bitCast(pair.rhs)))); + try pairs.append(allocator, pair); + } + return required; +} + +test "live: NVIDIA f32 multiplication preserves exact NVFP4 scale semantics" { + const allocator = testing.allocator; + var pairs: std.ArrayList(MultiplicationPair) = .empty; + defer pairs.deinit(allocator); + + const boundaries = [_]MultiplicationPair{ + .{ .lhs = 0x0000_0000, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x3fc0_0000, .rhs = 0x0000_0000 }, + .{ .lhs = 0x0000_0001, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x007f_ffff, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x0080_0000, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x0080_0001, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x0080_0000, .rhs = 0x3400_0000 }, + .{ .lhs = 0x0080_0000, .rhs = 0x337f_ffff }, + .{ .lhs = 0x0080_0000, .rhs = 0x3380_0000 }, + .{ .lhs = 0x0080_0000, .rhs = 0x3380_0001 }, + .{ .lhs = 0x007f_ffff, .rhs = 0x3f7f_ffff }, + .{ .lhs = 0x007f_ffff, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x007f_ffff, .rhs = 0x3f80_0001 }, + .{ .lhs = 0x7f7f_ffff, .rhs = 0x3f7f_ffff }, + .{ .lhs = 0x7f7f_ffff, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x7f7f_ffff, .rhs = 0x3f80_0001 }, + .{ .lhs = 0x7f00_0000, .rhs = 0x3fff_fffe }, + .{ .lhs = 0x7f00_0000, .rhs = 0x3fff_ffff }, + .{ .lhs = 0x7f00_0000, .rhs = 0x4000_0000 }, + .{ .lhs = 0x7f00_0000, .rhs = 0x4000_0001 }, + .{ .lhs = 0x7f80_0000, .rhs = 0x0000_0000 }, + .{ .lhs = 0x7f80_0000, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x7f80_0001, .rhs = 0x3f80_0000 }, + .{ .lhs = 0x7fc0_1234, .rhs = 0x3f80_0000 }, + }; + for (boundaries) |pair| try appendSignedMultiplicationPair(&pairs, allocator, pair); + const order_pairs = try appendNvFp4MultiplicationPairs(&pairs, allocator); + + for (boundaries) |pair| { + try testing.expect(containsMultiplicationPair(pairs.items, pair)); + try testing.expect(containsMultiplicationPair(pairs.items, .{ .lhs = pair.lhs ^ 0x8000_0000, .rhs = pair.rhs })); + try testing.expect(containsMultiplicationPair(pairs.items, .{ .lhs = pair.lhs, .rhs = pair.rhs ^ 0x8000_0000 })); + try testing.expect(containsMultiplicationPair(pairs.items, .{ .lhs = pair.lhs ^ 0x8000_0000, .rhs = pair.rhs ^ 0x8000_0000 })); + } + for (order_pairs) |pair| try testing.expect(containsMultiplicationPair(pairs.items, pair)); + try testing.expectEqual(boundaries.len * 4 + order_pairs.len, pairs.items.len); + + var func = try buildF32BinaryKernel(allocator, .mul); + defer func.deinit(); + var h = try Harness.open(); + defer h.deinit(); + var launch = try h.compile(&func, runner_abi); + defer launch.deinit(); + + const lhs_offset: usize = 12; + const rhs_offset: usize = 20; + const output_offset: usize = 28; + const active_bytes = pairs.items.len * @sizeOf(u32); + var lhs = try guardedBuffer(&h, lhs_offset, active_bytes); + var rhs = try guardedBuffer(&h, rhs_offset, active_bytes); + var output = try guardedBuffer(&h, output_offset, active_bytes); + for (pairs.items, 0..) |pair, index| { + std.mem.writeInt(u32, lhs.bytes[lhs_offset + index * 4 ..][0..4], pair.lhs, .little); + std.mem.writeInt(u32, rhs.bytes[rhs_offset + index * 4 ..][0..4], pair.rhs, .little); + } + + launch.setPtr(launch.kernel.launch.params[0].offset, output.va + output_offset); + launch.setPtr(launch.kernel.launch.params[1].offset, lhs.va + lhs_offset); + launch.setPtr(launch.kernel.launch.params[2].offset, rhs.va + rhs_offset); + try launch.run(.{ @intCast(pairs.items.len), 1, 1 }); + + for (pairs.items, 0..) |pair, index| { + const expected = @as(f32, @bitCast(pair.lhs)) * @as(f32, @bitCast(pair.rhs)); + const actual_bits = output.read(u32, output_offset / 4 + index); + const actual: f32 = @bitCast(actual_bits); + if (std.math.isNan(expected)) { + try testing.expect(std.math.isNan(actual)); + } else { + try testing.expectEqual(@as(u32, @bitCast(expected)), actual_bits); + } + } + try expectGuard(lhs, lhs_offset, active_bytes); + try expectGuard(rhs, rhs_offset, active_bytes); + try expectGuard(output, output_offset, active_bytes); +} + +test "live: NVIDIA exact f32 division matches the shared adversarial corpus" { + const allocator = testing.allocator; + var corpus = try division_cases.build(allocator); + defer corpus.deinit(); + + var func = try buildF32BinaryKernel(allocator, .div); + defer func.deinit(); + var h = try Harness.open(); + defer h.deinit(); + var launch = try h.compile(&func, runner_abi); + defer launch.deinit(); + var unfused_launch = try h.compileOpts(&func, runner_abi, .{ .contract_fma = false }); + defer unfused_launch.deinit(); + try h.ensureCodeCapacity(launch.kernel.code.len * @sizeOf(u32)); + try h.ensureCodeCapacity(unfused_launch.kernel.code.len * @sizeOf(u32)); + + const numerator_offset: usize = 12; + const denominator_offset: usize = 20; + const output_offset: usize = 28; + const active_bytes = corpus.pairs.len * @sizeOf(u32); + var numerators = try guardedBuffer(&h, numerator_offset, active_bytes); + var denominators = try guardedBuffer(&h, denominator_offset, active_bytes); + var output = try guardedBuffer(&h, output_offset, active_bytes); + var unfused_output = try guardedBuffer(&h, output_offset, active_bytes); + for (corpus.pairs, 0..) |pair, index| { + std.mem.writeInt(u32, numerators.bytes[numerator_offset + index * 4 ..][0..4], pair.numerator, .little); + std.mem.writeInt(u32, denominators.bytes[denominator_offset + index * 4 ..][0..4], pair.denominator, .little); + } + + launch.setPtr(launch.kernel.launch.params[0].offset, output.va + output_offset); + launch.setPtr(launch.kernel.launch.params[1].offset, numerators.va + numerator_offset); + launch.setPtr(launch.kernel.launch.params[2].offset, denominators.va + denominator_offset); + try launch.run(.{ @intCast(corpus.pairs.len), 1, 1 }); + unfused_launch.setPtr(unfused_launch.kernel.launch.params[0].offset, unfused_output.va + output_offset); + unfused_launch.setPtr(unfused_launch.kernel.launch.params[1].offset, numerators.va + numerator_offset); + unfused_launch.setPtr(unfused_launch.kernel.launch.params[2].offset, denominators.va + denominator_offset); + try unfused_launch.run(.{ @intCast(corpus.pairs.len), 1, 1 }); + + for (corpus.pairs, 0..) |pair, index| { + const expected = @as(f32, @bitCast(pair.numerator)) / @as(f32, @bitCast(pair.denominator)); + const actual_bits = output.read(u32, output_offset / 4 + index); + const unfused_bits = unfused_output.read(u32, output_offset / 4 + index); + try testing.expectEqual(actual_bits, unfused_bits); + const actual: f32 = @bitCast(actual_bits); + if (std.math.isNan(expected)) { + try testing.expect(std.math.isNan(actual)); + } else { + const Observation = struct { numerator: u32, denominator: u32, result: u32 }; + try testing.expectEqual( + Observation{ .numerator = pair.numerator, .denominator = pair.denominator, .result = @bitCast(expected) }, + Observation{ .numerator = pair.numerator, .denominator = pair.denominator, .result = actual_bits }, + ); + } + } + try expectGuard(numerators, numerator_offset, active_bytes); + try expectGuard(denominators, denominator_offset, active_bytes); + try expectGuard(output, output_offset, active_bytes); + try expectGuard(unfused_output, output_offset, active_bytes); +} + +fn buildNvFp4BufferKernel( + allocator: std.mem.Allocator, + dequantize: bool, + block_application: ir.nvfp4.ScaleApplication, + global_application: ir.nvfp4.ScaleApplication, +) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const ptr_t = try func.types.ptrGlobal(); + const b = try func.appendBlock(); + const out = try func.appendBlockParam(b, ptr_t); + const values = try func.appendBlockParam(b, ptr_t); + const blocks = try func.appendBlockParam(b, ptr_t); + const globals = try func.appendBlockParam(b, ptr_t); + const gid = try func.appendBlockParam(b, u32_t); + try gpu_abi.attrs.setBuiltin(&func, gid, .global_id_x); + + const byte = try binImm(&func, b, u32_t, .mul, gid, 4); + const payload_byte = if (dequantize) gid else byte; + const value = try func.appendInst(b, if (dequantize) u8_t else f32_t, .{ .load = .{ + .ptr = try ptrAddVal(&func, b, ptr_t, values, payload_byte), + } }); + const block_scale = try func.appendInst(b, u8_t, .{ .load = .{ + .ptr = try ptrAddVal(&func, b, ptr_t, blocks, gid), + } }); + const global_scale = try func.appendInst(b, f32_t, .{ .load = .{ + .ptr = try ptrAddVal(&func, b, ptr_t, globals, byte), + } }); + const conversion: ir.function.NvFp4Convert = .{ + .value = value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = block_application, + .global_application = global_application, + }; + const converted = try func.appendInst(b, if (dequantize) f32_t else u8_t, if (dequantize) + .{ .dequantize_nvfp4 = conversion } + else + .{ .quantize_nvfp4 = conversion }); + const result = if (dequantize) + try func.appendInst(b, u32_t, .{ .unary = .{ .op = .reinterpret, .value = converted } }) + else + converted; + const output_byte = if (dequantize) try binImm(&func, b, u32_t, .mul, gid, 4) else gid; + try func.appendStore(b, result, try ptrAddVal(&func, b, ptr_t, out, output_byte)); + func.setTerminator(b, .{ .ret = ir.function.Ret.none() }); + return func; +} + +fn buildReorderedNvFp4BufferKernel( + allocator: std.mem.Allocator, + dequantize: bool, + block_application: ir.nvfp4.ScaleApplication, + global_application: ir.nvfp4.ScaleApplication, +) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const ptr_t = try func.types.ptrGlobal(); + const b = try func.appendBlock(); + const out = try func.appendBlockParam(b, ptr_t); + const values = try func.appendBlockParam(b, ptr_t); + const blocks = try func.appendBlockParam(b, ptr_t); + const globals = try func.appendBlockParam(b, ptr_t); + const gid = try func.appendBlockParam(b, u32_t); + try gpu_abi.attrs.setBuiltin(&func, gid, .global_id_x); + + const byte = try binImm(&func, b, u32_t, .mul, gid, 4); + const payload_byte = if (dequantize) gid else byte; + const value = try func.appendInst(b, if (dequantize) u8_t else f32_t, .{ .load = .{ + .ptr = try ptrAddVal(&func, b, ptr_t, values, payload_byte), + } }); + const block_payload = try func.appendInst(b, u8_t, .{ .load = .{ + .ptr = try ptrAddVal(&func, b, ptr_t, blocks, gid), + } }); + const global_scale = try func.appendInst(b, f32_t, .{ .load = .{ + .ptr = try ptrAddVal(&func, b, ptr_t, globals, byte), + } }); + const block_scale = try func.appendInst(b, f32_t, .{ .decode_low_float = .{ .format = .f8_e4m3, .value = block_payload } }); + const unit_payload = try func.appendInst(b, u8_t, .{ .iconst = 0x38 }); + const unit_scale = try func.appendInst(b, f32_t, .{ .fconst = 1.0 }); + const result = if (dequantize) result: { + const global_first = try func.appendInst(b, f32_t, .{ .dequantize_nvfp4 = .{ + .value = value, + .block_scale = unit_payload, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = global_application, + } }); + const reordered = try bin(&func, b, f32_t, if (block_application == .multiply) .mul else .div, global_first, block_scale); + break :result try func.appendInst(b, u32_t, .{ .unary = .{ .op = .reinterpret, .value = reordered } }); + } else result: { + const block_first = try bin(&func, b, f32_t, if (block_application == .multiply) .div else .mul, value, block_scale); + const global_second = try bin(&func, b, f32_t, if (global_application == .multiply) .div else .mul, block_first, global_scale); + break :result try func.appendInst(b, u8_t, .{ .quantize_nvfp4 = .{ + .value = global_second, + .block_scale = unit_payload, + .global_scale = unit_scale, + .block_application = .divide, + .global_application = .divide, + } }); + }; + const output_byte = if (dequantize) try binImm(&func, b, u32_t, .mul, gid, 4) else gid; + try func.appendStore(b, result, try ptrAddVal(&func, b, ptr_t, out, output_byte)); + func.setTerminator(b, .{ .ret = ir.function.Ret.none() }); + return func; +} + +fn runNvFp4BufferControl( + allocator: std.mem.Allocator, + h: *Harness, + dequantize: bool, + reordered: bool, + value_bits: u32, + block_payload: u8, + global_bits: u32, + block_application: ir.nvfp4.ScaleApplication, + global_application: ir.nvfp4.ScaleApplication, +) !u32 { + var func = if (reordered) + try buildReorderedNvFp4BufferKernel(allocator, dequantize, block_application, global_application) + else + try buildNvFp4BufferKernel(allocator, dequantize, block_application, global_application); + defer func.deinit(); + var launch = try h.compile(&func, runner_abi); + defer launch.deinit(); + try h.ensureCodeCapacity(launch.kernel.code.len * @sizeOf(u32)); + + const value_offset: usize = if (dequantize) 3 else 12; + const block_offset: usize = 5; + const global_offset: usize = 12; + const output_offset: usize = 20; + const value_bytes: usize = if (dequantize) 1 else 4; + const output_bytes: usize = if (dequantize) 4 else 1; + var value = try guardedBuffer(h, value_offset, value_bytes); + var block = try guardedBuffer(h, block_offset, 1); + var global = try guardedBuffer(h, global_offset, 4); + var output = try guardedBuffer(h, output_offset, output_bytes); + if (dequantize) + value.bytes[value_offset] = @truncate(value_bits) + else + std.mem.writeInt(u32, value.bytes[value_offset..][0..4], value_bits, .little); + block.bytes[block_offset] = block_payload; + std.mem.writeInt(u32, global.bytes[global_offset..][0..4], global_bits, .little); + launch.setPtr(launch.kernel.launch.params[0].offset, output.va + output_offset); + launch.setPtr(launch.kernel.launch.params[1].offset, value.va + value_offset); + launch.setPtr(launch.kernel.launch.params[2].offset, block.va + block_offset); + launch.setPtr(launch.kernel.launch.params[3].offset, global.va + global_offset); + try launch.run(.{ 1, 1, 1 }); + + const result: u32 = if (dequantize) output.read(u32, output_offset / 4) else output.read(u8, output_offset); + try expectGuard(value, value_offset, value_bytes); + try expectGuard(block, block_offset, 1); + try expectGuard(global, global_offset, 4); + try expectGuard(output, output_offset, output_bytes); + return result; +} + +fn nvFp4QuantizeValues(allocator: std.mem.Allocator) ![]u32 { + var values: std.ArrayList(u32) = .empty; + errdefer values.deinit(allocator); + var seen: std.AutoHashMapUnmanaged(u32, void) = .empty; + defer seen.deinit(allocator); + for (0..16) |payload| try appendUnique(@bitCast(ir.nvfp4.decodeE2M1(@intCast(payload))), &seen, &values, allocator); + const midpoints = [_]u32{ 0x3e80_0000, 0x3f40_0000, 0x3fa0_0000, 0x3fe0_0000, 0x4020_0000, 0x4060_0000, 0x40a0_0000 }; + for (midpoints) |midpoint| inline for (.{ @as(u32, 0), @as(u32, 0x8000_0000) }) |sign| { + try appendTriplet(midpoint | sign, &seen, &values, allocator); + }; + const specials = [_]u32{ + 0x0000_0000, 0x8000_0000, 0x0000_0001, 0x8000_0001, + 0x007f_ffff, 0x807f_ffff, 0x0080_0000, 0x8080_0000, + 0x7f7f_ffff, 0xff7f_ffff, 0x7f80_0000, 0xff80_0000, + 0x7f80_0001, 0xff80_0001, 0x7fc0_1234, 0xffc0_1234, + }; + for (specials) |bits| try appendUnique(bits, &seen, &values, allocator); + + for (0..16) |payload| try testing.expect(seen.contains(@bitCast(ir.nvfp4.decodeE2M1(@intCast(payload))))); + for (midpoints) |midpoint| inline for (.{ @as(u32, 0), @as(u32, 0x8000_0000) }) |sign| { + try expectCorpusTriplet(&seen, midpoint | sign); + }; + for (specials) |bits| try testing.expect(seen.contains(bits)); + return values.toOwnedSlice(allocator); +} + +test "live: NVIDIA NVFP4 conversion matches exhaustive scales and quantization boundaries" { + const allocator = testing.allocator; + const policies = [_]ir.nvfp4.ScaleApplication{ .multiply, .divide }; + const dequant_globals = [_]u32{ 0x0000_0000, 0x8000_0000, 0x3f00_0000, 0xbf80_0000, 0x7f80_0000, 0xff80_0000, 0x7fc0_1234 }; + const quant_blocks = [_]u8{ 0x00, 0xb8, 0x7e, 0x7f }; + const quant_globals = [_]u32{ 0x0000_0000, 0x8000_0000, 0xbf80_0000, 0x3f00_0000, 0x7f80_0000, 0xff80_0000, 0x7fc0_1234 }; + const quant_values = try nvFp4QuantizeValues(allocator); + defer allocator.free(quant_values); + try testing.expectEqual(@as(usize, 7), dequant_globals.len); + try testing.expectEqual(@as(usize, 4), quant_blocks.len); + try testing.expectEqual(@as(usize, 7), quant_globals.len); + + var h = try Harness.open(); + defer h.deinit(); + inline for (policies) |block_application| inline for (policies) |global_application| { + const dequant_count = 256 * 256 * dequant_globals.len; + const carrier_offset: usize = 3; + const block_offset: usize = 5; + const global_offset: usize = 12; + const dequant_output_offset: usize = 20; + var carriers = try guardedBuffer(&h, carrier_offset, dequant_count); + var blocks = try guardedBuffer(&h, block_offset, dequant_count); + var globals = try guardedBuffer(&h, global_offset, dequant_count * 4); + var dequant_output = try guardedBuffer(&h, dequant_output_offset, dequant_count * 4); + var at: usize = 0; + for (dequant_globals) |global| for (0..256) |block| for (0..256) |carrier| { + carriers.bytes[carrier_offset + at] = @intCast(carrier); + blocks.bytes[block_offset + at] = @intCast(block); + std.mem.writeInt(u32, globals.bytes[global_offset + at * 4 ..][0..4], global, .little); + at += 1; + }; + try testing.expectEqual(dequant_count, at); + + var dequant_func = try buildNvFp4BufferKernel(allocator, true, block_application, global_application); + defer dequant_func.deinit(); + var dequant_launch = try h.compile(&dequant_func, runner_abi); + defer dequant_launch.deinit(); + try h.ensureCodeCapacity(dequant_launch.kernel.code.len * @sizeOf(u32)); + const batch_count: usize = 32 * 1024; + var batch_start: usize = 0; + while (batch_start < dequant_count) : (batch_start += batch_count) { + const count = @min(batch_count, dequant_count - batch_start); + dequant_launch.setPtr(dequant_launch.kernel.launch.params[0].offset, dequant_output.va + dequant_output_offset + batch_start * 4); + dequant_launch.setPtr(dequant_launch.kernel.launch.params[1].offset, carriers.va + carrier_offset + batch_start); + dequant_launch.setPtr(dequant_launch.kernel.launch.params[2].offset, blocks.va + block_offset + batch_start); + dequant_launch.setPtr(dequant_launch.kernel.launch.params[3].offset, globals.va + global_offset + batch_start * 4); + try dequant_launch.run(.{ @intCast(count), 1, 1 }); + } + + at = 0; + for (dequant_globals) |global| for (0..256) |block| for (0..256) |carrier| { + const expected = ir.nvfp4.dequantize(@intCast(carrier), @intCast(block), @bitCast(global), block_application, global_application); + const actual_bits = dequant_output.read(u32, dequant_output_offset / 4 + at); + const actual: f32 = @bitCast(actual_bits); + if (std.math.isNan(expected)) { + try testing.expect(std.math.isNan(actual)); + } else { + try testing.expectEqual(@as(u32, @bitCast(expected)), actual_bits); + } + at += 1; + }; + try expectGuard(carriers, carrier_offset, dequant_count); + try expectGuard(blocks, block_offset, dequant_count); + try expectGuard(globals, global_offset, dequant_count * 4); + try expectGuard(dequant_output, dequant_output_offset, dequant_count * 4); + + const quant_count = quant_values.len * quant_blocks.len * quant_globals.len; + const value_offset: usize = 12; + const quant_block_offset: usize = 5; + const quant_global_offset: usize = 20; + const quant_output_offset: usize = 3; + var values = try guardedBuffer(&h, value_offset, quant_count * 4); + var quant_block_buffer = try guardedBuffer(&h, quant_block_offset, quant_count); + var quant_global_buffer = try guardedBuffer(&h, quant_global_offset, quant_count * 4); + var quant_output = try guardedBuffer(&h, quant_output_offset, quant_count); + at = 0; + for (quant_globals) |global| for (quant_blocks) |block| for (quant_values) |value| { + std.mem.writeInt(u32, values.bytes[value_offset + at * 4 ..][0..4], value, .little); + quant_block_buffer.bytes[quant_block_offset + at] = block; + std.mem.writeInt(u32, quant_global_buffer.bytes[quant_global_offset + at * 4 ..][0..4], global, .little); + at += 1; + }; + try testing.expectEqual(quant_count, at); + + var quant_func = try buildNvFp4BufferKernel(allocator, false, block_application, global_application); + defer quant_func.deinit(); + var quant_launch = try h.compile(&quant_func, runner_abi); + defer quant_launch.deinit(); + try h.ensureCodeCapacity(quant_launch.kernel.code.len * @sizeOf(u32)); + quant_launch.setPtr(quant_launch.kernel.launch.params[0].offset, quant_output.va + quant_output_offset); + quant_launch.setPtr(quant_launch.kernel.launch.params[1].offset, values.va + value_offset); + quant_launch.setPtr(quant_launch.kernel.launch.params[2].offset, quant_block_buffer.va + quant_block_offset); + quant_launch.setPtr(quant_launch.kernel.launch.params[3].offset, quant_global_buffer.va + quant_global_offset); + try quant_launch.run(.{ @intCast(quant_count), 1, 1 }); + + at = 0; + for (quant_globals) |global| for (quant_blocks) |block| for (quant_values) |value| { + const expected = ir.nvfp4.quantize(@bitCast(value), block, @bitCast(global), block_application, global_application); + try testing.expectEqual(expected, quant_output.read(u8, quant_output_offset + at)); + at += 1; + }; + try expectGuard(values, value_offset, quant_count * 4); + try expectGuard(quant_block_buffer, quant_block_offset, quant_count); + try expectGuard(quant_global_buffer, quant_global_offset, quant_count * 4); + try expectGuard(quant_output, quant_output_offset, quant_count); + }; +} + +fn applyNvFp4Scale(value: f32, scale: f32, application: ir.nvfp4.ScaleApplication) f32 { + return if (application == .multiply) value * scale else value / scale; +} + +fn undoNvFp4Scale(value: f32, scale: f32, application: ir.nvfp4.ScaleApplication) f32 { + return if (application == .multiply) value / scale else value * scale; +} + +fn runF32BinaryControl(allocator: std.mem.Allocator, h: *Harness, lhs_bits: u32, rhs_bits: u32, op: ir.function.BinOp) !u32 { + var func = try buildF32BinaryKernel(allocator, op); + defer func.deinit(); + var launch = try h.compile(&func, runner_abi); + defer launch.deinit(); + try h.ensureCodeCapacity(launch.kernel.code.len * @sizeOf(u32)); + + const lhs_offset: usize = 12; + const rhs_offset: usize = 20; + const output_offset: usize = 28; + var lhs = try guardedBuffer(h, lhs_offset, 4); + var rhs = try guardedBuffer(h, rhs_offset, 4); + var output = try guardedBuffer(h, output_offset, 4); + std.mem.writeInt(u32, lhs.bytes[lhs_offset..][0..4], lhs_bits, .little); + std.mem.writeInt(u32, rhs.bytes[rhs_offset..][0..4], rhs_bits, .little); + launch.setPtr(launch.kernel.launch.params[0].offset, output.va + output_offset); + launch.setPtr(launch.kernel.launch.params[1].offset, lhs.va + lhs_offset); + launch.setPtr(launch.kernel.launch.params[2].offset, rhs.va + rhs_offset); + try launch.run(.{ 1, 1, 1 }); + const result = output.read(u32, output_offset / 4); + try expectGuard(lhs, lhs_offset, 4); + try expectGuard(rhs, rhs_offset, 4); + try expectGuard(output, output_offset, 4); + return result; +} + +fn runNvFp4UnitQuantizeControl(allocator: std.mem.Allocator, h: *Harness, value_bits: u32) !u8 { + var func = try buildNvFp4BufferKernel(allocator, false, .divide, .divide); + defer func.deinit(); + var launch = try h.compile(&func, runner_abi); + defer launch.deinit(); + try h.ensureCodeCapacity(launch.kernel.code.len * @sizeOf(u32)); + + const value_offset: usize = 12; + const block_offset: usize = 5; + const global_offset: usize = 20; + const output_offset: usize = 3; + var value = try guardedBuffer(h, value_offset, 4); + var block = try guardedBuffer(h, block_offset, 1); + var global = try guardedBuffer(h, global_offset, 4); + var output = try guardedBuffer(h, output_offset, 1); + std.mem.writeInt(u32, value.bytes[value_offset..][0..4], value_bits, .little); + block.bytes[block_offset] = 0x38; + std.mem.writeInt(u32, global.bytes[global_offset..][0..4], 0x3f80_0000, .little); + launch.setPtr(launch.kernel.launch.params[0].offset, output.va + output_offset); + launch.setPtr(launch.kernel.launch.params[1].offset, value.va + value_offset); + launch.setPtr(launch.kernel.launch.params[2].offset, block.va + block_offset); + launch.setPtr(launch.kernel.launch.params[3].offset, global.va + global_offset); + try launch.run(.{ 1, 1, 1 }); + const result = output.read(u8, output_offset); + try expectGuard(value, value_offset, 4); + try expectGuard(block, block_offset, 1); + try expectGuard(global, global_offset, 4); + try expectGuard(output, output_offset, 1); + return result; +} + +test "live: NVIDIA NVFP4 preserves canonical scale order against reordered controls" { + const allocator = testing.allocator; + const policies = [_]ir.nvfp4.ScaleApplication{ .multiply, .divide }; + const dequantize_cases = [_]struct { payload: u8, block_scale: u8, global_bits: u32, block: ir.nvfp4.ScaleApplication, global: ir.nvfp4.ScaleApplication, expected: u32, reordered: u32 }{ + .{ .payload = 0x03, .block_scale = 0x03, .global_bits = 0x3e00_0001, .block = .multiply, .global = .multiply, .expected = 0x3a90_0001, .reordered = 0x3a90_0002 }, + .{ .payload = 0x01, .block_scale = 0x03, .global_bits = 0x3e00_0001, .block = .multiply, .global = .divide, .expected = 0x3cbf_ffff, .reordered = 0x3cbf_fffe }, + .{ .payload = 0x01, .block_scale = 0x05, .global_bits = 0x3e00_0001, .block = .divide, .global = .multiply, .expected = 0x40cc_cccf, .reordered = 0x40cc_ccce }, + .{ .payload = 0x01, .block_scale = 0x03, .global_bits = 0x3e00_0001, .block = .divide, .global = .divide, .expected = 0x442a_aaaa, .reordered = 0x442a_aaa9 }, + }; + const quantize_cases = [_]struct { value_bits: u32, block_scale: u8, global_bits: u32, block: ir.nvfp4.ScaleApplication, global: ir.nvfp4.ScaleApplication, expected: u8, reordered: u8 }{ + .{ .value_bits = 0x3e00_0522, .block_scale = 0x23, .global_bits = 0x3e54_d004, .block = .multiply, .global = .multiply, .expected = 0x06, .reordered = 0x05 }, + .{ .value_bits = 0x3e00_154f, .block_scale = 0x0f, .global_bits = 0x3ed1_dd10, .block = .multiply, .global = .divide, .expected = 0x04, .reordered = 0x03 }, + .{ .value_bits = 0x3e00_154f, .block_scale = 0x47, .global_bits = 0x3f20_1aa3, .block = .divide, .global = .multiply, .expected = 0x01, .reordered = 0x02 }, + .{ .value_bits = 0x3e00_399b, .block_scale = 0x55, .global_bits = 0x3eeb_e49b, .block = .divide, .global = .divide, .expected = 0x01, .reordered = 0x02 }, + }; + + for (dequantize_cases) |case| { + const block_scale = try ir.low_float.decode(.f8_e4m3, case.block_scale); + const global_scale: f32 = @bitCast(case.global_bits); + const canonical = ir.nvfp4.dequantize(case.payload, case.block_scale, global_scale, case.block, case.global); + const reordered = applyNvFp4Scale(applyNvFp4Scale(ir.nvfp4.decodeE2M1(case.payload), global_scale, case.global), block_scale, case.block); + try testing.expectEqual(case.expected, @as(u32, @bitCast(canonical))); + try testing.expectEqual(case.reordered, @as(u32, @bitCast(reordered))); + try testing.expect(case.expected != case.reordered); + } + for (quantize_cases) |case| { + const value: f32 = @bitCast(case.value_bits); + const block_scale = try ir.low_float.decode(.f8_e4m3, case.block_scale); + const global_scale: f32 = @bitCast(case.global_bits); + const canonical = ir.nvfp4.quantize(value, case.block_scale, global_scale, case.block, case.global); + const reordered = ir.nvfp4.encodeE2M1(undoNvFp4Scale(undoNvFp4Scale(value, block_scale, case.block), global_scale, case.global)); + try testing.expectEqual(case.expected, canonical); + try testing.expectEqual(case.reordered, reordered); + try testing.expect(case.expected != case.reordered); + } + + var h = try Harness.open(); + defer h.deinit(); + inline for (policies, 0..) |block_application, block_index| inline for (policies, 0..) |global_application, global_index| { + const case_index = block_index * policies.len + global_index; + const dequantize_case = dequantize_cases[case_index]; + const quantize_case = quantize_cases[case_index]; + try testing.expectEqual(block_application, dequantize_case.block); + try testing.expectEqual(global_application, dequantize_case.global); + try testing.expectEqual(block_application, quantize_case.block); + try testing.expectEqual(global_application, quantize_case.global); + + const canonical_dequantize = try runNvFp4BufferControl(allocator, &h, true, false, dequantize_case.payload, dequantize_case.block_scale, dequantize_case.global_bits, block_application, global_application); + try testing.expectEqual(dequantize_case.expected, canonical_dequantize); + const reordered_dequantize = try runNvFp4BufferControl(allocator, &h, true, true, dequantize_case.payload, dequantize_case.block_scale, dequantize_case.global_bits, block_application, global_application); + try testing.expectEqual(dequantize_case.reordered, reordered_dequantize); + + const canonical_quantize = try runNvFp4BufferControl(allocator, &h, false, false, quantize_case.value_bits, quantize_case.block_scale, quantize_case.global_bits, block_application, global_application); + try testing.expectEqual(@as(u32, quantize_case.expected), canonical_quantize); + const decoded_block_bits: u32 = @bitCast(try ir.low_float.decode(.f8_e4m3, quantize_case.block_scale)); + + const value: f32 = @bitCast(quantize_case.value_bits); + const block_scale: f32 = @bitCast(decoded_block_bits); + const global_scale: f32 = @bitCast(quantize_case.global_bits); + const expected_block_first = undoNvFp4Scale(value, block_scale, block_application); + const expected_global_second = undoNvFp4Scale(expected_block_first, global_scale, global_application); + const block_first_bits = try runF32BinaryControl(allocator, &h, quantize_case.value_bits, decoded_block_bits, if (block_application == .multiply) .div else .mul); + try testing.expectEqual(@as(u32, @bitCast(expected_block_first)), block_first_bits); + const global_second_bits = try runF32BinaryControl(allocator, &h, block_first_bits, quantize_case.global_bits, if (global_application == .multiply) .div else .mul); + try testing.expectEqual(@as(u32, @bitCast(expected_global_second)), global_second_bits); + const reordered_quantize_output = try runNvFp4UnitQuantizeControl(allocator, &h, global_second_bits); + + const expected_global_first = undoNvFp4Scale(value, global_scale, global_application); + const expected_local_second = undoNvFp4Scale(expected_global_first, block_scale, block_application); + const global_first_bits = try runF32BinaryControl(allocator, &h, quantize_case.value_bits, quantize_case.global_bits, if (global_application == .multiply) .div else .mul); + try testing.expectEqual(@as(u32, @bitCast(expected_global_first)), global_first_bits); + const local_second_bits = try runF32BinaryControl(allocator, &h, global_first_bits, decoded_block_bits, if (block_application == .multiply) .div else .mul); + try testing.expectEqual(@as(u32, @bitCast(expected_local_second)), local_second_bits); + const canonical_unit_output = try runNvFp4UnitQuantizeControl(allocator, &h, local_second_bits); + try testing.expectEqual(quantize_case.expected, canonical_unit_output); + try testing.expectEqual(quantize_case.reordered, reordered_quantize_output); + }; +} + +fn buildNvFp4PackedMemoryKernel(allocator: std.mem.Allocator, element_count: usize) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const ptr_t = try func.types.ptrGlobal(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const b = try func.appendBlock(); + const packed_input = try func.appendBlockParam(b, ptr_t); + const scales = try func.appendBlockParam(b, ptr_t); + const globals = try func.appendBlockParam(b, ptr_t); + const decoded_output = try func.appendBlockParam(b, ptr_t); + const float_input = try func.appendBlockParam(b, ptr_t); + const packed_output = try func.appendBlockParam(b, ptr_t); + const global_scale = try func.appendInst(b, f32_t, .{ .load = .{ .ptr = globals } }); + for (0..element_count) |index| { + const packed_at = if (index < 2) packed_input else try func.appendArithImm(b, ptr_t, .add, packed_input, @intCast(index / 2)); + const scale_at = if (index < 16) scales else try func.appendArithImm(b, ptr_t, .add, scales, @intCast(index / 16)); + const packed_byte = try func.appendInst(b, u8_t, .{ .load = .{ .ptr = packed_at } }); + const packed_wide = try func.appendInst(b, u32_t, .{ .convert = .{ .value = packed_byte } }); + const positioned = if (index & 1 == 0) packed_wide else try func.appendArithImm(b, u32_t, .shr, packed_wide, 4); + const nibble_wide = try func.appendArithImm(b, u32_t, .bit_and, positioned, 0x0f); + const nibble = try func.appendInst(b, u8_t, .{ .convert = .{ .value = nibble_wide } }); + const block_scale = try func.appendInst(b, u8_t, .{ .load = .{ .ptr = scale_at } }); + const decoded = try func.appendInst(b, f32_t, .{ .dequantize_nvfp4 = .{ + .value = nibble, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + } }); + const decoded_at = if (index == 0) decoded_output else try func.appendArithImm(b, ptr_t, .add, decoded_output, @intCast(index * 4)); + try func.appendStore(b, decoded, decoded_at); + + const float_at = if (index == 0) float_input else try func.appendArithImm(b, ptr_t, .add, float_input, @intCast(index * 4)); + const source = try func.appendInst(b, f32_t, .{ .load = .{ .ptr = float_at } }); + const encoded = try func.appendInst(b, u8_t, .{ .quantize_nvfp4 = .{ + .value = source, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .divide, + .global_application = .multiply, + } }); + const encoded_wide = try func.appendInst(b, u32_t, .{ .convert = .{ .value = encoded } }); + const output_at = if (index < 2) packed_output else try func.appendArithImm(b, ptr_t, .add, packed_output, @intCast(index / 2)); + const old_byte = try func.appendInst(b, u8_t, .{ .load = .{ .ptr = output_at } }); + const old_wide = try func.appendInst(b, u32_t, .{ .convert = .{ .value = old_byte } }); + const preserved = try func.appendArithImm(b, u32_t, .bit_and, old_wide, if (index & 1 == 0) 0xf0 else 0x0f); + const placed = if (index & 1 == 0) encoded_wide else try func.appendArithImm(b, u32_t, .shl, encoded_wide, 4); + const combined = try bin(&func, b, u32_t, .bit_or, preserved, placed); + const output_byte = try func.appendInst(b, u8_t, .{ .convert = .{ .value = combined } }); + try func.appendStore(b, output_byte, output_at); + } + func.setTerminator(b, .{ .ret = ir.function.Ret.none() }); + return func; +} + +test "live: NVIDIA NVFP4 packed memory crosses a scale block and preserves its tail sibling" { + const allocator = testing.allocator; + const element_count = 19; + const packed_count = (element_count + 1) / 2; + const packed_values = [_]u8{ 0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe, 0x21, 0x43 }; + const scale_values = [_]u8{ 0x39, 0xb8 }; + const global_scale_bits: u32 = 0x3f40_0000; + const source_bits = [_]u32{ + 0x0000_0000, 0x8000_0000, 0x3e80_0000, 0x3e80_0001, 0x3f40_0000, + 0x3f40_0001, 0x3fa0_0000, 0x3fa0_0001, 0x3fe0_0000, 0x4020_0000, + 0x4060_0000, 0x40a0_0000, 0x7f7f_ffff, 0xff7f_ffff, 0x7f80_0000, + 0xff80_0000, 0x7f80_0001, 0xff80_0001, 0x7fc0_1234, + }; + try testing.expect(element_count > 16); + try testing.expect(element_count & 1 == 1); + try testing.expectEqual(element_count, source_bits.len); + try testing.expectEqual(packed_count, packed_values.len); + try testing.expectEqual(@as(u8, 0), packed_values[0] & 0x0f); + try testing.expectEqual(@as(u8, 1), packed_values[0] >> 4); + try testing.expect(scale_values[0] != scale_values[1]); + + var func = try buildNvFp4PackedMemoryKernel(allocator, element_count); + defer func.deinit(); + var h = try Harness.open(); + defer h.deinit(); + var launch = try h.compile(&func, runner_abi); + defer launch.deinit(); + try h.ensureCodeCapacity(launch.kernel.code.len * @sizeOf(u32)); + + const packed_input_offset: usize = 3; + const scale_offset: usize = 5; + const global_offset: usize = 12; + const decoded_offset: usize = 20; + const float_offset: usize = 28; + const packed_output_offset: usize = 7; + var packed_input = try guardedBuffer(&h, packed_input_offset, packed_count); + var scales = try guardedBuffer(&h, scale_offset, scale_values.len); + var globals = try guardedBuffer(&h, global_offset, 4); + var decoded_output = try guardedBuffer(&h, decoded_offset, element_count * 4); + var float_input = try guardedBuffer(&h, float_offset, element_count * 4); + var packed_output = try guardedBuffer(&h, packed_output_offset, packed_count); + @memcpy(packed_input.bytes[packed_input_offset..][0..packed_count], &packed_values); + @memcpy(scales.bytes[scale_offset..][0..scale_values.len], &scale_values); + std.mem.writeInt(u32, globals.bytes[global_offset..][0..4], global_scale_bits, .little); + for (source_bits, 0..) |bits, index| { + std.mem.writeInt(u32, float_input.bytes[float_offset + index * 4 ..][0..4], bits, .little); + } + + launch.setPtr(launch.kernel.launch.params[0].offset, packed_input.va + packed_input_offset); + launch.setPtr(launch.kernel.launch.params[1].offset, scales.va + scale_offset); + launch.setPtr(launch.kernel.launch.params[2].offset, globals.va + global_offset); + launch.setPtr(launch.kernel.launch.params[3].offset, decoded_output.va + decoded_offset); + launch.setPtr(launch.kernel.launch.params[4].offset, float_input.va + float_offset); + launch.setPtr(launch.kernel.launch.params[5].offset, packed_output.va + packed_output_offset); + try launch.run(.{ 1, 1, 1 }); + + var expected_output = [_]u8{low_float_guard} ** packed_count; + const global_scale: f32 = @bitCast(global_scale_bits); + for (0..element_count) |index| { + const payload = if (index & 1 == 0) packed_values[index / 2] & 0x0f else packed_values[index / 2] >> 4; + const scale = scale_values[index / 16]; + const expected_decoded = ir.nvfp4.dequantize(payload, scale, global_scale, .multiply, .divide); + const actual_bits = decoded_output.read(u32, decoded_offset / 4 + index); + if (std.math.isNan(expected_decoded)) { + try testing.expect(std.math.isNan(@as(f32, @bitCast(actual_bits)))); + } else { + try testing.expectEqual(@as(u32, @bitCast(expected_decoded)), actual_bits); + } + const encoded = ir.nvfp4.quantize(@bitCast(source_bits[index]), scale, global_scale, .divide, .multiply); + if (index & 1 == 0) { + expected_output[index / 2] = (expected_output[index / 2] & 0xf0) | encoded; + } else { + expected_output[index / 2] = (expected_output[index / 2] & 0x0f) | (encoded << 4); + } + } + try testing.expectEqualSlices(u8, &expected_output, packed_output.bytes[packed_output_offset..][0..packed_count]); + try testing.expectEqual(@as(u8, low_float_guard & 0xf0), packed_output.bytes[packed_output_offset + packed_count - 1] & 0xf0); + try expectGuard(packed_input, packed_input_offset, packed_count); + try expectGuard(scales, scale_offset, scale_values.len); + try expectGuard(globals, global_offset, 4); + try expectGuard(decoded_output, decoded_offset, element_count * 4); + try expectGuard(float_input, float_offset, element_count * 4); + try expectGuard(packed_output, packed_output_offset, packed_count); +} + +fn appendUnique(bits: u32, seen: *std.AutoHashMapUnmanaged(u32, void), values: *std.ArrayList(u32), allocator: std.mem.Allocator) !void { + const gop = try seen.getOrPut(allocator, bits); + if (!gop.found_existing) try values.append(allocator, bits); +} + +fn appendTriplet(center: u32, seen: *std.AutoHashMapUnmanaged(u32, void), values: *std.ArrayList(u32), allocator: std.mem.Allocator) !void { + try appendUnique(center -% 1, seen, values, allocator); + try appendUnique(center, seen, values, allocator); + try appendUnique(center +% 1, seen, values, allocator); +} + +fn expectCorpusTriplet(seen: *const std.AutoHashMapUnmanaged(u32, void), center: u32) !void { + try testing.expect(seen.contains(center -% 1)); + try testing.expect(seen.contains(center)); + try testing.expect(seen.contains(center +% 1)); +} + +fn positiveFiniteMax(format: ir.low_float.Format) u16 { + return switch (format) { + .bf16 => 0x7f7f, + .f8_e4m3 => 0x007e, + .f8_e5m2 => 0x007b, + }; +} + +fn positiveMaxSubnormal(format: ir.low_float.Format) u16 { + return switch (format) { + .bf16 => 0x007f, + .f8_e4m3 => 0x0007, + .f8_e5m2 => 0x0003, + }; +} + +fn midpointBits(format: ir.low_float.Format, lower: u16, upper: u16) !u32 { + const lo: u32 = @bitCast(try ir.low_float.decode(format, lower)); + const hi: u32 = @bitCast(try ir.low_float.decode(format, upper)); + try testing.expect(lo < hi); + try testing.expect((hi - lo) & 1 == 0); + return lo + (hi - lo) / 2; +} + +fn lowFloatEncodeCorpus(allocator: std.mem.Allocator, format: ir.low_float.Format) ![]u32 { + var seen: std.AutoHashMapUnmanaged(u32, void) = .empty; + defer seen.deinit(allocator); + var values: std.ArrayList(u32) = .empty; + errdefer values.deinit(allocator); + + // Every f32 exponent field, five mantissas, and both signs. This also includes signed + // zero, infinities, and several signaling and quiet NaNs. + const mantissas = [_]u32{ 0, 1, 0x003f_ffff, 0x007f_fffe, 0x007f_ffff }; + for (0..256) |exponent| { + for (mantissas) |mantissa| { + inline for (.{ @as(u32, 0), @as(u32, 0x8000_0000) }) |sign| { + try appendUnique(sign | (@as(u32, @intCast(exponent)) << 23) | mantissa, &seen, &values, allocator); + } + } + } + + // Every adjacent pair of nonnegative finite destination values, with the exact f32 + // midpoint and its immediate bit neighbours, then the sign-reflected three values. + const finite_max = positiveFiniteMax(format); + var interval_count: usize = 0; + var lower: u16 = 0; + while (lower < finite_max) : (lower += 1) { + const midpoint = try midpointBits(format, lower, lower + 1); + try appendTriplet(midpoint, &seen, &values, allocator); + try appendTriplet(midpoint | 0x8000_0000, &seen, &values, allocator); + interval_count += 1; + } + try testing.expectEqual(@as(usize, finite_max), interval_count); + + // The committed shared boundary corpus is part of the live corpus too. + for (ir.low_float.bf16_encode_boundary_cases) |case| if (format == .bf16) { + try appendUnique(case.source, &seen, &values, allocator); + }; + for (ir.low_float.e4_encode_boundary_cases) |case| if (format == .f8_e4m3) { + try appendUnique(case.source, &seen, &values, allocator); + try appendUnique(case.source | 0x8000_0000, &seen, &values, allocator); + }; + for (ir.low_float.e5_encode_boundary_cases) |case| if (format == .f8_e5m2) { + try appendUnique(case.source, &seen, &values, allocator); + try appendUnique(case.source | 0x8000_0000, &seen, &values, allocator); + }; + for (ir.low_float.fp8_encode_special_cases) |case| if (format != .bf16) { + try appendUnique(case.source, &seen, &values, allocator); + }; + + // Name the storage-format boundaries independently of the committed tables: zero to + // subnormal, subnormal to normal, maximum finite, and overflow/saturation. Each gets + // both immediate f32 bit neighbours and both signs. + const first_mid = try midpointBits(format, 0, 1); + const last_subnormal = positiveMaxSubnormal(format); + const normal_mid = try midpointBits(format, last_subnormal, last_subnormal + 1); + const max_bits: u32 = @bitCast(try ir.low_float.decode(format, finite_max)); + const overflow_mid: u32 = switch (format) { + .bf16 => 0x7f7f_8000, + .f8_e4m3 => 0x43e8_0000, + .f8_e5m2 => 0x4770_0000, + }; + for ([_]u32{ first_mid, normal_mid, max_bits, overflow_mid }) |boundary| { + try appendTriplet(boundary, &seen, &values, allocator); + try appendTriplet(boundary | 0x8000_0000, &seen, &values, allocator); + } + + // The guarded width proof requires an odd active element count. Preserve deduplication: + // add one otherwise-unused deterministic f32 bit pattern only when parity demands it. + var filler: u32 = 0x1234_5678; + while (values.items.len % 2 == 0) : (filler +%= 1) try appendUnique(filler, &seen, &values, allocator); + try testing.expect(values.items.len % 2 == 1); + + // Audit the FINAL deduplicated corpus rather than trusting the loops that attempted to + // populate it. A shortened append helper or a skipped append therefore fails here, before + // any device allocation or launch can hide the missing coverage. + var exponent_seen = [_]bool{false} ** 256; + for (values.items) |bits| exponent_seen[@intCast((bits >> 23) & 0xff)] = true; + for (exponent_seen) |covered| try testing.expect(covered); + + lower = 0; + while (lower < finite_max) : (lower += 1) { + const midpoint = try midpointBits(format, lower, lower + 1); + try expectCorpusTriplet(&seen, midpoint); + try expectCorpusTriplet(&seen, midpoint | 0x8000_0000); + } + + for (ir.low_float.bf16_encode_boundary_cases) |case| if (format == .bf16) { + try testing.expect(seen.contains(case.source)); + }; + for (ir.low_float.e4_encode_boundary_cases) |case| if (format == .f8_e4m3) { + try testing.expect(seen.contains(case.source)); + try testing.expect(seen.contains(case.source | 0x8000_0000)); + }; + for (ir.low_float.e5_encode_boundary_cases) |case| if (format == .f8_e5m2) { + try testing.expect(seen.contains(case.source)); + try testing.expect(seen.contains(case.source | 0x8000_0000)); + }; + for (ir.low_float.fp8_encode_special_cases) |case| if (format != .bf16) { + try testing.expect(seen.contains(case.source)); + }; + for ([_]u32{ first_mid, normal_mid, max_bits, overflow_mid }) |boundary| { + try expectCorpusTriplet(&seen, boundary); + try expectCorpusTriplet(&seen, boundary | 0x8000_0000); + } + + return values.toOwnedSlice(allocator); +} + +test "live: NVIDIA low-float decode exhaustively matches the shared oracle with guarded narrow buffers" { + const allocator = testing.allocator; + var h = try Harness.open(); + defer h.deinit(); + + for ([_]ir.low_float.Format{ .bf16, .f8_e4m3, .f8_e5m2 }) |format| { + var func = try buildLowFloatBufferKernel(allocator, format, true); + defer func.deinit(); + var launch = try h.compile(&func, runner_abi); + defer launch.deinit(); + + const payload_bytes: usize = format.payloadBits() / 8; + const payload_count: usize = if (format == .bf16) 65536 else 256; + const count = payload_count + 1; // deliberately odd; the final payload repeats zero + const input_offset: usize = if (payload_bytes == 1) 3 else 6; + const output_offset: usize = 4; + var input = try guardedBuffer(&h, input_offset, count * payload_bytes); + var output = try guardedBuffer(&h, output_offset, count * @sizeOf(u32)); + + for (0..count) |i| { + const payload: u16 = @intCast(i % payload_count); + if (payload_bytes == 1) { + input.bytes[input_offset + i] = @truncate(payload); + } else { + std.mem.writeInt(u16, input.bytes[input_offset + i * 2 ..][0..2], payload, .little); + } + } + + launch.setPtr(launch.kernel.launch.params[0].offset, output.va + output_offset); + launch.setPtr(launch.kernel.launch.params[1].offset, input.va + input_offset); + try launch.run(.{ @intCast(count), 1, 1 }); + + for (0..count) |i| { + const payload: u16 = @intCast(i % payload_count); + const expected: u32 = @bitCast(try ir.low_float.decode(format, payload)); + try testing.expectEqual(expected, output.read(u32, output_offset / 4 + i)); + } + try expectGuard(input, input_offset, count * payload_bytes); + try expectGuard(output, output_offset, count * @sizeOf(u32)); + } +} + +test "live: NVIDIA low-float encode matches the deterministic boundary corpus with guarded narrow buffers" { + const allocator = testing.allocator; + var h = try Harness.open(); + defer h.deinit(); + + for ([_]ir.low_float.Format{ .bf16, .f8_e4m3, .f8_e5m2 }) |format| { + const corpus = try lowFloatEncodeCorpus(allocator, format); + defer allocator.free(corpus); + try testing.expect(corpus.len % 2 == 1); + + var func = try buildLowFloatBufferKernel(allocator, format, false); + defer func.deinit(); + var launch = try h.compile(&func, runner_abi); + defer launch.deinit(); + + const payload_bytes: usize = format.payloadBits() / 8; + const input_offset: usize = 4; + const output_offset: usize = if (payload_bytes == 1) 3 else 6; + var input = try guardedBuffer(&h, input_offset, corpus.len * @sizeOf(u32)); + var output = try guardedBuffer(&h, output_offset, corpus.len * payload_bytes); + for (corpus, 0..) |bits, i| { + std.mem.writeInt(u32, input.bytes[input_offset + i * 4 ..][0..4], bits, .little); + } + + launch.setPtr(launch.kernel.launch.params[0].offset, output.va + output_offset); + launch.setPtr(launch.kernel.launch.params[1].offset, input.va + input_offset); + try launch.run(.{ @intCast(corpus.len), 1, 1 }); + + for (corpus, 0..) |bits, i| { + const expected = ir.low_float.encode(format, @bitCast(bits)); + const got: u16 = if (payload_bytes == 1) + output.read(u8, output_offset + i) + else + output.read(u16, output_offset / 2 + i); + try testing.expectEqual(expected, got); + } + try expectGuard(input, input_offset, corpus.len * @sizeOf(u32)); + try expectGuard(output, output_offset, corpus.len * payload_bytes); + } +} + +test "NVIDIA production allocator compiles overlapping low-float chains with retained results" { + const allocator = testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const bool_t = try func.types.intern(.bool); + const ptr_t = try func.types.ptrGlobal(); + const b = try func.appendBlock(); + const out = try func.appendBlockParam(b, ptr_t); + const input = try func.appendBlockParam(b, ptr_t); + + // Device memory supplies every narrow value. Two predicates are deliberately defined + // before all six conversions and consumed after them, keeping real predicate values live + // across the expansion's own compare/select chains in the production Wimmer allocator. + const bf_payload = try func.appendInst(b, u16_t, .{ .load = .{ .ptr = input } }); + const e4_payload = try func.appendInst(b, u8_t, .{ .load = .{ .ptr = try ptrAdd(&func, b, ptr_t, input, 2) } }); + const e5_payload = try func.appendInst(b, u8_t, .{ .load = .{ .ptr = try ptrAdd(&func, b, ptr_t, input, 3) } }); + const x = try func.appendInst(b, f32_t, .{ .load = .{ .ptr = try ptrAdd(&func, b, ptr_t, input, 4) } }); + const y = try func.appendInst(b, f32_t, .{ .load = .{ .ptr = try ptrAdd(&func, b, ptr_t, input, 8) } }); + const float_pred = try func.appendInst(b, bool_t, .{ .icmp = .{ .op = .lt, .lhs = x, .rhs = y } }); + const int_pred = try func.appendInst(b, bool_t, .{ .icmp = .{ .op = .lt, .lhs = e4_payload, .rhs = e5_payload } }); + + const decoded_bf = try func.appendInst(b, f32_t, .{ .decode_low_float = .{ .format = .bf16, .value = bf_payload } }); + const decoded_e4 = try func.appendInst(b, f32_t, .{ .decode_low_float = .{ .format = .f8_e4m3, .value = e4_payload } }); + const decoded_e5 = try func.appendInst(b, f32_t, .{ .decode_low_float = .{ .format = .f8_e5m2, .value = e5_payload } }); + const encoded_bf = try func.appendInst(b, u16_t, .{ .encode_low_float = .{ .format = .bf16, .value = x } }); + const encoded_e4 = try func.appendInst(b, u8_t, .{ .encode_low_float = .{ .format = .f8_e4m3, .value = y } }); + const encoded_e5 = try func.appendInst(b, u8_t, .{ .encode_low_float = .{ .format = .f8_e5m2, .value = x } }); + + const partial = try bin(&func, b, f32_t, .add, decoded_bf, decoded_e4); + const sum = try bin(&func, b, f32_t, .add, partial, decoded_e5); + const float_pick = try func.appendInst(b, u8_t, .{ .select = .{ .cond = float_pred, .then = encoded_e4, .@"else" = encoded_e5 } }); + const final_pick = try func.appendInst(b, u8_t, .{ .select = .{ .cond = int_pred, .then = float_pick, .@"else" = encoded_e4 } }); + try func.appendStore(b, encoded_bf, out); + try func.appendStore(b, final_pick, try ptrAdd(&func, b, ptr_t, out, 2)); + try func.appendStore(b, encoded_e5, try ptrAdd(&func, b, ptr_t, out, 3)); + try func.appendStore(b, sum, try ptrAdd(&func, b, ptr_t, out, 4)); + func.setTerminator(b, .{ .ret = ir.function.Ret.none() }); + + var kernel = try isel.compileKernel(allocator, &func, runner_abi); + defer kernel.deinit(allocator); + try testing.expect(kernel.code.len != 0); + try testing.expectEqual(@as(usize, 2), kernel.launch.params.len); +} + /// `(idx.z * extent.y + idx.y) * extent.x + idx.x` as IR, which is the launch contract's /// own linearization: x fastest and z slowest. See `gpu.kernel.linearIndex`. fn linear3(func: *Function, b: Block, ty: ir.types.Type, idx: [3]Value, extent: [3]u32) !Value { diff --git a/libs/vulcan-target/riscv64/isel.zig b/libs/vulcan-target/riscv64/isel.zig index 946042a..34592a6 100644 --- a/libs/vulcan-target/riscv64/isel.zig +++ b/libs/vulcan-target/riscv64/isel.zig @@ -847,6 +847,127 @@ fn storeFloat(allocator: std.mem.Allocator, code: *std.ArrayList(u32), alloc: *c } } +/// Emit an exact u32-to-f32 bit reinterpret for already-resolved allocator locations. Integer and +/// float spill slots use their established eight-byte frame strides, while the actual float load or +/// store is 32-bit. Keeping this helper location-driven makes every resident/spilled combination +/// explicit and independently testable. +fn emitU32ToF32Reinterpret( + allocator: std.mem.Allocator, + code: *std.ArrayList(u32), + spill_base: u32, + float_spill_base: u32, + src: IntLoc, + dst: FloatLoc, + float_tmp: FReg, +) std.mem.Allocator.Error!void { + const rs = switch (src) { + .reg => |r| r, + .slot => |slot| blk: { + try code.append(allocator, encode.ld(spill_scratch0, .x2, @intCast(spill_base + slot * 8))); + break :blk spill_scratch0; + }, + }; + const rd = switch (dst) { + .reg => |r| r, + .slot => float_tmp, + }; + try code.append(allocator, encode.fmv_w_x(rd, rs)); + switch (dst) { + .reg => {}, + .slot => |slot| try code.append(allocator, encode.fsw(rd, .x2, @intCast(float_spill_base + slot * 8))), + } +} + +/// Emit an exact f32-to-u32 bit reinterpret for already-resolved allocator locations. `fmv.x.w` +/// sign-extends bit 31 to XLEN on RV64, so the shift pair explicitly clears the upper 32 bits before +/// the value can participate in unsigned arithmetic, widening, spilling, or an integer return. +fn emitF32ToU32Reinterpret( + allocator: std.mem.Allocator, + code: *std.ArrayList(u32), + spill_base: u32, + float_spill_base: u32, + src: FloatLoc, + dst: IntLoc, + float_tmp: FReg, +) std.mem.Allocator.Error!void { + const rs = switch (src) { + .reg => |r| r, + .slot => |slot| blk: { + try code.append(allocator, encode.flw(float_tmp, .x2, @intCast(float_spill_base + slot * 8))); + break :blk float_tmp; + }, + }; + const rd = switch (dst) { + .reg => |r| r, + .slot => spill_scratch0, + }; + try code.append(allocator, encode.fmv_x_w(rd, rs)); + try code.append(allocator, encode.slli(rd, rd, 32)); + try code.append(allocator, encode.srli(rd, rd, 32)); + switch (dst) { + .reg => {}, + .slot => |slot| try code.append(allocator, encode.sd(rd, .x2, @intCast(spill_base + slot * 8))), + } +} + +fn emitIntSelectArm( + allocator: std.mem.Allocator, + code: *std.ArrayList(u32), + spill_base: u32, + src: IntLoc, + dst: IntLoc, +) std.mem.Allocator.Error!void { + const rs = switch (src) { + .reg => |r| r, + .slot => |slot| blk: { + try code.append(allocator, encode.ld(spill_scratch1, .x2, @intCast(spill_base + slot * 8))); + break :blk spill_scratch1; + }, + }; + const rd = switch (dst) { + .reg => |r| r, + .slot => spill_scratch0, + }; + if (rd != rs) try code.append(allocator, encode.addi(rd, rs, 0)); + switch (dst) { + .reg => {}, + .slot => |slot| try code.append(allocator, encode.sd(rd, .x2, @intCast(spill_base + slot * 8))), + } +} + +/// Emit a scalar integer select without requiring any operand or result to remain resident. Only +/// the chosen arm is reloaded, so the condition and two alternatives never need three spill +/// scratches simultaneously. The local branch distances are patched immediately and stay within a +/// handful of instructions. +fn emitSpillSafeIntSelect( + allocator: std.mem.Allocator, + code: *std.ArrayList(u32), + spill_base: u32, + cond_loc: IntLoc, + then_loc: IntLoc, + else_loc: IntLoc, + dst_loc: IntLoc, +) std.mem.Allocator.Error!void { + const cond = switch (cond_loc) { + .reg => |r| r, + .slot => |slot| blk: { + try code.append(allocator, encode.ld(spill_scratch0, .x2, @intCast(spill_base + slot * 8))); + break :blk spill_scratch0; + }, + }; + const branch_at = code.items.len; + try code.append(allocator, encode.beq(cond, .x0, 0)); + try emitIntSelectArm(allocator, code, spill_base, then_loc, dst_loc); + const jump_at = code.items.len; + try code.append(allocator, encode.jal(.x0, 0)); + const else_at = code.items.len; + try emitIntSelectArm(allocator, code, spill_base, else_loc, dst_loc); + const end_at = code.items.len; + + code.items[branch_at] = encode.beq(cond, .x0, @intCast((else_at - branch_at) * 4)); + code.items[jump_at] = encode.jal(.x0, @intCast((end_at - jump_at) * 4)); +} + /// The sp-relative byte offset of f128 value `v`'s LOW 64-bit half in its 16-byte class-4 slot (the /// HIGH half sits at `+8`). Every f128 value is spill-resident (its register pool is empty), so it /// always has a `quad_spill` entry. A missing one is a codegen bug, not a runtime condition. @@ -1934,6 +2055,14 @@ fn usesInInst(func: *const Function, inst: ir.function.Inst, v: Value) usize { if (s.then == v) c += 1; if (s.@"else" == v) c += 1; }, + .decode_low_float, .encode_low_float => |cv| { + if (cv.value == v) c += 1; + }, + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + if (cv.value == v) c += 1; + if (cv.block_scale == v) c += 1; + if (cv.global_scale == v) c += 1; + }, .load => |l| { if (l.ptr == v) c += 1; }, @@ -2063,17 +2192,24 @@ fn riscv64ClassOf(ctx: *const anyopaque, func: *const Function, v: Value) u16 { return 0; } -/// `RegDescription.useKind` for riscv64: every operand needs a register. riscv64 has no memory -/// operands, and some sites (for example the fused compare-and-branch) cannot reload a spilled operand, so -/// `must_have_register` is both conservative and correct. Unused params are the generic hook shape. +/// `RegDescription.useKind` for riscv64. Most operands need a register because riscv64 has no memory +/// operands and some sites, such as fused compare-and-branch, cannot reload a spilled operand. Scalar +/// select and same-width reinterpret are the exceptions: their emitters explicitly handle slot operands. fn riscv64UseKind(ctx: *const anyopaque, func: *const Function, inst: ir.function.Inst, operand: Value) wimmer.UseKind { _ = ctx; - _ = inst; // An f128 operand (class 4) has no register to occupy: its class pool is empty, so it is always // read from its 16-byte slot. Forcing `must_have_register` would make the allocator try to place it // in a register the class does not have. It must be `should_have_register` (the shared allocator // then leaves it in its slot, which the f128 isel sites read directly). if (isQuad(func, func.valueType(operand))) return .should_have_register; + // These two scalar forms explicitly reload every operand through the reserved class scratches. + // Letting Wimmer leave an operand in its slot is what makes their spill-safe emission reachable + // under real low-float expansion pressure instead of only through direct helper tests. + switch (func.insts.items[@intFromEnum(inst)].op) { + .select => return .should_have_register, + .unary => |u| if (u.op == .reinterpret) return .should_have_register, + else => {}, + } return .must_have_register; } @@ -3285,7 +3421,112 @@ fn alignPadWords(words: usize, fetch_align: u16) usize { /// under a model with `features.riscv64.zicbop` set reaches the new path. `prefetch.r` is /// ORI-shaped (see encode.zig), so unlike the VPU path this one is execution-validated: it /// decodes as a harmless no-op on any qemu-riscv64 host, Zicbop or not. +fn reloadedAt(alloc: *const Allocation, value: Value, pos: usize) bool { + for (alloc.actions.items) |action| { + if (action.at == pos and action.value == value and action.kind == .reload) return true; + } + return false; +} + +fn intHasSlot(alloc: *const Allocation, value: Value) bool { + if (alloc.int_spill.contains(value)) return true; + if (alloc.segments.get(value)) |segments| for (segments) |segment| switch (segment.loc) { + .slot => return true, + .reg => {}, + }; + return false; +} + +fn floatHasSlot(alloc: *const Allocation, value: Value) bool { + if (alloc.float_spill.contains(value)) return true; + if (alloc.float_segments.get(value)) |segments| for (segments) |segment| switch (segment.loc) { + .slot => return true, + .reg => {}, + }; + return false; +} + +fn intSpilledAtUse(alloc: *const Allocation, value: Value, pos: usize) bool { + return switch (intLocationAt(alloc, value, pos)) { + .slot => true, + .reg => reloadedAt(alloc, value, pos), + }; +} + +fn floatSpilledAtUse(alloc: *const Allocation, value: Value, pos: usize) bool { + return switch (floatLocationAt(alloc, value, pos)) { + .slot => true, + .reg => reloadedAt(alloc, value, pos), + }; +} + +fn collectLowFloatSpillEvidence(func: *const Function, alloc: *const Allocation, evidence: *LowFloatSpillEvidence) void { + evidence.int_spill_count = alloc.spill_count; + evidence.float_spill_count = alloc.float_spill_count; + evidence.split_action_count = @intCast(alloc.actions.items.len); + var select_cond_spilled = false; + var select_then_spilled = false; + var select_else_spilled = false; + var select_result_spilled = false; + var pos: usize = 0; + for (0..func.blockCount()) |bi| { + const insts = func.blockInsts(@enumFromInt(bi)); + for (insts, 0..) |inst, index| { + const inst_pos = pos + 1 + index; + const result = func.instResult(inst); + switch (func.opcode(inst)) { + .select => |select| if (result) |value| { + select_cond_spilled = select_cond_spilled or intSpilledAtUse(alloc, select.cond, inst_pos); + select_then_spilled = select_then_spilled or intSpilledAtUse(alloc, select.then, inst_pos); + select_else_spilled = select_else_spilled or intSpilledAtUse(alloc, select.@"else", inst_pos); + select_result_spilled = select_result_spilled or intHasSlot(alloc, value); + }, + .unary => |unary| if (unary.op == .reinterpret) if (result) |value| { + const source_float = isFloat(func, func.valueType(unary.value)); + const result_float = isFloat(func, func.valueType(value)); + if (!source_float and result_float and + intSpilledAtUse(alloc, unary.value, inst_pos) and floatHasSlot(alloc, value)) evidence.u32_to_f32_source_and_result = true; + if (source_float and !result_float and + floatSpilledAtUse(alloc, unary.value, inst_pos) and intHasSlot(alloc, value)) evidence.f32_to_u32_source_and_result = true; + }, + else => {}, + } + } + pos += insts.len + 2; + } + // A fresh SSA definition receives a register at its defining position; a "spilled result" is + // therefore evidenced by the subsequent slot segment/store action, while operands are proved + // spilled at the use itself (direct slot or a same-position reload action). + evidence.select_condition_spilled_at_use = select_cond_spilled; + evidence.select_then_spilled_at_use = select_then_spilled; + evidence.select_else_spilled_at_use = select_else_spilled; + evidence.select_result_has_slot = select_result_spilled; + evidence.select_operands_and_result = select_cond_spilled and select_then_spilled and select_else_spilled and select_result_spilled; +} + +pub const LowFloatSpillEvidence = struct { + select_operands_and_result: bool = false, + select_condition_spilled_at_use: bool = false, + select_then_spilled_at_use: bool = false, + select_else_spilled_at_use: bool = false, + select_result_has_slot: bool = false, + u32_to_f32_source_and_result: bool = false, + f32_to_u32_source_and_result: bool = false, + int_spill_count: u32 = 0, + float_spill_count: u32 = 0, + split_action_count: u32 = 0, +}; + +pub fn compileFunctionWithLowFloatSpillEvidence(allocator: std.mem.Allocator, func: *const Function, caps: ModelCaps, evidence: *LowFloatSpillEvidence) Error!Compiled { + evidence.* = .{}; + return compileFunctionInternal(allocator, func, caps, evidence); +} + pub fn compileFunction(allocator: std.mem.Allocator, func: *const Function, caps: ModelCaps) Error!Compiled { + return compileFunctionInternal(allocator, func, caps, null); +} + +fn compileFunctionInternal(allocator: std.mem.Allocator, func: *const Function, caps: ModelCaps, spill_evidence: ?*LowFloatSpillEvidence) Error!Compiled { // f16 lowering has two modes (see `ModelCaps.zfh`). Software emulation (no Zfh, the default): // an f16 is held as its f32 widening in a float register and every boundary rounds via the // inline convert routines (`emitHalfToFloat`/`emitFloatToHalf`). Those routines need dedicated @@ -3319,6 +3560,9 @@ pub fn compileFunction(allocator: std.mem.Allocator, func: *const Function, caps var work = try func.clone(allocator); defer work.deinit(); + _ = try ir.expand.expandNvFp4(allocator, &work); + _ = try ir.expand.expandLowFloat(allocator, &work); + // Lower binary128 arithmetic, compares, conversions, and sqrt to soft-fp libcalls on the clone, // before any numbering or edge splitting, so the call clobbers and the f128 GPR-pair argument // placement are visible to the allocator. f128 data movement stays native and is emitted below. @@ -3372,6 +3616,8 @@ pub fn compileFunction(allocator: std.mem.Allocator, func: *const Function, caps var alloc = try translateAllocation(allocator, &work, caps.vpu, &walloc); defer alloc.deinit(allocator); + if (spill_evidence) |evidence| collectLowFloatSpillEvidence(&work, &alloc, evidence); + // An edge move on an if-edge cannot be realized by the `.@"if"` emission (it only branches), so // reject rather than drop it. Register phis land on jump edges, which the `.jump` path replays. if (edgeMoveOnIfEdge(&work, &alloc)) return error.Unsupported; @@ -3907,6 +4153,8 @@ fn emitFromAllocation(allocator: std.mem.Allocator, func: *const Function, caps: action_cursor += 1; } switch (func.opcode(inst)) { + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, .arith => |a| { // Fused multiply-add/sub: when this is a scalar float `mul` that is the // single-use, immediately-preceding operand of the next add/sub, skip its @@ -4365,33 +4613,50 @@ fn emitFromAllocation(allocator: std.mem.Allocator, func: *const Function, caps: if (fuse_addr_hi_lo) std.debug.assert(code.items.len == hi + 2); }, .select => |sel| { - // `cond ? then : else`, lowered to a short forward branch. The - // result register is distinct from the operands (drawn while - // they are still live), so there is no aliasing hazard. if (isFloat(func, func.valueType(sel.then))) return error.Unsupported; // float select: later - const rd = switch (intLocationAt(alloc, func.instResult(inst).?, inst_pos)) { - .reg => |r| r, - .slot => return error.Unsupported, - }; - // The three operands each need a live register through the branch sequence, more - // than the two int spill scratches can reload. A spilled operand (for example a spilled - // int block param) is rejected cleanly rather than panicking on the unwrap. - // Resident in every currently-compiling case (byte-identical there). - const cond = switch (intLocationAt(alloc, sel.cond, inst_pos)) { - .reg => |r| r, - .slot => return error.Unsupported, - }; - const then_r = switch (intLocationAt(alloc, sel.then, inst_pos)) { - .reg => |r| r, - .slot => return error.Unsupported, - }; - const else_r = switch (intLocationAt(alloc, sel.@"else", inst_pos)) { - .reg => |r| r, - .slot => return error.Unsupported, - }; - try code.append(allocator, encode.addi(rd, then_r, 0)); // rd = then - try code.append(allocator, encode.bne(cond, .x0, 8)); // if cond != 0, keep then - try code.append(allocator, encode.addi(rd, else_r, 0)); // else rd = else + try emitSpillSafeIntSelect( + allocator, + &code, + spill_base, + intLocationAt(alloc, sel.cond, inst_pos), + intLocationAt(alloc, sel.then, inst_pos), + intLocationAt(alloc, sel.@"else", inst_pos), + intLocationAt(alloc, func.instResult(inst).?, inst_pos), + ); + }, + .unary => |u| { + if (u.op != .reinterpret) return error.Unsupported; + const result = func.instResult(inst).?; + const src_ty = func.valueType(u.value); + const dst_ty = func.valueType(result); + const src_float = isFloat(func, src_ty); + const dst_float = isFloat(func, dst_ty); + if (src_float == dst_float) return error.Unsupported; + if (src_float) { + if (!isUnsignedInt(func, dst_ty) or intBits(func, dst_ty) != 32 or + isHalf(func, src_ty) or is64Float(func, src_ty) or isQuad(func, src_ty)) return error.Unsupported; + try emitF32ToU32Reinterpret( + allocator, + &code, + spill_base, + float_spill_base, + floatLocationAt(alloc, u.value, inst_pos), + intLocationAt(alloc, result, inst_pos), + fspill0, + ); + } else { + if (!isUnsignedInt(func, src_ty) or intBits(func, src_ty) != 32 or + isHalf(func, dst_ty) or is64Float(func, dst_ty) or isQuad(func, dst_ty)) return error.Unsupported; + try emitU32ToF32Reinterpret( + allocator, + &code, + spill_base, + float_spill_base, + intLocationAt(alloc, u.value, inst_pos), + floatLocationAt(alloc, result, inst_pos), + fspill0, + ); + } }, .convert => |cv| { const result = func.instResult(inst).?; @@ -7697,16 +7962,90 @@ test "selectFunctionForModel fires the alignment hook from river-rc1.ma, matches // the class scratch (int `spill_scratch0`, float `float_scratch`/`float_spill_scratch0_vpu` in vpu // mode, RVV `vector_scratch`, VPU `float_scratch`). // -// A natural Wimmer allocation never reaches this shape on riscv64, for the same structural reason -// documented at aarch64's and x86_64's own `.slot_to_slot` tests: `riscv64UseKind` makes every operand -// use `must_have_register`, so a placed value alternates register, slot, register, and two adjacent -// slot segments never arise. The shared `wimmer.zig` `orderMoves`/`orderIntraActions` machinery also +// A natural Wimmer allocation never reaches this shape on riscv64. Most operands use +// `must_have_register`, while select/reinterpret operands which may remain in a slot need no adjacent +// slot-to-slot transition. The shared `wimmer.zig` `orderMoves`/`orderIntraActions` machinery also // expands any slot-to-slot parallel-move shuffle through the class scratch before it ever becomes an // `Action` (see `SplitAction`'s doc comment), so `walloc.actions` never carries one either. These // tests exercise the exact mechanism (`emitSplitAction`'s `.slot_to_slot` arm) directly with a // hand-built `SplitAction`, one per register class, the same way the aarch64/x86_64 bridges do. // =========================================================================== +test "u32/f32 reinterpret emission covers every resident and spilled location pair" { + const allocator = std.testing.allocator; + const spill_base: u32 = 16; + const float_spill_base: u32 = 80; + + inline for (.{ false, true }) |src_spilled| { + inline for (.{ false, true }) |dst_spilled| { + var code: std.ArrayList(u32) = .empty; + defer code.deinit(allocator); + const src: IntLoc = if (src_spilled) .{ .slot = 2 } else .{ .reg = .x10 }; + const dst: FloatLoc = if (dst_spilled) .{ .slot = 3 } else .{ .reg = .f11 }; + try emitU32ToF32Reinterpret(allocator, &code, spill_base, float_spill_base, src, dst, float_spill_scratch0); + + var expected: std.ArrayList(u32) = .empty; + defer expected.deinit(allocator); + const rs: Reg = if (src_spilled) spill_scratch0 else .x10; + const rd: FReg = if (dst_spilled) float_spill_scratch0 else .f11; + if (src_spilled) try expected.append(allocator, encode.ld(rs, .x2, spill_base + 2 * 8)); + try expected.append(allocator, encode.fmv_w_x(rd, rs)); + if (dst_spilled) try expected.append(allocator, encode.fsw(rd, .x2, float_spill_base + 3 * 8)); + try std.testing.expectEqualSlices(u32, expected.items, code.items); + } + } + + inline for (.{ false, true }) |src_spilled| { + inline for (.{ false, true }) |dst_spilled| { + var code: std.ArrayList(u32) = .empty; + defer code.deinit(allocator); + const src: FloatLoc = if (src_spilled) .{ .slot = 4 } else .{ .reg = .f12 }; + const dst: IntLoc = if (dst_spilled) .{ .slot = 5 } else .{ .reg = .x11 }; + try emitF32ToU32Reinterpret(allocator, &code, spill_base, float_spill_base, src, dst, float_spill_scratch0); + + var expected: std.ArrayList(u32) = .empty; + defer expected.deinit(allocator); + const rs: FReg = if (src_spilled) float_spill_scratch0 else .f12; + const rd: Reg = if (dst_spilled) spill_scratch0 else .x11; + if (src_spilled) try expected.append(allocator, encode.flw(rs, .x2, float_spill_base + 4 * 8)); + try expected.append(allocator, encode.fmv_x_w(rd, rs)); + try expected.append(allocator, encode.slli(rd, rd, 32)); + try expected.append(allocator, encode.srli(rd, rd, 32)); + if (dst_spilled) try expected.append(allocator, encode.sd(rd, .x2, spill_base + 5 * 8)); + try std.testing.expectEqualSlices(u32, expected.items, code.items); + } + } +} + +test "integer select emission reloads spilled condition arms and result" { + const allocator = std.testing.allocator; + const spill_base: u32 = 24; + var code: std.ArrayList(u32) = .empty; + defer code.deinit(allocator); + try emitSpillSafeIntSelect( + allocator, + &code, + spill_base, + .{ .slot = 0 }, + .{ .slot = 1 }, + .{ .slot = 2 }, + .{ .slot = 4 }, + ); + + const expected = [_]u32{ + encode.ld(spill_scratch0, .x2, spill_base + 0 * 8), + encode.beq(spill_scratch0, .x0, 20), + encode.ld(spill_scratch1, .x2, spill_base + 1 * 8), + encode.addi(spill_scratch0, spill_scratch1, 0), + encode.sd(spill_scratch0, .x2, spill_base + 4 * 8), + encode.jal(.x0, 16), + encode.ld(spill_scratch1, .x2, spill_base + 2 * 8), + encode.addi(spill_scratch0, spill_scratch1, 0), + encode.sd(spill_scratch0, .x2, spill_base + 4 * 8), + }; + try std.testing.expectEqualSlices(u32, &expected, code.items); +} + test "emitSplitAction .slot_to_slot expands to reload+store through the int scratch" { const allocator = std.testing.allocator; var func = Function.init(allocator); @@ -7840,6 +8179,36 @@ test "a barrier is rejected, not dropped like a prefetch" { try std.testing.expectError(error.Unsupported, selectFunction(allocator, &func)); } +test "all low float formats and directions compile without mutating caller IR" { + const allocator = std.testing.allocator; + inline for ([_]ir.low_float.Format{ .bf16, .f8_e4m3, .f8_e5m2 }) |format| { + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) payload_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = format }; + const result = try func.appendInst(block, if (decode_direction) f32_t else payload_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ .namespace = "test", .key = "preserve", .value = .flag } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + const before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + const code = try selectFunction(allocator, &func); + allocator.free(code); + const after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try std.testing.expectEqualStrings(before, after); + try std.testing.expectEqualSlices(u8, bits_before, bits_after); + } + } +} + test "an atomic is rejected, not dropped like a prefetch" { // This backend emits no A-extension instruction, so there is no honest lowering. Both // IR forms are refused, and refusing must not be a panic. diff --git a/libs/vulcan-target/riscv64/tests/qemu_user.zig b/libs/vulcan-target/riscv64/tests/qemu_user.zig index fb3db03..4af34e9 100644 --- a/libs/vulcan-target/riscv64/tests/qemu_user.zig +++ b/libs/vulcan-target/riscv64/tests/qemu_user.zig @@ -5,11 +5,441 @@ const std = @import("std"); const ir = @import("vulcan-ir"); const cases = @import("cases.zig"); +const division_cases = @import("../../tests/f32_division_cases.zig"); +const emit = @import("../emit.zig"); +const encode = @import("../encode.zig"); const harness = @import("harness.zig"); const isel = @import("../isel.zig"); const Function = ir.function.Function; +const low_float_pressure_lanes = 24; +const select_pressure_lanes = 20; +const reinterpret_pressure_lanes = 24; + +fn buildLowFloatSpillPressure(func: *Function) !void { + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const input = try func.appendBlockParam(block, u8_t); + const anchor_bits = try func.appendInst(block, u32_t, .{ .convert = .{ .value = input } }); + var reinterpret_sources: [reinterpret_pressure_lanes]ir.function.Value = undefined; + for (&reinterpret_sources, 0..) |*source, index| { + const mask = try func.appendInst(block, u32_t, .{ .iconst = @intCast(0x3000 + index * 37) }); + source.* = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = anchor_bits, .rhs = mask } }); + } + const bool_t = try func.types.intern(.bool); + var select_conds: [select_pressure_lanes]ir.function.Value = undefined; + var select_thens: [select_pressure_lanes]ir.function.Value = undefined; + var select_elses: [select_pressure_lanes]ir.function.Value = undefined; + for (0..select_pressure_lanes) |index| { + const match = try func.appendInst(block, u8_t, .{ .iconst = @intCast(index) }); + select_conds[index] = try func.appendInst(block, bool_t, .{ .icmp = .{ .op = .eq, .lhs = input, .rhs = match } }); + const then_mask = try func.appendInst(block, u32_t, .{ .iconst = @intCast(0x1000 + index * 17) }); + const else_mask = try func.appendInst(block, u32_t, .{ .iconst = @intCast(0x2000 + index * 29) }); + select_thens[index] = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = anchor_bits, .rhs = then_mask } }); + select_elses[index] = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = anchor_bits, .rhs = else_mask } }); + } + + // Build every payload before decoding any of them. They remain simultaneously live into the + // decode phase and force the expanded integer selects and the u32->f32 boundary through slots. + var payloads: [low_float_pressure_lanes]ir.function.Value = undefined; + for (&payloads, 0..) |*payload, index| payload.* = try func.appendArithImm(block, u8_t, .add, input, @intCast(index)); + var select_results: [select_pressure_lanes]ir.function.Value = undefined; + for (&select_results, select_conds, select_thens, select_elses) |*result, cond, then_value, else_value| { + result.* = try func.appendInst(block, u32_t, .{ .select = .{ .cond = cond, .then = then_value, .@"else" = else_value } }); + } + var reinterpret_floats: [reinterpret_pressure_lanes]ir.function.Value = undefined; + for (&reinterpret_floats, reinterpret_sources) |*value, source| value.* = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = source } }); + + // Keep more decoded floats live than the 22-register scalar-float pool. Encoding starts only + // after all decodes, so both reinterpret directions encounter real float and integer pressure. + var decoded: [low_float_pressure_lanes]ir.function.Value = undefined; + for (&decoded, payloads) |*value, payload| value.* = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .format = .f8_e4m3, .value = payload } }); + var reinterpret_roundtrips: [reinterpret_pressure_lanes]ir.function.Value = undefined; + for (&reinterpret_roundtrips, reinterpret_floats) |*value, source| value.* = try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = source } }); + + var encoded: [low_float_pressure_lanes]ir.function.Value = undefined; + for (&encoded, decoded) |*payload, value| payload.* = try func.appendInst(block, u8_t, .{ .encode_low_float = .{ .format = .f8_e4m3, .value = value } }); + + var result = try func.appendInst(block, u32_t, .{ .convert = .{ .value = encoded[0] } }); + for (encoded[1..]) |payload| { + const wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = payload } }); + result = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = result, .rhs = wide } }); + } + for (reinterpret_roundtrips) |roundtrip| result = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = result, .rhs = roundtrip } }); + for (select_results) |selected| result = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = result, .rhs = selected } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); +} + +fn pressureReference(input: u8) u32 { + var result: u32 = 0; + for (0..select_pressure_lanes) |index| { + const mask: u32 = if (input == index) @intCast(0x1000 + index * 17) else @intCast(0x2000 + index * 29); + result ^= @as(u32, input) ^ mask; + } + for (0..reinterpret_pressure_lanes) |index| result ^= @as(u32, input) ^ @as(u32, @intCast(0x3000 + index * 37)); + for (0..low_float_pressure_lanes) |index| { + const payload: u8 = input + @as(u8, @intCast(index)); + const value = ir.low_float.decode(.f8_e4m3, payload) catch unreachable; + result ^= ir.low_float.encode(.f8_e4m3, value); + } + return result; +} + +fn buildNvFp4SpillPressure(func: *Function) !void { + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const bool_t = try func.types.intern(.bool); + const block = try func.appendBlock(); + const input = try func.appendBlockParam(block, u8_t); + const anchor_bits = try func.appendInst(block, u32_t, .{ .convert = .{ .value = input } }); + var reinterpret_sources: [reinterpret_pressure_lanes]ir.function.Value = undefined; + for (&reinterpret_sources, 0..) |*source, index| { + const mask = try func.appendInst(block, u32_t, .{ .iconst = @intCast(0x3000 + index * 37) }); + source.* = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = anchor_bits, .rhs = mask } }); + } + var select_conds: [select_pressure_lanes]ir.function.Value = undefined; + var select_thens: [select_pressure_lanes]ir.function.Value = undefined; + var select_elses: [select_pressure_lanes]ir.function.Value = undefined; + for (0..select_pressure_lanes) |index| { + const match = try func.appendInst(block, u8_t, .{ .iconst = @intCast(index) }); + select_conds[index] = try func.appendInst(block, bool_t, .{ .icmp = .{ .op = .eq, .lhs = input, .rhs = match } }); + const then_mask = try func.appendInst(block, u32_t, .{ .iconst = @intCast(0x1000 + index * 17) }); + const else_mask = try func.appendInst(block, u32_t, .{ .iconst = @intCast(0x2000 + index * 29) }); + select_thens[index] = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = anchor_bits, .rhs = then_mask } }); + select_elses[index] = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = anchor_bits, .rhs = else_mask } }); + } + var payloads: [low_float_pressure_lanes]ir.function.Value = undefined; + for (&payloads, 0..) |*payload, index| payload.* = try func.appendArithImm(block, u8_t, .add, input, @intCast(index)); + var select_results: [select_pressure_lanes]ir.function.Value = undefined; + for (&select_results, select_conds, select_thens, select_elses) |*result, cond, then_value, else_value| { + result.* = try func.appendInst(block, u32_t, .{ .select = .{ .cond = cond, .then = then_value, .@"else" = else_value } }); + } + var reinterpret_floats: [reinterpret_pressure_lanes]ir.function.Value = undefined; + for (&reinterpret_floats, reinterpret_sources) |*value, source| value.* = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = source } }); + + const block_scale = try func.appendInst(block, u8_t, .{ .iconst = 0x38 }); + const global_scale = try func.appendInst(block, f32_t, .{ .fconst = 1.0 }); + var decoded: [low_float_pressure_lanes]ir.function.Value = undefined; + for (&decoded, payloads) |*value, payload| value.* = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = payload, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .multiply, + } }); + var reinterpret_roundtrips: [reinterpret_pressure_lanes]ir.function.Value = undefined; + for (&reinterpret_roundtrips, reinterpret_floats) |*value, source| value.* = try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = source } }); + var encoded: [low_float_pressure_lanes]ir.function.Value = undefined; + for (&encoded, decoded) |*payload, value| payload.* = try func.appendInst(block, u8_t, .{ .quantize_nvfp4 = .{ + .value = value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .multiply, + } }); + + var result = try func.appendInst(block, u32_t, .{ .convert = .{ .value = encoded[0] } }); + for (encoded[1..]) |payload| { + const wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = payload } }); + result = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = result, .rhs = wide } }); + } + for (reinterpret_roundtrips) |roundtrip| result = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = result, .rhs = roundtrip } }); + for (select_results) |selected| result = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = result, .rhs = selected } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); +} + +fn nvfp4PressureReference(input: u8) u32 { + var result: u32 = 0; + for (0..select_pressure_lanes) |index| { + const mask: u32 = if (input == index) @intCast(0x1000 + index * 17) else @intCast(0x2000 + index * 29); + result ^= @as(u32, input) ^ mask; + } + for (0..reinterpret_pressure_lanes) |index| result ^= @as(u32, input) ^ @as(u32, @intCast(0x3000 + index * 37)); + for (0..low_float_pressure_lanes) |index| result ^= (@as(u32, input) + @as(u32, @intCast(index))) & 0x0f; + return result; +} + +fn buildIntToFloatBits(func: *Function) !void { + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const ptr_t = try func.types.ptrGlobal(); + const block = try func.appendBlock(); + const bits = try func.appendBlockParam(block, u32_t); + const value = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = bits } }); + const slot = try func.appendInst(block, ptr_t, .{ .alloca = .{ .elem = u32_t } }); + try func.appendStore(block, value, slot); + const result = try func.appendInst(block, u32_t, .{ .load = .{ .ptr = slot } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); +} + +fn buildFloatToIntBits(func: *Function) !void { + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const ptr_t = try func.types.ptrGlobal(); + const block = try func.appendBlock(); + const bits = try func.appendBlockParam(block, u32_t); + const slot = try func.appendInst(block, ptr_t, .{ .alloca = .{ .elem = u32_t } }); + try func.appendStore(block, bits, slot); + const value = try func.appendInst(block, f32_t, .{ .load = .{ .ptr = slot } }); + const result = try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = value } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); +} + +test "qemu-user-riscv scalar u32/f32 reinterpret preserves every low bit" { + const patterns = [_]u32{ + 0x0000_0000, + 0x8000_0000, + 0x3f80_0000, + 0x7f80_0000, + 0x7f80_0001, + 0x7fc1_2345, + 0xffa5_4321, + 0xffff_ffff, + }; + inline for (.{ buildIntToFloatBits, buildFloatToIntBits }) |build| { + var func = Function.init(std.testing.allocator); + defer func.deinit(); + try build(&func); + for (patterns) |pattern| { + const arg: i64 = @as(i32, @bitCast(pattern)); + const got = harness.runFunc(std.testing.io, std.testing.allocator, &func, &.{arg}, harness.qemu_user) catch |err| switch (err) { + error.SkipZigTest => return error.SkipZigTest, + else => return err, + }; + try std.testing.expectEqual(@as(i64, pattern), got); + } + } +} + +test "qemu-user-riscv low float production expansion executes through spilled selects and reinterprets" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + try buildLowFloatSpillPressure(&func); + + var evidence: isel.LowFloatSpillEvidence = .{}; + var compiled = try isel.compileFunctionWithLowFloatSpillEvidence(allocator, &func, .{}, &evidence); + defer compiled.deinit(allocator); + try std.testing.expect(evidence.int_spill_count > 0); + try std.testing.expect(evidence.float_spill_count > 0); + try std.testing.expect(evidence.split_action_count > 0); + try std.testing.expect(evidence.select_condition_spilled_at_use); + try std.testing.expect(evidence.select_then_spilled_at_use); + try std.testing.expect(evidence.select_else_spilled_at_use); + try std.testing.expect(evidence.select_result_has_slot); + try std.testing.expect(evidence.select_operands_and_result); + try std.testing.expect(evidence.u32_to_f32_source_and_result); + try std.testing.expect(evidence.f32_to_u32_source_and_result); + + for ([_]u8{ 0, 1, 17, 64, 127, 191 }) |input| { + const got = harness.runCode(std.testing.io, allocator, compiled.code, &.{input}, harness.qemu_user) catch |err| switch (err) { + error.SkipZigTest => return error.SkipZigTest, + else => return err, + }; + try std.testing.expectEqual(@as(i64, pressureReference(input)), got); + } +} + +test "qemu-user-riscv nvfp4 production expansion executes with integer and float spills" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + try buildNvFp4SpillPressure(&func); + + var evidence: isel.LowFloatSpillEvidence = .{}; + var compiled = try isel.compileFunctionWithLowFloatSpillEvidence(allocator, &func, .{}, &evidence); + defer compiled.deinit(allocator); + try std.testing.expect(evidence.int_spill_count > 0); + try std.testing.expect(evidence.float_spill_count > 0); + try std.testing.expect(evidence.split_action_count > 0); + try std.testing.expect(evidence.select_condition_spilled_at_use); + try std.testing.expect(evidence.select_then_spilled_at_use); + try std.testing.expect(evidence.select_else_spilled_at_use); + try std.testing.expect(evidence.select_result_has_slot); + try std.testing.expect(evidence.select_operands_and_result); + try std.testing.expect(evidence.u32_to_f32_source_and_result); + try std.testing.expect(evidence.f32_to_u32_source_and_result); + + for ([_]u8{ 0, 1, 17, 64, 127, 191 }) |input| { + const got = harness.runCode(std.testing.io, allocator, compiled.code, &.{input}, harness.qemu_user) catch |err| switch (err) { + error.SkipZigTest => return error.SkipZigTest, + else => return err, + }; + try std.testing.expectEqual(@as(i64, nvfp4PressureReference(input)), got); + } +} + +fn appendU32Immediate(allocator: std.mem.Allocator, words: *std.ArrayList(u32), register: encode.Reg, value: u32) !void { + const high: u20 = @truncate((value +% 0x800) >> 12); + const low: i12 = @bitCast(@as(u12, @truncate(value))); + try words.append(allocator, encode.lui(register, high)); + try words.append(allocator, encode.addi(register, register, low)); +} + +fn instructionOffset(from: usize, to: usize) i13 { + return @intCast((@as(isize, @intCast(to)) - @as(isize, @intCast(from))) * 4); +} + +fn runDivisionBatch( + io: std.Io, + allocator: std.mem.Allocator, + division_code: []const u32, + pairs: []const division_cases.Pair, + first_corpus_index: u32, +) !u64 { + std.debug.assert(pairs.len > 0); + std.debug.assert(pairs.len <= 16 * 1024); + + var wrapper: std.ArrayList(u32) = .empty; + defer wrapper.deinit(allocator); + + const data_address_index = wrapper.items.len; + try wrapper.append(allocator, encode.auipc(.x8, 0)); + try wrapper.append(allocator, encode.addi(.x8, .x8, 0)); + try appendU32Immediate(allocator, &wrapper, .x9, @intCast(pairs.len)); + try appendU32Immediate(allocator, &wrapper, .x21, first_corpus_index); + try wrapper.append(allocator, encode.addi(.x18, .x0, 0)); + try wrapper.append(allocator, encode.addi(.x19, .x0, 0)); + try appendU32Immediate(allocator, &wrapper, .x22, 0x7f80_0000); + try appendU32Immediate(allocator, &wrapper, .x24, 0x7fc0_0000); + + const loop_index = wrapper.items.len; + try wrapper.append(allocator, encode.lwu(.x10, .x8, 0)); + try wrapper.append(allocator, encode.lwu(.x11, .x8, 4)); + try wrapper.append(allocator, encode.lwu(.x20, .x8, 8)); + const call_index = wrapper.items.len; + try wrapper.append(allocator, encode.jal(.x1, 0)); + try wrapper.append(allocator, encode.slli(.x5, .x10, 33)); + try wrapper.append(allocator, encode.srli(.x5, .x5, 33)); + const nan_branch_index = wrapper.items.len; + try wrapper.append(allocator, encode.bltu(.x22, .x5, 0)); + const mismatch_branch_index = wrapper.items.len; + try wrapper.append(allocator, encode.bne(.x10, .x20, 0)); + + const continue_index = wrapper.items.len; + try wrapper.append(allocator, encode.addi(.x8, .x8, 12)); + try wrapper.append(allocator, encode.addi(.x9, .x9, -1)); + try wrapper.append(allocator, encode.addi(.x21, .x21, 1)); + try wrapper.append(allocator, encode.bne(.x9, .x0, instructionOffset(wrapper.items.len, loop_index))); + try wrapper.append(allocator, encode.slli(.x19, .x19, 32)); + try wrapper.append(allocator, encode.or_(.x10, .x19, .x18)); + try wrapper.append(allocator, encode.addi(.x2, .x2, -16)); + try wrapper.append(allocator, encode.sd(.x10, .x2, 0)); + try wrapper.append(allocator, encode.addi(.x10, .x0, 1)); + try wrapper.append(allocator, encode.addi(.x11, .x2, 0)); + try wrapper.append(allocator, encode.addi(.x12, .x0, 8)); + try wrapper.append(allocator, encode.addi(.x17, .x0, 64)); + try wrapper.append(allocator, encode.ecall()); + try wrapper.append(allocator, encode.addi(.x10, .x0, 0)); + try wrapper.append(allocator, encode.addi(.x17, .x0, 93)); + try wrapper.append(allocator, encode.ecall()); + + const mismatch_index = wrapper.items.len; + try wrapper.append(allocator, encode.addi(.x18, .x18, 1)); + try wrapper.append(allocator, encode.bne(.x19, .x0, instructionOffset(wrapper.items.len, continue_index))); + try wrapper.append(allocator, encode.addi(.x19, .x21, 0)); + try wrapper.append(allocator, encode.jal(.x0, instructionOffset(wrapper.items.len, continue_index))); + + const nan_index = wrapper.items.len; + try wrapper.append(allocator, encode.addi(.x10, .x24, 0)); + try wrapper.append(allocator, encode.jal(.x0, instructionOffset(wrapper.items.len, mismatch_branch_index))); + + const function_index = wrapper.items.len; + wrapper.items[call_index] = encode.jal(.x1, instructionOffset(call_index, function_index)); + wrapper.items[nan_branch_index] = encode.bltu(.x22, .x5, instructionOffset(nan_branch_index, nan_index)); + wrapper.items[mismatch_branch_index] = encode.bne(.x10, .x20, instructionOffset(mismatch_branch_index, mismatch_index)); + + var words: std.ArrayList(u32) = .empty; + defer words.deinit(allocator); + try words.appendSlice(allocator, wrapper.items); + try words.appendSlice(allocator, division_code); + const data_index = words.items.len; + const data_offset: i64 = @as(i64, @intCast((data_index - data_address_index) * 4)); + const data_high: u20 = @intCast((data_offset + 0x800) >> 12); + const data_low: i12 = @intCast(data_offset - (@as(i64, data_high) << 12)); + words.items[data_address_index] = encode.auipc(.x8, data_high); + words.items[data_address_index + 1] = encode.addi(.x8, .x8, data_low); + + var image: std.ArrayList(u8) = .empty; + defer image.deinit(allocator); + const code_bytes = try emit.emitBytes(allocator, words.items); + defer allocator.free(code_bytes); + try image.appendSlice(allocator, code_bytes); + for (pairs) |pair| { + const numerator: f32 = @bitCast(pair.numerator); + const denominator: f32 = @bitCast(pair.denominator); + const reference = numerator / denominator; + const expected: u32 = if (std.math.isNan(reference)) 0x7fc0_0000 else @bitCast(reference); + var record: [12]u8 = undefined; + std.mem.writeInt(u32, record[0..4], pair.numerator, .little); + std.mem.writeInt(u32, record[4..8], pair.denominator, .little); + std.mem.writeInt(u32, record[8..12], expected, .little); + try image.appendSlice(allocator, &record); + } + + const user_base: u64 = 0x10000; + const elf = try (@import("vulcan-link")).writeElfExec(.riscv64, allocator, image.items, image.items.len, user_base, user_base); + defer allocator.free(elf); + var temporary = std.testing.tmpDir(.{}); + defer temporary.cleanup(); + try temporary.dir.writeFile(io, .{ .sub_path = "firmware.elf", .data = elf, .flags = .{ .permissions = .executable_file } }); + const arguments = try harness.qemu_user.buildArgv(allocator, "firmware.elf"); + defer allocator.free(arguments); + const result = std.process.run(allocator, io, .{ .argv = arguments, .cwd = .{ .dir = temporary.dir } }) catch |err| switch (err) { + error.FileNotFound => return error.SkipZigTest, + else => return err, + }; + defer allocator.free(result.stdout); + defer allocator.free(result.stderr); + if (result.stdout.len < 8) return error.BackendFailed; + return std.mem.readInt(u64, result.stdout[result.stdout.len - 8 ..][0..8], .little); +} + +test "qemu-user-riscv executes shared audited exact f32 division corpus in bounded batches" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const numerator_bits = try func.appendBlockParam(block, u32_t); + const denominator_bits = try func.appendBlockParam(block, u32_t); + const numerator = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = numerator_bits } }); + const denominator = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = denominator_bits } }); + const quotient = try func.appendInst(block, f32_t, .{ .arith = .{ .op = .div, .lhs = numerator, .rhs = denominator } }); + const result = try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = quotient } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + try std.testing.expect(try ir.expand.expandF32Div(allocator, &func)); + var division_code = try harness.compileFunc(allocator, &func); + defer division_code.deinit(allocator); + var corpus = try division_cases.build(allocator); + defer corpus.deinit(); + const batch_size = 16 * 1024; + var batch_start: usize = 0; + while (batch_start < corpus.pairs.len) : (batch_start += batch_size) { + const batch_end = @min(batch_start + batch_size, corpus.pairs.len); + const evidence = runDivisionBatch( + std.testing.io, + allocator, + division_code.items, + corpus.pairs[batch_start..batch_end], + @intCast(batch_start + 1), + ) catch |err| switch (err) { + error.SkipZigTest => return error.SkipZigTest, + else => return err, + }; + const MismatchEvidence = struct { count: u32, first_corpus_index: u32 }; + try std.testing.expectEqual( + MismatchEvidence{ .count = 0, .first_corpus_index = 0 }, + MismatchEvidence{ .count = @truncate(evidence), .first_corpus_index = @truncate(evidence >> 32) }, + ); + } +} + test "qemu-user-riscv: shared codegen and optimization cases" { try cases.runAll(std.testing.io, std.testing.allocator, harness.qemu_user); } diff --git a/libs/vulcan-target/tests/cross_target.zig b/libs/vulcan-target/tests/cross_target.zig index 11274db..5cba493 100644 --- a/libs/vulcan-target/tests/cross_target.zig +++ b/libs/vulcan-target/tests/cross_target.zig @@ -32,6 +32,476 @@ fn buildMain(allocator: std.mem.Allocator) !Function { return f; } +fn buildNvFp4( + allocator: std.mem.Allocator, + dequantize: bool, + block_application: ir.nvfp4.ScaleApplication, + global_application: ir.nvfp4.ScaleApplication, + later_barrier: bool, +) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const value = try func.appendBlockParam(block, f32_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: ir.function.NvFp4Convert = .{ + .value = if (dequantize) payload else value, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = block_application, + .global_application = global_application, + }; + const result = try func.appendInst(block, if (dequantize) f32_t else u8_t, if (dequantize) .{ .dequantize_nvfp4 = conversion } else .{ .quantize_nvfp4 = conversion }); + try func.addAttr(.{ .inst = func.definingInst(result).? }, .{ .custom = .{ .namespace = "debug", .key = "nvfp4", .value = .{ .int = 42 } } }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ .namespace = "test", .key = "live", .value = .flag } }); + if (later_barrier) try func.appendBarrier(block, .workgroup); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +test "x86 rejects both nvfp4 directions without mutating caller IR" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |dequantize| { + var func = try buildNvFp4(allocator, dequantize, .multiply, .divide, false); + defer func.deinit(); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + try std.testing.expectError(error.Unsupported, target.native.writeObjectDataFor(allocator, .x86, &.{.{ .name = "convert", .func = &func }}, &.{})); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try std.testing.expectEqualSlices(u8, before, after); + } +} + +test "supported scalar targets compile every nvfp4 policy and direction without mutation" { + const allocator = std.testing.allocator; + inline for ([_]ir.nvfp4.ScaleApplication{ .multiply, .divide }) |block_application| { + inline for ([_]ir.nvfp4.ScaleApplication{ .multiply, .divide }) |global_application| { + inline for (.{ true, false }) |dequantize| { + var func = try buildNvFp4(allocator, dequantize, block_application, global_application, false); + defer func.deinit(); + const text_before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + inline for (.{ .x86_64, .aarch64, .riscv64 }) |arch| { + const object = try target.native.writeObjectDataFor(allocator, arch, &.{.{ .name = "convert", .func = &func }}, &.{}); + allocator.free(object); + const text_after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try std.testing.expectEqualStrings(text_before, text_after); + try std.testing.expectEqualSlices(u8, bits_before, bits_after); + } + } + } + } +} + +test "supported nvfp4 expansion reaches a later backend error without caller mutation" { + const allocator = std.testing.allocator; + inline for (.{ .x86_64, .aarch64, .riscv64 }) |arch| { + var func = try buildNvFp4(allocator, true, .multiply, .divide, true); + defer func.deinit(); + const text_before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + try std.testing.expectError(error.Unsupported, target.native.writeObjectDataFor(allocator, arch, &.{.{ .name = "convert", .func = &func }}, &.{})); + const text_after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try std.testing.expectEqualStrings(text_before, text_after); + try std.testing.expectEqualSlices(u8, bits_before, bits_after); + } +} + +fn buildNvFp4IntegerAbi( + allocator: std.mem.Allocator, + dequantize: bool, + block_application: ir.nvfp4.ScaleApplication, + global_application: ir.nvfp4.ScaleApplication, +) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const input_bits = try func.appendBlockParam(block, u32_t); + const block_bits = try func.appendBlockParam(block, u32_t); + const global_bits = try func.appendBlockParam(block, u32_t); + const carrier = try func.appendInst(block, u8_t, .{ .convert = .{ .value = input_bits } }); + const block_scale = try func.appendInst(block, u8_t, .{ .convert = .{ .value = block_bits } }); + const input_f32 = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = input_bits } }); + const global_scale = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = global_bits } }); + const conversion: ir.function.NvFp4Convert = .{ + .value = if (dequantize) carrier else input_f32, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = block_application, + .global_application = global_application, + }; + const converted = try func.appendInst(block, if (dequantize) f32_t else u8_t, if (dequantize) .{ .dequantize_nvfp4 = conversion } else .{ .quantize_nvfp4 = conversion }); + const result = try func.appendInst(block, u32_t, if (dequantize) .{ .unary = .{ .op = .reinterpret, .value = converted } } else .{ .convert = .{ .value = converted } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +fn buildReorderedNvFp4IntegerAbi( + allocator: std.mem.Allocator, + dequantize: bool, + block_application: ir.nvfp4.ScaleApplication, + global_application: ir.nvfp4.ScaleApplication, +) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const input_bits = try func.appendBlockParam(block, u32_t); + const block_bits = try func.appendBlockParam(block, u32_t); + const global_bits = try func.appendBlockParam(block, u32_t); + const carrier = try func.appendInst(block, u8_t, .{ .convert = .{ .value = input_bits } }); + const block_payload = try func.appendInst(block, u8_t, .{ .convert = .{ .value = block_bits } }); + const input_f32 = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = input_bits } }); + const global_scale = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = global_bits } }); + const block_scale = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .format = .f8_e4m3, .value = block_payload } }); + const unit_payload = try func.appendInst(block, u8_t, .{ .iconst = 0x38 }); + const unit_scale = try func.appendInst(block, f32_t, .{ .fconst = 1.0 }); + + const result = if (dequantize) result: { + const global_first = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = carrier, + .block_scale = unit_payload, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = global_application, + } }); + const reordered = try func.appendInst(block, f32_t, .{ .arith = .{ + .op = if (block_application == .multiply) .mul else .div, + .lhs = global_first, + .rhs = block_scale, + } }); + break :result try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = reordered } }); + } else result: { + const block_first = try func.appendInst(block, f32_t, .{ .arith = .{ + .op = if (block_application == .multiply) .div else .mul, + .lhs = input_f32, + .rhs = block_scale, + } }); + const global_second = try func.appendInst(block, f32_t, .{ .arith = .{ + .op = if (global_application == .multiply) .div else .mul, + .lhs = block_first, + .rhs = global_scale, + } }); + const payload = try func.appendInst(block, u8_t, .{ .quantize_nvfp4 = .{ + .value = global_second, + .block_scale = unit_payload, + .global_scale = unit_scale, + .block_application = .multiply, + .global_application = .multiply, + } }); + break :result try func.appendInst(block, u32_t, .{ .convert = .{ .value = payload } }); + }; + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +fn buildNvFp4ArithmeticBoundary(allocator: std.mem.Allocator, op: ir.function.BinOp) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const carrier_bits = try func.appendBlockParam(block, u32_t); + const block_bits = try func.appendBlockParam(block, u32_t); + const global_bits = try func.appendBlockParam(block, u32_t); + const rhs_bits = try func.appendBlockParam(block, u32_t); + const carrier = try func.appendInst(block, u8_t, .{ .convert = .{ .value = carrier_bits } }); + const block_scale = try func.appendInst(block, u8_t, .{ .convert = .{ .value = block_bits } }); + const global_scale = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = global_bits } }); + const rhs = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = rhs_bits } }); + const decoded = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = carrier, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .multiply, + } }); + const combined = try func.appendInst(block, f32_t, .{ .arith = .{ .op = op, .lhs = decoded, .rhs = rhs } }); + const result = try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = combined } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +fn buildNvFp4ArithmeticBoundaryMain( + allocator: std.mem.Allocator, + op: ir.function.BinOp, + carrier_bits: u32, + block_bits: u32, + global_bits: u32, + rhs_bits: u32, + expected_bits: u32, +) !Function { + var func = Function.init(allocator); + errdefer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const carrier = try func.appendInst(block, u8_t, .{ .iconst = carrier_bits }); + const block_scale = try func.appendInst(block, u8_t, .{ .iconst = block_bits }); + const global_scale = try func.appendInst(block, f32_t, .{ .fconst = @as(f32, @bitCast(global_bits)) }); + const rhs = try func.appendInst(block, f32_t, .{ .fconst = @as(f32, @bitCast(rhs_bits)) }); + const decoded = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = carrier, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .multiply, + } }); + const combined = try func.appendInst(block, f32_t, .{ .arith = .{ .op = op, .lhs = decoded, .rhs = rhs } }); + const actual = try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = combined } }); + const expected = try func.appendInst(block, u32_t, .{ .iconst = expected_bits }); + const difference = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = actual, .rhs = expected } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(difference) }); + return func; +} + +test "native nvfp4 integer-ABI wrappers execute exact reference conversions" { + if (builtin.cpu.arch != .x86_64 and builtin.cpu.arch != .aarch64 and builtin.cpu.arch != .riscv64) return error.SkipZigTest; + const allocator = std.testing.allocator; + const ConvertFn = *const fn (u32, u32, u32) callconv(.c) u32; + const policies = [_]ir.nvfp4.ScaleApplication{ .multiply, .divide }; + const globals = [_]u32{ 0x0000_0000, 0xbf80_0000, 0x3f00_0000, 0x7f80_0000, 0x7fc0_1234 }; + const dequantize_order_cases = [_]struct { payload: u8, block_scale: u8, global_bits: u32, block: ir.nvfp4.ScaleApplication, global: ir.nvfp4.ScaleApplication, expected: u32, reordered: u32 }{ + .{ .payload = 0x03, .block_scale = 0x03, .global_bits = 0x3e00_0001, .block = .multiply, .global = .multiply, .expected = 0x3a90_0001, .reordered = 0x3a90_0002 }, + .{ .payload = 0x01, .block_scale = 0x03, .global_bits = 0x3e00_0001, .block = .multiply, .global = .divide, .expected = 0x3cbf_ffff, .reordered = 0x3cbf_fffe }, + .{ .payload = 0x01, .block_scale = 0x05, .global_bits = 0x3e00_0001, .block = .divide, .global = .multiply, .expected = 0x40cc_cccf, .reordered = 0x40cc_ccce }, + .{ .payload = 0x01, .block_scale = 0x03, .global_bits = 0x3e00_0001, .block = .divide, .global = .divide, .expected = 0x442a_aaaa, .reordered = 0x442a_aaa9 }, + }; + const quantize_order_cases = [_]struct { value_bits: u32, block_scale: u8, global_bits: u32, block: ir.nvfp4.ScaleApplication, global: ir.nvfp4.ScaleApplication, expected: u8, reordered: u8 }{ + .{ .value_bits = 0x3e00_0522, .block_scale = 0x23, .global_bits = 0x3e54_d004, .block = .multiply, .global = .multiply, .expected = 0x06, .reordered = 0x05 }, + .{ .value_bits = 0x3e00_154f, .block_scale = 0x0f, .global_bits = 0x3ed1_dd10, .block = .multiply, .global = .divide, .expected = 0x04, .reordered = 0x03 }, + .{ .value_bits = 0x3e00_154f, .block_scale = 0x47, .global_bits = 0x3f20_1aa3, .block = .divide, .global = .multiply, .expected = 0x01, .reordered = 0x02 }, + .{ .value_bits = 0x3e00_399b, .block_scale = 0x55, .global_bits = 0x3eeb_e49b, .block = .divide, .global = .divide, .expected = 0x01, .reordered = 0x02 }, + }; + + inline for (policies) |block_application| inline for (policies) |global_application| { + var dequantize = try buildNvFp4IntegerAbi(allocator, true, block_application, global_application); + defer dequantize.deinit(); + var dequant_code = try target.native.jitFunction(allocator, &dequantize); + defer dequant_code.deinit(); + const dequant_fn = dequant_code.entry(ConvertFn, 0); + for (globals) |global_bits| for (0..256) |carrier| for (0..256) |block_scale| { + const expected = ir.nvfp4.dequantize(@intCast(carrier), @intCast(block_scale), @bitCast(global_bits), block_application, global_application); + const actual: f32 = @bitCast(dequant_fn(@intCast(carrier), @intCast(block_scale), global_bits)); + if (std.math.isNan(expected)) { + try std.testing.expect(std.math.isNan(actual)); + } else { + try std.testing.expectEqual(@as(u32, @bitCast(expected)), @as(u32, @bitCast(actual))); + } + }; + for (dequantize_order_cases) |case| if (case.block == block_application and case.global == global_application) { + try std.testing.expectEqual(case.expected, dequant_fn(case.payload, case.block_scale, case.global_bits)); + }; + var reordered_dequantize = try buildReorderedNvFp4IntegerAbi(allocator, true, block_application, global_application); + defer reordered_dequantize.deinit(); + var reordered_dequant_code = try target.native.jitFunction(allocator, &reordered_dequantize); + defer reordered_dequant_code.deinit(); + const reordered_dequant_fn = reordered_dequant_code.entry(ConvertFn, 0); + for (dequantize_order_cases) |case| if (case.block == block_application and case.global == global_application) { + const actual = reordered_dequant_fn(case.payload, case.block_scale, case.global_bits); + try std.testing.expectEqual(case.reordered, actual); + try std.testing.expect(actual != case.expected); + }; + + var quantize = try buildNvFp4IntegerAbi(allocator, false, block_application, global_application); + defer quantize.deinit(); + var quant_code = try target.native.jitFunction(allocator, &quantize); + defer quant_code.deinit(); + const quant_fn = quant_code.entry(ConvertFn, 0); + + // Unit scales make every policy pair classify the supplied value itself. + // Pin every exact E2M1 value and both signs of every midpoint with its two + // adjacent bit patterns, so a changed tie comparison fails generated code. + for (0..16) |payload| { + const value_bits: u32 = @bitCast(ir.nvfp4.decodeE2M1(@intCast(payload))); + try std.testing.expectEqual(@as(u32, @intCast(payload)), quant_fn(value_bits, 0x38, 0x3f80_0000)); + } + const midpoint_cases = [_]struct { bits: u32, lower: u8, tie: u8, upper: u8 }{ + .{ .bits = 0x3e80_0000, .lower = 0, .tie = 0, .upper = 1 }, + .{ .bits = 0x3f40_0000, .lower = 1, .tie = 2, .upper = 2 }, + .{ .bits = 0x3fa0_0000, .lower = 2, .tie = 2, .upper = 3 }, + .{ .bits = 0x3fe0_0000, .lower = 3, .tie = 4, .upper = 4 }, + .{ .bits = 0x4020_0000, .lower = 4, .tie = 4, .upper = 5 }, + .{ .bits = 0x4060_0000, .lower = 5, .tie = 6, .upper = 6 }, + .{ .bits = 0x40a0_0000, .lower = 6, .tie = 6, .upper = 7 }, + }; + for (midpoint_cases) |case| inline for (.{ @as(u32, 0), @as(u32, 0x8000_0000) }) |sign| { + try std.testing.expectEqual(@as(u32, case.lower | @as(u8, @truncate(sign >> 28))), quant_fn((case.bits - 1) | sign, 0x38, 0x3f80_0000)); + try std.testing.expectEqual(@as(u32, case.tie | @as(u8, @truncate(sign >> 28))), quant_fn(case.bits | sign, 0x38, 0x3f80_0000)); + try std.testing.expectEqual(@as(u32, case.upper | @as(u8, @truncate(sign >> 28))), quant_fn((case.bits + 1) | sign, 0x38, 0x3f80_0000)); + }; + + const values = [_]u32{ + 0x0000_0000, 0x8000_0000, 0x0000_0001, 0x8000_0001, + 0x3e80_0000, 0x3f40_0000, 0x3fa0_0000, 0x3fe0_0000, + 0x4020_0000, 0x4060_0000, 0x40a0_0000, 0x7f7f_ffff, + 0xff7f_ffff, 0x7f80_0000, 0xff80_0000, 0x7f80_0001, + 0xff80_0001, 0x7fc0_1234, 0xffc0_1234, + }; + const block_scales = [_]u8{ 0x00, 0x38, 0xb8, 0x7e, 0x7f }; + for (values) |value_bits| for (block_scales) |block_scale| for (globals) |global_bits| { + const expected = ir.nvfp4.quantize(@bitCast(value_bits), block_scale, @bitCast(global_bits), block_application, global_application); + try std.testing.expectEqual(@as(u32, expected), quant_fn(value_bits, block_scale, global_bits)); + }; + for (quantize_order_cases) |case| if (case.block == block_application and case.global == global_application) { + try std.testing.expectEqual(@as(u32, case.expected), quant_fn(case.value_bits, case.block_scale, case.global_bits)); + }; + var reordered_quantize = try buildReorderedNvFp4IntegerAbi(allocator, false, block_application, global_application); + defer reordered_quantize.deinit(); + var reordered_quant_code = try target.native.jitFunction(allocator, &reordered_quantize); + defer reordered_quant_code.deinit(); + const reordered_quant_fn = reordered_quant_code.entry(ConvertFn, 0); + for (quantize_order_cases) |case| if (case.block == block_application and case.global == global_application) { + const actual = reordered_quant_fn(case.value_bits, case.block_scale, case.global_bits); + try std.testing.expectEqual(@as(u32, case.reordered), actual); + try std.testing.expect(actual != case.expected); + }; + }; +} + +test "native nvfp4 dequantize keeps the rounding boundary before add and subtract" { + if (builtin.cpu.arch != .x86_64 and builtin.cpu.arch != .aarch64 and builtin.cpu.arch != .riscv64) return error.SkipZigTest; + const allocator = std.testing.allocator; + const BoundaryFn = *const fn (u32, u32, u32, u32) callconv(.c) u32; + const cases = [_]struct { op: ir.function.BinOp, carrier: u32, block: u32, global: u32, rhs: u32, expected: u32, fused: u32 }{ + .{ .op = .add, .carrier = 0x01, .block = 0x34, .global = 0x3e00_0001, .rhs = 0x3e00_0001, .expected = 0x3e30_0002, .fused = 0x3e30_0001 }, + .{ .op = .sub, .carrier = 0x01, .block = 0x31, .global = 0x3e00_0001, .rhs = 0x3e00_0001, .expected = 0xbdb8_0002, .fused = 0xbdb8_0001 }, + }; + for (cases) |case| { + var func = try buildNvFp4ArithmeticBoundary(allocator, case.op); + defer func.deinit(); + var code = try target.native.jitFunction(allocator, &func); + defer code.deinit(); + const actual = code.entry(BoundaryFn, 0)(case.carrier, case.block, case.global, case.rhs); + try std.testing.expectEqual(case.expected, actual); + try std.testing.expect(actual != case.fused); + } +} + +test "native nvfp4 packed memory crosses a scale block and preserves its tail sibling" { + if (builtin.cpu.arch != .x86_64 and builtin.cpu.arch != .aarch64 and builtin.cpu.arch != .riscv64) return error.SkipZigTest; + const allocator = std.testing.allocator; + const element_count = 19; + const packed_count = (element_count + 1) / 2; + var func = Function.init(allocator); + defer func.deinit(); + const ptr_t = try func.types.ptrGlobal(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const packed_input = try func.appendBlockParam(block, ptr_t); + const scales = try func.appendBlockParam(block, ptr_t); + const global_bits = try func.appendBlockParam(block, u32_t); + const decoded_output = try func.appendBlockParam(block, ptr_t); + const float_input = try func.appendBlockParam(block, ptr_t); + const packed_output = try func.appendBlockParam(block, ptr_t); + const global_scale = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = global_bits } }); + for (0..element_count) |index| { + const packed_at = if (index < 2) packed_input else try func.appendArithImm(block, ptr_t, .add, packed_input, @intCast(index / 2)); + const scale_at = if (index < 16) scales else try func.appendArithImm(block, ptr_t, .add, scales, @intCast(index / 16)); + const packed_byte = try func.appendInst(block, u8_t, .{ .load = .{ .ptr = packed_at } }); + const packed_wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = packed_byte } }); + const positioned = if (index & 1 == 0) + packed_wide + else + try func.appendArithImm(block, u32_t, .shr, packed_wide, 4); + const nibble_wide = try func.appendArithImm(block, u32_t, .bit_and, positioned, 0x0f); + const nibble = try func.appendInst(block, u8_t, .{ .convert = .{ .value = nibble_wide } }); + const block_scale = try func.appendInst(block, u8_t, .{ .load = .{ .ptr = scale_at } }); + const decoded = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = nibble, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .multiply, + .global_application = .divide, + } }); + const decoded_at = if (index == 0) decoded_output else try func.appendArithImm(block, ptr_t, .add, decoded_output, @intCast(index * 4)); + try func.appendStore(block, decoded, decoded_at); + + const float_at = if (index == 0) float_input else try func.appendArithImm(block, ptr_t, .add, float_input, @intCast(index * 4)); + const source = try func.appendInst(block, f32_t, .{ .load = .{ .ptr = float_at } }); + const encoded = try func.appendInst(block, u8_t, .{ .quantize_nvfp4 = .{ + .value = source, + .block_scale = block_scale, + .global_scale = global_scale, + .block_application = .divide, + .global_application = .multiply, + } }); + const encoded_wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = encoded } }); + const output_at = if (index < 2) packed_output else try func.appendArithImm(block, ptr_t, .add, packed_output, @intCast(index / 2)); + const old_byte = try func.appendInst(block, u8_t, .{ .load = .{ .ptr = output_at } }); + const old_wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = old_byte } }); + const preserved = try func.appendArithImm(block, u32_t, .bit_and, old_wide, if (index & 1 == 0) 0xf0 else 0x0f); + const placed = if (index & 1 == 0) + encoded_wide + else + try func.appendArithImm(block, u32_t, .shl, encoded_wide, 4); + const combined = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_or, .lhs = preserved, .rhs = placed } }); + const output_byte = try func.appendInst(block, u8_t, .{ .convert = .{ .value = combined } }); + try func.appendStore(block, output_byte, output_at); + } + func.setTerminator(block, .{ .ret = ir.function.Ret.none() }); + + var code = try target.native.jitFunction(allocator, &func); + defer code.deinit(); + const run = code.entry(*const fn (*const u8, *const u8, u32, *u32, *const u32, *u8) callconv(.c) void, 0); + const packed_values = [_]u8{ 0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe, 0x21, 0x43 }; + const scale_values = [_]u8{ 0x39, 0xb8 }; + const global_scale_bits: u32 = 0x3f40_0000; + const source_bits = [_]u32{ + 0x0000_0000, 0x8000_0000, 0x3e80_0000, 0x3e80_0001, 0x3f40_0000, + 0x3f40_0001, 0x3fa0_0000, 0x3fa0_0001, 0x3fe0_0000, 0x4020_0000, + 0x4060_0000, 0x40a0_0000, 0x7f7f_ffff, 0xff7f_ffff, 0x7f80_0000, + 0xff80_0000, 0x7f80_0001, 0xff80_0001, 0x7fc0_1234, + }; + var decoded_bits = [_]u32{0} ** element_count; + var output = [_]u8{0xa5} ** packed_count; + var expected_output = output; + run(&packed_values[0], &scale_values[0], global_scale_bits, &decoded_bits[0], &source_bits[0], &output[0]); + const global_scale_value: f32 = @bitCast(global_scale_bits); + for (0..element_count) |index| { + const payload = if (index & 1 == 0) packed_values[index / 2] & 0x0f else packed_values[index / 2] >> 4; + const scale = scale_values[index / 16]; + const expected_decoded = ir.nvfp4.dequantize(payload, scale, global_scale_value, .multiply, .divide); + if (std.math.isNan(expected_decoded)) { + try std.testing.expect(std.math.isNan(@as(f32, @bitCast(decoded_bits[index])))); + } else { + try std.testing.expectEqual(@as(u32, @bitCast(expected_decoded)), decoded_bits[index]); + } + const encoded = ir.nvfp4.quantize(@bitCast(source_bits[index]), scale, global_scale_value, .divide, .multiply); + if (index & 1 == 0) { + expected_output[index / 2] = (expected_output[index / 2] & 0xf0) | encoded; + } else { + expected_output[index / 2] = (expected_output[index / 2] & 0x0f) | (encoded << 4); + } + } + try std.testing.expectEqualSlices(u8, &expected_output, &output); + try std.testing.expectEqual(@as(u8, 0xa0), output[packed_count - 1] & 0xf0); +} + /// Run `argv[0]`, already on PATH, or a native `./a.elf`, against `elf` written to a /// fresh tmp dir, and return its exit code. Returns `error.SkipZigTest` when the /// runner, `qemu-` when `argv[0]` names one, is not installed. @@ -231,6 +701,46 @@ test "cross-target: writeObjectDataFor(.riscv64, ...) emits+links+runs to exit 4 try std.testing.expectEqual(@as(u8, 42), code); } +fn runRiscvMain(allocator: std.mem.Allocator, func: *const Function) !u8 { + const obj = try target.native.writeObjectDataFor(allocator, .riscv64, &.{.{ .name = "main", .func = func }}, &.{}); + defer allocator.free(obj); + const encode = target.riscv64.encode; + const stub_len: u64 = 12; + const base: u64 = 0x10000; + var image = try ld.linkObjects(allocator, &.{obj}, base + stub_len); + defer image.deinit(allocator); + const main_addr: i64 = @intCast(image.addressOf("main").?); + const stub = [_]u32{ + encode.jal(.x1, @intCast(main_addr - @as(i64, @intCast(base)))), + encode.addi(.x17, .x0, 93), + encode.ecall(), + }; + var program: std.ArrayList(u8) = .empty; + defer program.deinit(allocator); + try program.appendSlice(allocator, std.mem.sliceAsBytes(&stub)); + try program.appendSlice(allocator, image.code); + const elf = try ld.writeElfExec(.riscv64, allocator, program.items, stub_len + image.memsz, base, base); + defer allocator.free(elf); + return runElf(allocator, std.testing.io, elf, &.{ "qemu-riscv64", "./a.elf" }); +} + +test "qemu-riscv64 nvfp4 retains separate rounding before add and subtract" { + const allocator = std.testing.allocator; + const cases = [_]struct { op: ir.function.BinOp, carrier: u32, block: u32, global: u32, rhs: u32, expected: u32 }{ + .{ .op = .add, .carrier = 0x01, .block = 0x34, .global = 0x3e00_0001, .rhs = 0x3e00_0001, .expected = 0x3e30_0002 }, + .{ .op = .sub, .carrier = 0x01, .block = 0x31, .global = 0x3e00_0001, .rhs = 0x3e00_0001, .expected = 0xbdb8_0002 }, + }; + for (cases) |case| { + var func = try buildNvFp4ArithmeticBoundaryMain(allocator, case.op, case.carrier, case.block, case.global, case.rhs, case.expected); + defer func.deinit(); + const exit_code = runRiscvMain(allocator, &func) catch |err| switch (err) { + error.SkipZigTest => return error.SkipZigTest, + else => return err, + }; + try std.testing.expectEqual(@as(u8, 0), exit_code); + } +} + /// `main(x)` in the shape that made the x86_64 register allocator refuse a function: `keep` is an /// argument of the call AND is read after the call, twelve other values are live across the same /// call, and six more arguments die at it. Every gpr the pool holds is then wanted at the call diff --git a/libs/vulcan-target/tests/f32_division_cases.zig b/libs/vulcan-target/tests/f32_division_cases.zig new file mode 100644 index 0000000..20910c9 --- /dev/null +++ b/libs/vulcan-target/tests/f32_division_cases.zig @@ -0,0 +1,184 @@ +const std = @import("std"); + +pub const Pair = struct { + numerator: u32, + denominator: u32, +}; + +pub const Halfway = struct { + numerator: u32, + denominator: u32, + expected: u32, +}; + +pub const halfway_cases = [_]Halfway{ + .{ .numerator = 0x0000_0001, .denominator = 0x4000_0000, .expected = 0x0000_0000 }, + .{ .numerator = 0x0000_0005, .denominator = 0x4000_0000, .expected = 0x0000_0002 }, + .{ .numerator = 0x0000_0009, .denominator = 0x4000_0000, .expected = 0x0000_0004 }, + .{ .numerator = 0x0000_000d, .denominator = 0x4000_0000, .expected = 0x0000_0006 }, + .{ .numerator = 0x0000_0011, .denominator = 0x4000_0000, .expected = 0x0000_0008 }, + .{ .numerator = 0x0000_0015, .denominator = 0x4000_0000, .expected = 0x0000_000a }, + .{ .numerator = 0x0000_0019, .denominator = 0x4000_0000, .expected = 0x0000_000c }, + .{ .numerator = 0x0000_001d, .denominator = 0x4000_0000, .expected = 0x0000_000e }, + .{ .numerator = 0x0000_0021, .denominator = 0x4000_0000, .expected = 0x0000_0010 }, + .{ .numerator = 0x0000_0025, .denominator = 0x4000_0000, .expected = 0x0000_0012 }, + .{ .numerator = 0x0000_0029, .denominator = 0x4000_0000, .expected = 0x0000_0014 }, + .{ .numerator = 0x0000_002d, .denominator = 0x4000_0000, .expected = 0x0000_0016 }, + .{ .numerator = 0x0000_0031, .denominator = 0x4000_0000, .expected = 0x0000_0018 }, + .{ .numerator = 0x0000_0035, .denominator = 0x4000_0000, .expected = 0x0000_001a }, + .{ .numerator = 0x0000_0039, .denominator = 0x4000_0000, .expected = 0x0000_001c }, + .{ .numerator = 0x0000_003d, .denominator = 0x4000_0000, .expected = 0x0000_001e }, + .{ .numerator = 0x0000_0003, .denominator = 0x4000_0000, .expected = 0x0000_0002 }, + .{ .numerator = 0x0000_0007, .denominator = 0x4000_0000, .expected = 0x0000_0004 }, + .{ .numerator = 0x0000_000b, .denominator = 0x4000_0000, .expected = 0x0000_0006 }, + .{ .numerator = 0x0000_000f, .denominator = 0x4000_0000, .expected = 0x0000_0008 }, + .{ .numerator = 0x0000_0013, .denominator = 0x4000_0000, .expected = 0x0000_000a }, + .{ .numerator = 0x0000_0017, .denominator = 0x4000_0000, .expected = 0x0000_000c }, + .{ .numerator = 0x0000_001b, .denominator = 0x4000_0000, .expected = 0x0000_000e }, + .{ .numerator = 0x0000_001f, .denominator = 0x4000_0000, .expected = 0x0000_0010 }, + .{ .numerator = 0x0000_0023, .denominator = 0x4000_0000, .expected = 0x0000_0012 }, + .{ .numerator = 0x0000_0027, .denominator = 0x4000_0000, .expected = 0x0000_0014 }, + .{ .numerator = 0x0000_002b, .denominator = 0x4000_0000, .expected = 0x0000_0016 }, + .{ .numerator = 0x0000_002f, .denominator = 0x4000_0000, .expected = 0x0000_0018 }, + .{ .numerator = 0x0000_0033, .denominator = 0x4000_0000, .expected = 0x0000_001a }, + .{ .numerator = 0x0000_0037, .denominator = 0x4000_0000, .expected = 0x0000_001c }, + .{ .numerator = 0x0000_003b, .denominator = 0x4000_0000, .expected = 0x0000_001e }, + .{ .numerator = 0x0000_003f, .denominator = 0x4000_0000, .expected = 0x0000_0020 }, +}; + +pub const named_cases = [_]Pair{ + .{ .numerator = 0x0000_0000, .denominator = 0x3f80_0000 }, + .{ .numerator = 0x8000_0000, .denominator = 0x3f80_0000 }, + .{ .numerator = 0x0000_0001, .denominator = 0x3f80_0000 }, + .{ .numerator = 0x0000_0001, .denominator = 0x4000_0000 }, + .{ .numerator = 0x0000_0001, .denominator = 0x3fff_ffff }, + .{ .numerator = 0x0000_0001, .denominator = 0x4000_0001 }, + .{ .numerator = 0x007f_ffff, .denominator = 0x3f80_0000 }, + .{ .numerator = 0x0080_0000, .denominator = 0x3f80_0000 }, + .{ .numerator = 0x0080_0001, .denominator = 0x3f80_0000 }, + .{ .numerator = 0x0080_0000, .denominator = 0x4000_0000 }, + .{ .numerator = 0x0100_0000, .denominator = 0x4000_0000 }, + .{ .numerator = 0x0100_0001, .denominator = 0x4000_0000 }, + .{ .numerator = 0x7f7f_ffff, .denominator = 0x3f80_0000 }, + .{ .numerator = 0x7f7f_ffff, .denominator = 0x3f7f_ffff }, + .{ .numerator = 0x7f7f_fffe, .denominator = 0x3f7f_ffff }, + .{ .numerator = 0x3f80_0000, .denominator = 0x7f7f_ffff }, + .{ .numerator = 0x7f80_0000, .denominator = 0x3f80_0000 }, + .{ .numerator = 0xff80_0000, .denominator = 0x3f80_0000 }, + .{ .numerator = 0x3f80_0000, .denominator = 0x0000_0000 }, + .{ .numerator = 0xbf80_0000, .denominator = 0x0000_0000 }, +}; + +pub const Corpus = struct { + allocator: std.mem.Allocator, + pairs: []Pair, + + pub fn deinit(self: *Corpus) void { + self.allocator.free(self.pairs); + self.* = undefined; + } +}; + +fn nextRandom(state: *u64) u32 { + state.* ^= state.* >> 12; + state.* ^= state.* << 25; + state.* ^= state.* >> 27; + return @truncate(state.* *% 0x2545_f491_4f6c_dd1d); +} + +fn appendUnique(allocator: std.mem.Allocator, seen: *std.AutoHashMapUnmanaged(Pair, void), pairs: *std.ArrayList(Pair), pair: Pair) !void { + const result = try seen.getOrPut(allocator, pair); + if (!result.found_existing) try pairs.append(allocator, pair); +} + +pub fn build(allocator: std.mem.Allocator) !Corpus { + var seen: std.AutoHashMapUnmanaged(Pair, void) = .empty; + defer seen.deinit(allocator); + var pairs: std.ArrayList(Pair) = .empty; + errdefer pairs.deinit(allocator); + + const classes = [_]u32{ + 0x0000_0000, 0x8000_0000, 0x0000_0001, 0x8000_0001, + 0x007f_ffff, 0x807f_ffff, 0x0080_0000, 0x8080_0000, + 0x3f80_0000, 0xbf80_0000, 0x7f7f_ffff, 0xff7f_ffff, + 0x7f80_0000, 0xff80_0000, 0x7f80_0001, 0xff80_0001, + 0x7fc0_0001, 0xffc0_0001, + }; + for (classes) |numerator| for (classes) |denominator| { + try appendUnique(allocator, &seen, &pairs, .{ .numerator = numerator, .denominator = denominator }); + }; + + const mantissas = [_]u32{ 0, 1, 0x003f_ffff, 0x007f_fffe, 0x007f_ffff }; + for (0..256) |numerator_exponent| for (0..256) |denominator_exponent| { + for (mantissas, 0..) |numerator_mantissa, template_index| { + const denominator_mantissa = mantissas[(template_index + numerator_exponent + denominator_exponent) % mantissas.len]; + const numerator_sign: u32 = if ((numerator_exponent + denominator_exponent + template_index) & 1 == 0) 0 else 0x8000_0000; + const denominator_sign: u32 = if ((numerator_exponent * 3 + denominator_exponent + template_index) & 2 == 0) 0 else 0x8000_0000; + try appendUnique(allocator, &seen, &pairs, .{ + .numerator = numerator_sign | (@as(u32, @intCast(numerator_exponent)) << 23) | numerator_mantissa, + .denominator = denominator_sign | (@as(u32, @intCast(denominator_exponent)) << 23) | denominator_mantissa, + }); + } + }; + + for (named_cases) |case| try appendUnique(allocator, &seen, &pairs, case); + for (halfway_cases) |case| { + try appendUnique(allocator, &seen, &pairs, .{ .numerator = case.numerator, .denominator = case.denominator }); + try appendUnique(allocator, &seen, &pairs, .{ .numerator = case.numerator - 1, .denominator = case.denominator }); + try appendUnique(allocator, &seen, &pairs, .{ .numerator = case.numerator + 1, .denominator = case.denominator }); + try appendUnique(allocator, &seen, &pairs, .{ .numerator = case.numerator | 0x8000_0000, .denominator = case.denominator }); + } + + var random_state: u64 = 0x6e76_6670_3464_6976; + var random_count: usize = 0; + while (random_count < 65_536) : (random_count += 1) { + try appendUnique(allocator, &seen, &pairs, .{ .numerator = nextRandom(&random_state), .denominator = nextRandom(&random_state) }); + } + try std.testing.expectEqual(@as(usize, 65_536), random_count); + + var exponent_pairs = [_]bool{false} ** 65_536; + var numerator_coverage = [_][5][2]bool{.{.{ false, false }} ** 5} ** 256; + var denominator_coverage = [_][5][2]bool{.{.{ false, false }} ** 5} ** 256; + for (pairs.items) |pair| { + const numerator_exponent: usize = @intCast((pair.numerator >> 23) & 0xff); + const denominator_exponent: usize = @intCast((pair.denominator >> 23) & 0xff); + exponent_pairs[numerator_exponent * 256 + denominator_exponent] = true; + for (mantissas, 0..) |mantissa, template_index| { + if (pair.numerator & 0x007f_ffff == mantissa) numerator_coverage[numerator_exponent][template_index][pair.numerator >> 31] = true; + if (pair.denominator & 0x007f_ffff == mantissa) denominator_coverage[denominator_exponent][template_index][pair.denominator >> 31] = true; + } + } + for (exponent_pairs) |covered| try std.testing.expect(covered); + for (numerator_coverage) |by_template| for (by_template) |by_sign| for (by_sign) |covered| try std.testing.expect(covered); + for (denominator_coverage) |by_template| for (by_template) |by_sign| for (by_sign) |covered| try std.testing.expect(covered); + for (named_cases) |case| try std.testing.expect(seen.contains(case)); + for (halfway_cases) |case| { + try std.testing.expect(seen.contains(.{ .numerator = case.numerator, .denominator = case.denominator })); + try std.testing.expect(seen.contains(.{ .numerator = case.numerator - 1, .denominator = case.denominator })); + try std.testing.expect(seen.contains(.{ .numerator = case.numerator + 1, .denominator = case.denominator })); + try std.testing.expect(seen.contains(.{ .numerator = case.numerator | 0x8000_0000, .denominator = case.denominator })); + } + random_state = 0x6e76_6670_3464_6976; + random_count = 0; + while (random_count < 65_536) : (random_count += 1) { + try std.testing.expect(seen.contains(.{ .numerator = nextRandom(&random_state), .denominator = nextRandom(&random_state) })); + } + try std.testing.expectEqual(@as(usize, 65_536), random_count); + + return .{ .allocator = allocator, .pairs = try pairs.toOwnedSlice(allocator) }; +} + +test "division corpus retains every required generated source" { + var corpus = try build(std.testing.allocator); + defer corpus.deinit(); + try std.testing.expect(corpus.pairs.len >= 256 * 256 * 5); + for (halfway_cases) |case| { + const denominator: f32 = @bitCast(case.denominator); + try std.testing.expectEqual(case.expected, @as(u32, @bitCast(@as(f32, @bitCast(case.numerator)) / denominator))); + const below: u32 = @bitCast(@as(f32, @bitCast(case.numerator - 1)) / denominator); + const above: u32 = @bitCast(@as(f32, @bitCast(case.numerator + 1)) / denominator); + try std.testing.expect(below <= case.expected); + try std.testing.expect(above >= case.expected); + try std.testing.expect(below != above); + } +} diff --git a/libs/vulcan-target/tests/matmul_expand.zig b/libs/vulcan-target/tests/matmul_expand.zig index 67b27a4..268dcb6 100644 --- a/libs/vulcan-target/tests/matmul_expand.zig +++ b/libs/vulcan-target/tests/matmul_expand.zig @@ -661,6 +661,10 @@ fn isScaling(opcode: ir.function.Opcode) bool { .struct_new, .extract, .convert, + .decode_low_float, + .encode_low_float, + .dequantize_nvfp4, + .quantize_nvfp4, .unary, .alloca, .call, diff --git a/libs/vulcan-target/tgsi.zig b/libs/vulcan-target/tgsi.zig index fe5fb20..fa4a5c6 100644 --- a/libs/vulcan-target/tgsi.zig +++ b/libs/vulcan-target/tgsi.zig @@ -818,6 +818,8 @@ pub fn lower(allocator: std.mem.Allocator, func: *const Function) Error![]u8 { try body.put("\n"); line += 1; }, + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, .icmp => |cmp| { // A comparison -> a TGSI set-op producing an integer boolean (~0/0). const r = result orelse return error.Unsupported; @@ -1413,6 +1415,65 @@ test "an f16 function is rejected cleanly, not miscompiled as f64" { try testing.expectError(error.Unsupported, lower(allocator, &func)); } +test "both low float directions are independently rejected in supported fragment shapes" { + const allocator = testing.allocator; + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + try func.addAttr(.func, .{ .custom = .{ .namespace = "vulcan.gpu", .key = "stage", .value = .{ .string = "fragment" } } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const i32_t = try func.types.intern(.{ .int = .{ .signedness = .signed, .bits = 32 } }); + const block = try func.appendBlock(); + const input = try func.appendBlockParam(block, if (decode_direction) u8_t else f32_t); + try func.addAttr(.{ .value = input }, .{ .custom = .{ .namespace = "vulcan.gpu", .key = "attr", .value = .{ .int = ATTR_GENERIC0 } } }); + const conversion: ir.function.LowFloatConvert = .{ .value = input, .format = .f8_e4m3 }; + const result = try func.appendInst( + block, + if (decode_direction) f32_t else u8_t, + if (decode_direction) + .{ .decode_low_float = conversion } + else + .{ .encode_low_float = conversion }, + ); + const output = try func.appendInst(block, i32_t, .{ .iconst = 0 }); + try func.addAttr(.{ .value = output }, .{ .custom = .{ .namespace = "vulcan.gpu", .key = "color_out", .value = .{ .int = 0 } } }); + try func.appendStore(block, result, output); + func.setTerminator(block, .{ .ret = ir.function.Ret.none() }); + try testing.expectError(error.Unsupported, lower(allocator, &func)); + } +} + +test "both nvfp4 directions reject live attributed shaders without mutation" { + const allocator = testing.allocator; + inline for (.{ true, false }) |dequantize| { + var func = Function.init(allocator); + defer func.deinit(); + try func.addAttr(.func, .{ .custom = .{ .namespace = "vulcan.gpu", .key = "stage", .value = .{ .string = "fragment" } } }); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const i32_t = try func.types.intern(.{ .int = .{ .signedness = .signed, .bits = 32 } }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const value = try func.appendBlockParam(block, f32_t); + const global_scale = try func.appendBlockParam(block, f32_t); + try func.addAttr(.{ .value = payload }, .{ .custom = .{ .namespace = "vulcan.gpu", .key = "attr", .value = .{ .int = ATTR_GENERIC0 } } }); + const conversion: ir.function.NvFp4Convert = .{ .value = if (dequantize) payload else value, .block_scale = block_scale, .global_scale = global_scale, .block_application = .multiply, .global_application = .divide }; + const result = try func.appendInst(block, if (dequantize) f32_t else u8_t, if (dequantize) .{ .dequantize_nvfp4 = conversion } else .{ .quantize_nvfp4 = conversion }); + const output = try func.appendInst(block, i32_t, .{ .iconst = 0 }); + try func.addAttr(.{ .value = output }, .{ .custom = .{ .namespace = "vulcan.gpu", .key = "color_out", .value = .{ .int = 0 } } }); + try func.appendStore(block, result, output); + func.setTerminator(block, .{ .ret = ir.function.Ret.none() }); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + try testing.expectError(error.Unsupported, lower(allocator, &func)); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try testing.expectEqualSlices(u8, before, after); + } +} + test "lower an add-with-constant vertex shader (arith_imm + fconst pool) to TGSI" { const allocator = testing.allocator; var func = Function.init(allocator); diff --git a/libs/vulcan-target/wasm/isel.zig b/libs/vulcan-target/wasm/isel.zig index c543580..7aa7729 100644 --- a/libs/vulcan-target/wasm/isel.zig +++ b/libs/vulcan-target/wasm/isel.zig @@ -887,6 +887,8 @@ fn emitInst( const result = func.instResult(inst); switch (op) { + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, // Wasm has no 128-bit float to hold one in. .fconst128 => return error.Unsupported, .iconst => |val| { @@ -1598,6 +1600,46 @@ test "a barrier is rejected, not dropped like a prefetch" { try std.testing.expectError(error.Unsupported, selectFunction(a, &func, null)); } +test "both low float directions are rejected while their results are live" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) u16_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = .bf16 }; + const result = try func.appendInst(block, if (decode_direction) f32_t else u16_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + try std.testing.expectError(error.Unsupported, selectFunction(allocator, &func, null)); + } +} + +test "both nvfp4 directions are rejected live without mutating caller IR" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |dequantize| { + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const payload = try func.appendBlockParam(block, u8_t); + const block_scale = try func.appendBlockParam(block, u8_t); + const value = try func.appendBlockParam(block, f32_t); + const global_scale = try func.appendBlockParam(block, f32_t); + const conversion: ir.function.NvFp4Convert = .{ .value = if (dequantize) payload else value, .block_scale = block_scale, .global_scale = global_scale, .block_application = .multiply, .global_application = .divide }; + const result = try func.appendInst(block, if (dequantize) f32_t else u8_t, if (dequantize) .{ .dequantize_nvfp4 = conversion } else .{ .quantize_nvfp4 = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + try std.testing.expectError(error.Unsupported, selectFunction(allocator, &func, null)); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try std.testing.expectEqualSlices(u8, before, after); + } +} + test "an atomic is rejected, not dropped like a prefetch" { // The wasm threads proposal spells an atomic read-modify-write, but this encoder does // not build one and the module declares no shared memory. Both forms are refused: the diff --git a/libs/vulcan-target/wimmer.zig b/libs/vulcan-target/wimmer.zig index b869310..8b69f97 100644 --- a/libs/vulcan-target/wimmer.zig +++ b/libs/vulcan-target/wimmer.zig @@ -394,6 +394,12 @@ fn visitOperands(func: *const Function, inst: Inst, ctx: anytype, comptime f: fn }, .extract => |e| f(ctx, e.aggregate, false), .convert => |cv| f(ctx, cv.value, false), + .decode_low_float, .encode_low_float => |cv| f(ctx, cv.value, false), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + f(ctx, cv.value, false); + f(ctx, cv.block_scale, false); + f(ctx, cv.global_scale, false); + }, .unary => |u| f(ctx, u.value, false), .load => |l| f(ctx, l.ptr, false), .store => |st| { diff --git a/libs/vulcan-target/x86/isel.zig b/libs/vulcan-target/x86/isel.zig index 9f8bd96..fc336a7 100644 --- a/libs/vulcan-target/x86/isel.zig +++ b/libs/vulcan-target/x86/isel.zig @@ -325,6 +325,8 @@ fn rebindSymbolName(func: *const Function, name: []const u8) []const u8 { /// never sees one. The switch here is exhaustive, so a new opcode must make a decision. fn unhandledStatement(op_code: ir.function.Opcode) Error { return switch (op_code) { + .decode_low_float, .encode_low_float => error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => error.Unsupported, // A tile multiply writes memory at `c`. This backend has no tile-multiply lowering, so // it fails closed rather than dropping the write. .matmul, @@ -425,6 +427,8 @@ fn lowerInst(allocator: std.mem.Allocator, ctx: *Ctx, inst: ir.function.Inst) Er } const result = func.instResult(inst) orelse return unhandledStatement(func.opcode(inst)); switch (func.opcode(inst)) { + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, .iconst => |c| { const rd = ctx.dst(result, scratch1); try ctx.put(allocator, encode.movImm(rd, @intCast(c))); @@ -1538,6 +1542,12 @@ fn forEachOperand(func: *const Function, inst: ir.function.Inst, fold: *const ad }, .extract => |e| f(ctx, e.aggregate, false), .convert => |cv| f(ctx, cv.value, false), + .decode_low_float, .encode_low_float => |cv| f(ctx, cv.value, false), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + f(ctx, cv.value, false); + f(ctx, cv.block_scale, false); + f(ctx, cv.global_scale, false); + }, .unary => |u| f(ctx, u.value, false), .load => f(ctx, fold.baseOf(func, inst), false), .store => |st| { @@ -1884,6 +1894,22 @@ test "a barrier is rejected, not dropped like a prefetch" { try std.testing.expectError(error.Unsupported, selectFunction(allocator, &func)); } +test "both low float directions are rejected while their results are live" { + const allocator = std.testing.allocator; + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const u16_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 16 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) u16_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = .bf16 }; + const result = try func.appendInst(block, if (decode_direction) f32_t else u16_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + try std.testing.expectError(error.Unsupported, selectFunction(allocator, &func)); + } +} + test "matmul, prefetch and a void call are refused, not a panic on the result unwrap" { // Each of these has no result. Before `unhandledStatement`, `lowerInst`'s result unwrap // was a bare `.?`, so any of them reaching this backend PANICKED with "attempt to use diff --git a/libs/vulcan-target/x86_64/isel.zig b/libs/vulcan-target/x86_64/isel.zig index 457e39d..4c11f8b 100644 --- a/libs/vulcan-target/x86_64/isel.zig +++ b/libs/vulcan-target/x86_64/isel.zig @@ -512,6 +512,44 @@ pub fn reHomeCountForTest(allocator: std.mem.Allocator, func: *const Function) E return forEachWimmerSegments(allocator, func, count.f); } +/// Test hook: expand NVFP4 through the production shared pass, allocate the +/// resulting scalar IR with Wimmer, and count integer and f32 values which own a +/// real spill segment. Fresh SSA definitions start in registers by allocator +/// invariant, so these slots demonstrate pressure-driven operand reloads or +/// later result spills without changing allocation policy for the test. +pub fn nvFp4SpillCountsForTest(allocator: std.mem.Allocator, func: *const Function) Error!struct { integer: usize, float: usize } { + var work = try func.clone(allocator); + defer work.deinit(); + _ = try ir.expand.expandNvFp4(allocator, &work); + try ir.critical_edge.splitCriticalEdges(allocator, &work); + var desc = try x86_64RegDescription(allocator, &work); + defer desc.deinit(allocator); + var walloc = try wimmer.allocate(allocator, &work, &desc); + defer walloc.deinit(allocator); + + var integer: usize = 0; + var float: usize = 0; + var it = walloc.segments.iterator(); + while (it.next()) |entry| { + var has_slot = false; + for (entry.value_ptr.*) |segment| { + if (segment.loc == .slot) { + has_slot = true; + break; + } + } + if (!has_slot) continue; + switch (work.types.type_kind(work.valueType(entry.key_ptr.*))) { + .int => integer += 1, + .float => |kind| if (kind == .f32) { + float += 1; + }, + else => {}, + } + } + return .{ .integer = integer, .float = float }; +} + /// Compile `func` to machine code plus its call relocations. The caller owns the /// result. This function delegates to `compileWithCaps` with the inert (all-false) /// `ModelCaps`, so it produces the same bytes no matter what `compileWithCaps` grows @@ -556,6 +594,9 @@ pub fn compileWithCaps(allocator: std.mem.Allocator, func: *const Function, caps var work = try func.clone(allocator); defer work.deinit(); + _ = try ir.expand.expandNvFp4(allocator, &work); + _ = try ir.expand.expandLowFloat(allocator, &work); + // Lower binary128 arithmetic, compares, conversions, and sqrt to soft-fp libcalls before // anything numbers the IR, so the call clobbers and argument placement are visible to the // register allocator. f128 data movement stays native and is untouched. @@ -1416,6 +1457,8 @@ fn lowerDirectCall(allocator: std.mem.Allocator, ctx: *Ctx, c: ir.function.Call, /// never sees one. The switch here is exhaustive, so a new opcode must make a decision. fn unhandledStatement(op_code: ir.function.Opcode) Error { return switch (op_code) { + .decode_low_float, .encode_low_float => error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => error.Unsupported, // A tile multiply writes memory at `c`. This backend has no tile-multiply lowering // (`expand.zig` rewrites a matmul into scalar loops for a target without one), so it // fails closed rather than dropping the write. @@ -2082,6 +2125,8 @@ fn lowerInst(allocator: std.mem.Allocator, ctx: *Ctx, inst: ir.function.Inst) Er const end_rel: i32 = @intCast(@as(i64, @intCast(ctx.code.items.len)) - @as(i64, @intCast(jmp_at + 4))); std.mem.writeInt(u32, ctx.code.items[jmp_at..][0..4], @bitCast(end_rel), .little); }, + .decode_low_float, .encode_low_float => return error.Unsupported, + .dequantize_nvfp4, .quantize_nvfp4 => return error.Unsupported, .convert => |cv| { // Numeric conversions: int to float or back (32-bit int, f32, or // f64), int to int (low bits), and f32 to f64 or back. @@ -3506,6 +3551,12 @@ fn forEachOperand(func: *const Function, inst: ir.function.Inst, fold: *const ad }, .extract => |e| f(ctx, e.aggregate, false), .convert => |cv| f(ctx, cv.value, false), + .decode_low_float, .encode_low_float => |cv| f(ctx, cv.value, false), + .dequantize_nvfp4, .quantize_nvfp4 => |cv| { + f(ctx, cv.value, false); + f(ctx, cv.block_scale, false); + f(ctx, cv.global_scale, false); + }, .unary => |u| f(ctx, u.value, false), // A folded load/store attributes its pointer use to the fold base, // the add's lhs, not the add's own result, so the base stays live @@ -4166,6 +4217,57 @@ test "a barrier is rejected, not dropped like a prefetch" { try std.testing.expectError(error.Unsupported, selectFunction(allocator, &func)); } +test "all low float formats and directions compile without mutating caller IR" { + const allocator = std.testing.allocator; + inline for ([_]ir.low_float.Format{ .bf16, .f8_e4m3, .f8_e5m2 }) |format| { + inline for (.{ true, false }) |decode_direction| { + var func = Function.init(allocator); + defer func.deinit(); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, if (decode_direction) payload_t else f32_t); + const conversion: ir.function.LowFloatConvert = .{ .value = source, .format = format }; + const result = try func.appendInst(block, if (decode_direction) f32_t else payload_t, if (decode_direction) .{ .decode_low_float = conversion } else .{ .encode_low_float = conversion }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ .namespace = "test", .key = "preserve", .value = .flag } }); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + const before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + const code = try selectFunction(allocator, &func); + allocator.free(code); + const after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try std.testing.expectEqualStrings(before, after); + try std.testing.expectEqualSlices(u8, bits_before, bits_after); + } + } +} + +test "a later compile error preserves low float IR bitcode and attributes" { + const allocator = std.testing.allocator; + var func = Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const source = try func.appendBlockParam(block, u8_t); + const result = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .value = source, .format = .f8_e4m3 } }); + try func.addAttr(.{ .value = result }, .{ .custom = .{ .namespace = "test", .key = "preserve", .value = .flag } }); + try func.appendBarrier(block, .workgroup); + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + const before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(before); + + try std.testing.expectError(error.Unsupported, selectFunction(allocator, &func)); + const after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(after); + try std.testing.expectEqualSlices(u8, before, after); +} + test "a matmul is refused, not a panic on the result unwrap" { // `matmul` has no result. Before `unhandledStatement`, `lowerInst`'s result unwrap was a // bare `.?`, so a matmul reaching this backend PANICKED with "attempt to use null value" diff --git a/libs/vulcan-target/x86_64/tests/native.zig b/libs/vulcan-target/x86_64/tests/native.zig index 9a2d897..13d16e1 100644 --- a/libs/vulcan-target/x86_64/tests/native.zig +++ b/libs/vulcan-target/x86_64/tests/native.zig @@ -4,6 +4,7 @@ //! x86-64 host this also exercises the JIT (coherent_jit) path. const std = @import("std"); +const builtin = @import("builtin"); const cases = @import("cases.zig"); const harness = @import("harness.zig"); const ir = @import("vulcan-ir"); @@ -11,6 +12,428 @@ const isel = @import("../isel.zig"); const disasm = @import("../disasm.zig"); const link = @import("../link.zig"); const mm = @import("vulcan-opt").microarch; +const jit = @import("../../coherent_jit.zig"); + +fn lowFloatWrapper(allocator: std.mem.Allocator, format: ir.low_float.Format, decode: bool) !ir.function.Function { + var func = ir.function.Function.init(allocator); + errdefer func.deinit(); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const input = try func.appendBlockParam(block, if (decode) payload_t else u32_t); + const result = if (decode) blk: { + const value = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .format = format, .value = input } }); + break :blk try func.appendInst(block, u32_t, .{ .unary = .{ .op = .reinterpret, .value = value } }); + } else blk: { + const value = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = input } }); + const payload = try func.appendInst(block, payload_t, .{ .encode_low_float = .{ .format = format, .value = value } }); + break :blk try func.appendInst(block, u32_t, .{ .convert = .{ .value = payload } }); + }; + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + return func; +} + +const LowFloatCode = struct { + buffer: jit.CodeBuffer, + + fn init(allocator: std.mem.Allocator, func: *const ir.function.Function) !LowFloatCode { + if (builtin.cpu.arch != .x86_64) return error.SkipZigTest; + const code = try isel.selectFunction(allocator, func); + defer allocator.free(code); + return .{ .buffer = try jit.CodeBuffer.map(code) }; + } + + fn deinit(self: *LowFloatCode) void { + self.buffer.deinit(); + } + + fn call(self: *const LowFloatCode, input: u32) u32 { + return self.buffer.entry(*const fn (u32) callconv(.c) u32, 0)(input); + } +}; + +test "x86_64 native low float decode is exhaustive and caller IR stays unchanged" { + const allocator = std.testing.allocator; + var distinguished_decode: [3][2]u32 = undefined; + for (std.enums.values(ir.low_float.Format)) |format| { + var func = try lowFloatWrapper(allocator, format, true); + defer func.deinit(); + try func.addAttr(.func, .{ .custom = .{ .namespace = "test", .key = "preserve", .value = .{ .int = 41 } } }); + const text_before = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_before); + const bits_before = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_before); + var code = try LowFloatCode.init(allocator, &func); + defer code.deinit(); + const text_after = try std.fmt.allocPrint(allocator, "{f}", .{func}); + defer allocator.free(text_after); + const bits_after = try ir.bitcode.encode(allocator, &func); + defer allocator.free(bits_after); + try std.testing.expectEqualStrings(text_before, text_after); + try std.testing.expectEqualSlices(u8, bits_before, bits_after); + + const limit: usize = if (format == .bf16) 65536 else 256; + for (0..limit) |raw| { + const payload: u16 = @intCast(raw); + const expected: u32 = @bitCast(try ir.low_float.decode(format, payload)); + try std.testing.expectEqual(expected, code.call(payload)); + } + distinguished_decode[@intFromEnum(format)] = .{ code.call(0x7c), code.call(0x7e) }; + } + try std.testing.expect(distinguished_decode[@intFromEnum(ir.low_float.Format.f8_e4m3)][0] != distinguished_decode[@intFromEnum(ir.low_float.Format.f8_e5m2)][0]); + try std.testing.expect(distinguished_decode[@intFromEnum(ir.low_float.Format.f8_e4m3)][1] != distinguished_decode[@intFromEnum(ir.low_float.Format.f8_e5m2)][1]); +} + +test "x86_64 native low float encode matches structured and boundary corpora" { + const allocator = std.testing.allocator; + const mantissas = [_]u32{ + 0, 1, 0x003f_ffff, 0x0040_0000, 0x0040_0001, 0x007f_7fff, + 0x007f_8000, 0x007f_8001, 0x007f_ffff, 0x0000_7fff, 0x0000_8000, 0x0000_8001, + }; + const boundaries = [_]u32{ + 0x0000_0000, 0x8000_0000, 0x3f80_0000, 0x3f80_7fff, 0x3f80_8000, + 0x3f80_8001, 0x3f81_8000, 0x3a7f_ffff, 0x3a80_0000, 0x3a80_0001, + 0x3b3f_ffff, 0x3b40_0000, 0x3b40_0001, 0x36ff_ffff, 0x3700_0000, + 0x3700_0001, 0x37bf_ffff, 0x37c0_0000, 0x37c0_0001, 0x7f7f_ffff, + 0xff7f_ffff, 0x7f80_0000, 0xff80_0000, 0x7f80_0001, 0xffa0_0001, + 0x7fc0_0000, 0xffc0_0000, + }; + + for (std.enums.values(ir.low_float.Format)) |format| { + var func = try lowFloatWrapper(allocator, format, false); + defer func.deinit(); + var code = try LowFloatCode.init(allocator, &func); + defer code.deinit(); + for (boundaries) |bits| { + try std.testing.expectEqual(@as(u32, ir.low_float.encode(format, @bitCast(bits))), code.call(bits)); + } + switch (format) { + .bf16 => for (ir.low_float.bf16_encode_boundary_cases) |case| { + try std.testing.expectEqual(@as(u32, case.expected), code.call(case.source)); + }, + .f8_e4m3 => for (ir.low_float.e4_encode_boundary_cases) |case| { + try std.testing.expectEqual(@as(u32, case.expected), code.call(case.source)); + try std.testing.expectEqual(@as(u32, case.expected | 0x80), code.call(case.source | 0x8000_0000)); + }, + .f8_e5m2 => for (ir.low_float.e5_encode_boundary_cases) |case| { + try std.testing.expectEqual(@as(u32, case.expected), code.call(case.source)); + try std.testing.expectEqual(@as(u32, case.expected | 0x80), code.call(case.source | 0x8000_0000)); + }, + } + if (format != .bf16) for (ir.low_float.fp8_encode_special_cases) |case| { + const expected = if (format == .f8_e4m3) case.e4 else case.e5; + try std.testing.expectEqual(@as(u32, expected), code.call(case.source)); + }; + for (0..256) |exponent| { + for ([_]u32{ 0, 0x8000_0000 }) |sign| { + for (mantissas) |mantissa| { + const bits = sign | (@as(u32, @intCast(exponent)) << 23) | mantissa; + try std.testing.expectEqual(@as(u32, ir.low_float.encode(format, @bitCast(bits))), code.call(bits)); + } + } + } + } +} + +test "x86_64 native low float decode then encode follows reference NaN quieting" { + const allocator = std.testing.allocator; + for (std.enums.values(ir.low_float.Format)) |format| { + var decode_func = try lowFloatWrapper(allocator, format, true); + defer decode_func.deinit(); + var encode_func = try lowFloatWrapper(allocator, format, false); + defer encode_func.deinit(); + var decode_code = try LowFloatCode.init(allocator, &decode_func); + defer decode_code.deinit(); + var encode_code = try LowFloatCode.init(allocator, &encode_func); + defer encode_code.deinit(); + const limit: usize = if (format == .bf16) 65536 else 256; + for (0..limit) |raw| { + const payload: u16 = @intCast(raw); + const expected = ir.low_float.encode(format, try ir.low_float.decode(format, payload)); + try std.testing.expectEqual(@as(u32, expected), encode_code.call(decode_code.call(payload))); + } + } +} + +test "x86_64 native low float memory path supports payload and f32 buffers" { + if (builtin.cpu.arch != .x86_64) return error.SkipZigTest; + const allocator = std.testing.allocator; + const element_count = 5; + const payload_values = [_]u16{ 0x00, 0x01, 0x3d, 0x7c, 0xff }; + const source_bits = [_]u32{ 0x0000_0000, 0x8000_0000, 0x3f88_0001, 0x7f80_0001, 0xff7f_ffff }; + for (std.enums.values(ir.low_float.Format)) |format| { + var func = ir.function.Function.init(allocator); + defer func.deinit(); + const ptr_t = try func.types.ptrGlobal(); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const decode_ptr = try func.appendBlockParam(block, ptr_t); + const float_out_ptr = try func.appendBlockParam(block, ptr_t); + const float_in_ptr = try func.appendBlockParam(block, ptr_t); + const encode_ptr = try func.appendBlockParam(block, ptr_t); + const payload_bytes: usize = format.payloadBits() / 8; + for (0..element_count) |index| { + const payload_offset: i64 = @intCast(index * payload_bytes); + const float_offset: i64 = @intCast(index * @sizeOf(u32)); + const payload_at = if (index == 0) decode_ptr else try func.appendArithImm(block, ptr_t, .add, decode_ptr, payload_offset); + const float_out_at = if (index == 0) float_out_ptr else try func.appendArithImm(block, ptr_t, .add, float_out_ptr, float_offset); + const float_in_at = if (index == 0) float_in_ptr else try func.appendArithImm(block, ptr_t, .add, float_in_ptr, float_offset); + const encode_at = if (index == 0) encode_ptr else try func.appendArithImm(block, ptr_t, .add, encode_ptr, payload_offset); + const payload = try func.appendInst(block, payload_t, .{ .load = .{ .ptr = payload_at } }); + const decoded = try func.appendInst(block, f32_t, .{ .decode_low_float = .{ .format = format, .value = payload } }); + try func.appendStore(block, decoded, float_out_at); + const source = try func.appendInst(block, f32_t, .{ .load = .{ .ptr = float_in_at } }); + const encoded = try func.appendInst(block, payload_t, .{ .encode_low_float = .{ .format = format, .value = source } }); + try func.appendStore(block, encoded, encode_at); + } + func.setTerminator(block, .{ .ret = ir.function.Ret.none() }); + + const code = try isel.selectFunction(allocator, &func); + defer allocator.free(code); + var buffer = try jit.CodeBuffer.map(code); + defer buffer.deinit(); + const run = buffer.entry(*const fn (*const u8, *u32, *const u32, *u8) callconv(.c) void, 0); + var input_storage = [_]u8{0xaa} ++ [_]u8{0} ** (element_count * 2); + var decoded_bits = [_]u32{0} ** element_count; + var output_storage = [_]u8{0} ** (element_count * 2); + for (payload_values, 0..) |payload_value, index| { + input_storage[1 + index * payload_bytes] = @truncate(payload_value); + if (format == .bf16) input_storage[2 + index * payload_bytes] = @truncate(payload_value >> 8); + } + run(&input_storage[1], &decoded_bits[0], &source_bits[0], &output_storage[0]); + for (payload_values, source_bits, 0..) |payload_value, source, index| { + try std.testing.expectEqual(@as(u32, @bitCast(try ir.low_float.decode(format, payload_value))), decoded_bits[index]); + const expected = ir.low_float.encode(format, @bitCast(source)); + try std.testing.expectEqual(@as(u8, @truncate(expected)), output_storage[index * payload_bytes]); + if (format == .bf16) try std.testing.expectEqual(@as(u8, @truncate(expected >> 8)), output_storage[index * payload_bytes + 1]); + } + } +} + +test "x86_64 native low float expansion survives integer spill pressure" { + if (builtin.cpu.arch != .x86_64) return error.SkipZigTest; + const allocator = std.testing.allocator; + var func = ir.function.Function.init(allocator); + defer func.deinit(); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const input = try func.appendBlockParam(block, u32_t); + var values: [24]ir.function.Value = undefined; + for (&values, 0..) |*slot, index| { + const salt = try func.appendInst(block, u32_t, .{ .iconst = @as(i64, @intCast(index * 0x10203)) }); + const bits = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = input, .rhs = salt } }); + const source = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = bits } }); + const format: ir.low_float.Format = @enumFromInt(index % 3); + const payload_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = format.payloadBits() } }); + const encoded = try func.appendInst(block, payload_t, .{ .encode_low_float = .{ .format = format, .value = source } }); + slot.* = try func.appendInst(block, u32_t, .{ .convert = .{ .value = encoded } }); + } + var result = values[0]; + for (values[1..]) |value| { + result = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = result, .rhs = value } }); + } + func.setTerminator(block, .{ .ret = ir.function.Ret.one(result) }); + + var code = try LowFloatCode.init(allocator, &func); + defer code.deinit(); + const inputs = [_]u32{ 0, 0x3f80_8001, 0x7f80_0001, 0xff7f_ffff }; + for (inputs) |input_bits| { + var expected: u32 = 0; + for (0..values.len) |index| { + const format: ir.low_float.Format = @enumFromInt(index % 3); + const bits = input_bits ^ @as(u32, @intCast(index * 0x10203)); + expected ^= ir.low_float.encode(format, @bitCast(bits)); + } + try std.testing.expectEqual(expected, code.call(input_bits)); + } +} + +test "x86_64 native NVFP4 packed memory crosses a scale block and preserves its tail sibling" { + if (builtin.cpu.arch != .x86_64) return error.SkipZigTest; + const allocator = std.testing.allocator; + const element_count = 19; + const packed_count = (element_count + 1) / 2; + var func = ir.function.Function.init(allocator); + defer func.deinit(); + const ptr_t = try func.types.ptrGlobal(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const packed_input = try func.appendBlockParam(block, ptr_t); + const scales = try func.appendBlockParam(block, ptr_t); + const global_bits = try func.appendBlockParam(block, u32_t); + const decoded_output = try func.appendBlockParam(block, ptr_t); + const float_input = try func.appendBlockParam(block, ptr_t); + const packed_output = try func.appendBlockParam(block, ptr_t); + const global_scale_value = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = global_bits } }); + for (0..element_count) |index| { + const packed_at = if (index < 2) packed_input else try func.appendArithImm(block, ptr_t, .add, packed_input, @intCast(index / 2)); + const scale_at = if (index < 16) scales else try func.appendArithImm(block, ptr_t, .add, scales, @intCast(index / 16)); + const packed_byte = try func.appendInst(block, u8_t, .{ .load = .{ .ptr = packed_at } }); + const packed_wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = packed_byte } }); + const positioned = if (index & 1 == 0) + packed_wide + else + try func.appendArithImm(block, u32_t, .shr, packed_wide, 4); + const nibble_wide = try func.appendArithImm(block, u32_t, .bit_and, positioned, 0x0f); + const nibble = try func.appendInst(block, u8_t, .{ .convert = .{ .value = nibble_wide } }); + const block_scale = try func.appendInst(block, u8_t, .{ .load = .{ .ptr = scale_at } }); + const decoded = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = nibble, + .block_scale = block_scale, + .global_scale = global_scale_value, + .block_application = .multiply, + .global_application = .divide, + } }); + const decoded_at = if (index == 0) decoded_output else try func.appendArithImm(block, ptr_t, .add, decoded_output, @intCast(index * 4)); + try func.appendStore(block, decoded, decoded_at); + + const float_at = if (index == 0) float_input else try func.appendArithImm(block, ptr_t, .add, float_input, @intCast(index * 4)); + const source = try func.appendInst(block, f32_t, .{ .load = .{ .ptr = float_at } }); + const encoded = try func.appendInst(block, u8_t, .{ .quantize_nvfp4 = .{ + .value = source, + .block_scale = block_scale, + .global_scale = global_scale_value, + .block_application = .divide, + .global_application = .multiply, + } }); + const encoded_wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = encoded } }); + const output_at = if (index < 2) packed_output else try func.appendArithImm(block, ptr_t, .add, packed_output, @intCast(index / 2)); + const old_byte = try func.appendInst(block, u8_t, .{ .load = .{ .ptr = output_at } }); + const old_wide = try func.appendInst(block, u32_t, .{ .convert = .{ .value = old_byte } }); + const preserved = try func.appendArithImm(block, u32_t, .bit_and, old_wide, if (index & 1 == 0) 0xf0 else 0x0f); + const placed = if (index & 1 == 0) + encoded_wide + else + try func.appendArithImm(block, u32_t, .shl, encoded_wide, 4); + const combined = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_or, .lhs = preserved, .rhs = placed } }); + const output_byte = try func.appendInst(block, u8_t, .{ .convert = .{ .value = combined } }); + try func.appendStore(block, output_byte, output_at); + } + func.setTerminator(block, .{ .ret = ir.function.Ret.none() }); + + const code = try isel.selectFunction(allocator, &func); + defer allocator.free(code); + var buffer = try jit.CodeBuffer.map(code); + defer buffer.deinit(); + const run = buffer.entry(*const fn (*const u8, *const u8, u32, *u32, *const u32, *u8) callconv(.c) void, 0); + const packed_values = [_]u8{ 0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe, 0x21, 0x43 }; + const scale_values = [_]u8{ 0x39, 0xb8 }; + const global_scale_bits: u32 = 0x3f40_0000; + const source_bits = [_]u32{ + 0x0000_0000, 0x8000_0000, 0x3e80_0000, 0x3e80_0001, 0x3f40_0000, + 0x3f40_0001, 0x3fa0_0000, 0x3fa0_0001, 0x3fe0_0000, 0x4020_0000, + 0x4060_0000, 0x40a0_0000, 0x7f7f_ffff, 0xff7f_ffff, 0x7f80_0000, + 0xff80_0000, 0x7f80_0001, 0xff80_0001, 0x7fc0_1234, + }; + var decoded_bits = [_]u32{0} ** element_count; + var output = [_]u8{0xa5} ** packed_count; + var expected_output = output; + run(&packed_values[0], &scale_values[0], global_scale_bits, &decoded_bits[0], &source_bits[0], &output[0]); + const global_scale: f32 = @bitCast(global_scale_bits); + for (0..element_count) |index| { + const payload = if (index & 1 == 0) packed_values[index / 2] & 0x0f else packed_values[index / 2] >> 4; + const scale = scale_values[index / 16]; + const expected_decoded = ir.nvfp4.dequantize(payload, scale, global_scale, .multiply, .divide); + if (std.math.isNan(expected_decoded)) { + try std.testing.expect(std.math.isNan(@as(f32, @bitCast(decoded_bits[index])))); + } else { + try std.testing.expectEqual(@as(u32, @bitCast(expected_decoded)), decoded_bits[index]); + } + const encoded = ir.nvfp4.quantize(@bitCast(source_bits[index]), scale, global_scale, .divide, .multiply); + if (index & 1 == 0) { + expected_output[index / 2] = (expected_output[index / 2] & 0xf0) | encoded; + } else { + expected_output[index / 2] = (expected_output[index / 2] & 0x0f) | (encoded << 4); + } + } + try std.testing.expectEqualSlices(u8, &expected_output, &output); + try std.testing.expectEqual(@as(u8, 0xa0), output[packed_count - 1] & 0xf0); +} + +test "x86_64 native NVFP4 expansion survives integer and float spill pressure" { + if (builtin.cpu.arch != .x86_64) return error.SkipZigTest; + const allocator = std.testing.allocator; + const lane_count = 24; + const block_scales = [_]u8{ 0x38, 0x39, 0xb8, 0x40 }; + var func = ir.function.Function.init(allocator); + defer func.deinit(); + const u8_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 8 } }); + const u32_t = try func.types.intern(.{ .int = .{ .signedness = .unsigned, .bits = 32 } }); + const f32_t = try func.types.intern(.{ .float = .f32 }); + const block = try func.appendBlock(); + const input = try func.appendBlockParam(block, u32_t); + const global_bits = try func.appendInst(block, u32_t, .{ .iconst = 0x3fc0_0000 }); + const global_scale = try func.appendInst(block, f32_t, .{ .unary = .{ .op = .reinterpret, .value = global_bits } }); + + // Define every carrier before conversion so the source bank exceeds the GPR + // set. Keep every decoded result alive until the second loop so the f32 bank + // independently exceeds the XMM set. Wimmer starts fresh definitions in + // registers; the allocator hook therefore observes later spill segments. + var carriers: [lane_count]ir.function.Value = undefined; + var scales: [lane_count]ir.function.Value = undefined; + for (0..lane_count) |index| { + const salt = try func.appendInst(block, u32_t, .{ .iconst = @as(i64, @intCast(index * 0x10203)) }); + const mixed = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = input, .rhs = salt } }); + carriers[index] = try func.appendInst(block, u8_t, .{ .convert = .{ .value = mixed } }); + scales[index] = try func.appendInst(block, u8_t, .{ .iconst = block_scales[index % block_scales.len] }); + } + + var decoded: [lane_count]ir.function.Value = undefined; + for (0..lane_count) |index| { + const block_application: ir.nvfp4.ScaleApplication = if (index & 1 == 0) .multiply else .divide; + const global_application: ir.nvfp4.ScaleApplication = if (index & 2 == 0) .multiply else .divide; + decoded[index] = try func.appendInst(block, f32_t, .{ .dequantize_nvfp4 = .{ + .value = carriers[index], + .block_scale = scales[index], + .global_scale = global_scale, + .block_application = block_application, + .global_application = global_application, + } }); + } + + var encoded: [lane_count]ir.function.Value = undefined; + for (0..lane_count) |index| { + const block_application: ir.nvfp4.ScaleApplication = if (index & 1 == 0) .multiply else .divide; + const global_application: ir.nvfp4.ScaleApplication = if (index & 2 == 0) .multiply else .divide; + const payload = try func.appendInst(block, u8_t, .{ .quantize_nvfp4 = .{ + .value = decoded[index], + .block_scale = scales[index], + .global_scale = global_scale, + .block_application = block_application, + .global_application = global_application, + } }); + encoded[index] = try func.appendInst(block, u32_t, .{ .convert = .{ .value = payload } }); + } + var digest = encoded[0]; + for (encoded[1..]) |payload| { + digest = try func.appendInst(block, u32_t, .{ .arith = .{ .op = .bit_xor, .lhs = digest, .rhs = payload } }); + } + func.setTerminator(block, .{ .ret = ir.function.Ret.one(digest) }); + + const spills = try isel.nvFp4SpillCountsForTest(allocator, &func); + try std.testing.expect(spills.integer > 0); + try std.testing.expect(spills.float > 0); + var code = try LowFloatCode.init(allocator, &func); + defer code.deinit(); + const inputs = [_]u32{ 0, 0x1234_5678, 0xffff_ffff }; + for (inputs) |input_bits| { + var expected: u32 = 0; + for (0..lane_count) |index| { + const carrier: u8 = @truncate(input_bits ^ @as(u32, @intCast(index * 0x10203))); + const block_scale = block_scales[index % block_scales.len]; + const block_application: ir.nvfp4.ScaleApplication = if (index & 1 == 0) .multiply else .divide; + const global_application: ir.nvfp4.ScaleApplication = if (index & 2 == 0) .multiply else .divide; + const value = ir.nvfp4.dequantize(carrier, block_scale, @bitCast(@as(u32, 0x3fc0_0000)), block_application, global_application); + expected ^= ir.nvfp4.quantize(value, block_scale, @bitCast(@as(u32, 0x3fc0_0000)), block_application, global_application); + } + try std.testing.expectEqual(expected, code.call(input_bits)); + } +} test "module disasm: linked functions get labels and a resolved, named call" { const a = std.testing.allocator; diff --git a/tools/uarch-bench.zig b/tools/uarch-bench.zig index 60392ba..23f5d66 100644 --- a/tools/uarch-bench.zig +++ b/tools/uarch-bench.zig @@ -508,7 +508,7 @@ fn customLatency(op: ir.function.Opcode) u32 { .arith => |a| customArith(a.op), .arith_imm => |a| customArith(a.op), .load => 3, - .convert, .unary => 2, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 2, // A dot is a multiply-class op (4-way multiply-accumulate), grouped with `mul`. .dot => 3, // This fictional part carries no tensor unit; a placeholder in case one is added. @@ -541,7 +541,7 @@ fn customThroughput(op: ir.function.Opcode, elem_float: bool) u32 { .arith => |a| customArithThroughput(a.op, elem_float), .arith_imm => |a| customArithThroughput(a.op, elem_float), .load => 1, - .convert, .unary => 1, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4, .unary => 1, .dot => 3, .matmul => 64, // no tensor unit here; non-pipelined placeholder .iconst, .fconst, .fconst128, .icmp, .select, .struct_new, .extract, .alloca, .call, .call_indirect, .global_addr, .store, .prefetch, .@"if" => 1, @@ -557,7 +557,7 @@ fn customUnit(op: ir.function.Opcode) opt.microarch.UnitClass { .arith => |a| customArithUnit(a.op), .arith_imm => |a| customArithUnit(a.op), .icmp, .select, .iconst, .fconst, .fconst128, .global_addr => .alu, - .convert => .fpsimd, + .convert, .decode_low_float, .encode_low_float, .dequantize_nvfp4, .quantize_nvfp4 => .fpsimd, .unary => |u| switch (u.op) { .reinterpret => .alu, .sqrt, .ceil, .floor, .trunc, .nearest => .fpsimd,