mirrors/zig
1
0
Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2025-12-06 05:44:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #22998 from jacobly0/x86_64-rewrite

Browse source 
x86_64: rewrite aggregate init
This commit is contained in:
Andrew Kelley 2025-03-08 14:27:57 -05:00 • committed by GitHub
commit 61c588d726
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 355 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

583
src/arch/x86_64/CodeGen.zig
View file

@ -2437,7 +2437,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
try cg.airArg(inst); try cg.airArg(inst);
try cg.resetTemps(); try cg.resetTemps(@enumFromInt(0));
cg.checkInvariantsAfterAirInst(); cg.checkInvariantsAfterAirInst();
}, },
else => break, else => break,
@ -2477,7 +2477,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
.shuffle => try cg.airShuffle(inst), .shuffle => try cg.airShuffle(inst),
.reduce => try cg.airReduce(inst), .reduce => try cg.airReduce(inst),
.reduce_optimized => try cg.airReduce(inst), .reduce_optimized => try cg.airReduce(inst),
.aggregate_init => try cg.airAggregateInit(inst),
// zig fmt: on // zig fmt: on
.arg => if (cg.debug_output != .none) { .arg => if (cg.debug_output != .none) {
@ -80843,6 +80842,74 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
for (ops[1..]) |op| try op.die(cg); for (ops[1..]) |op| try op.die(cg);
try res[0].finish(inst, &.{ty_op.operand}, ops[0..1], cg); try res[0].finish(inst, &.{ty_op.operand}, ops[0..1], cg);
}, },
.aggregate_init => |air_tag| if (use_old) try cg.airAggregateInit(inst) else fallback: {
const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
const agg_ty = ty_pl.ty.toType();
if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or
(agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst);
var res = try cg.tempAllocMem(agg_ty);
const reset_index = cg.next_temp_index;
var bt = cg.liveness.iterateBigTomb(inst);
switch (ip.indexToKey(agg_ty.toIntern())) {
inline .array_type, .vector_type => |sequence_type| {
const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..@intCast(sequence_type.len)]);
const elem_size = Type.fromInterned(sequence_type.child).abiSize(zcu);
var elem_disp: u31 = 0;
for (elems) |elem_ref| {
var elem = try cg.tempFromOperand(elem_ref, bt.feed());
try res.write(&elem, .{ .disp = elem_disp }, cg);
try elem.die(cg);
try cg.resetTemps(reset_index);
elem_disp += @intCast(elem_size);
}
if (@hasField(@TypeOf(sequence_type), "sentinel") and sequence_type.sentinel != .none) {
var sentinel = try cg.tempFromValue(.fromInterned(sequence_type.sentinel));
try res.write(&sentinel, .{ .disp = elem_disp }, cg);
try sentinel.die(cg);
}
},
.struct_type => {
const loaded_struct = ip.loadStructType(agg_ty.toIntern());
const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..loaded_struct.field_types.len]);
switch (loaded_struct.layout) {
.auto, .@"extern" => {
for (elems, 0..) |elem_ref, field_index| {
const elem_dies = bt.feed();
if (loaded_struct.fieldIsComptime(ip, field_index)) continue;
var elem = try cg.tempFromOperand(elem_ref, elem_dies);
try res.write(&elem, .{ .disp = @intCast(loaded_struct.offsets.get(ip)[field_index]) }, cg);
try elem.die(cg);
try cg.resetTemps(reset_index);
}
},
.@"packed" => return cg.fail("failed to select {s} {}", .{
@tagName(air_tag),
agg_ty.fmt(pt),
}),
}
},
.tuple_type => |tuple_type| {
const elems: []const Air.Inst.Ref = @ptrCast(cg.air.extra[ty_pl.payload..][0..tuple_type.types.len]);
var elem_disp: u31 = 0;
for (elems, 0..) |elem_ref, field_index| {
const elem_dies = bt.feed();
if (tuple_type.values.get(ip)[field_index] != .none) continue;
const field_type = Type.fromInterned(tuple_type.types.get(ip)[field_index]);
elem_disp = @intCast(field_type.abiAlignment(zcu).forward(elem_disp));
var elem = try cg.tempFromOperand(elem_ref, elem_dies);
try res.write(&elem, .{ .disp = elem_disp }, cg);
try elem.die(cg);
try cg.resetTemps(reset_index);
elem_disp += @intCast(field_type.abiSize(zcu));
}
},
else => return cg.fail("failed to select {s} {}", .{
@tagName(air_tag),
agg_ty.fmt(pt),
}),
}
try res.finish(inst, &.{}, &.{}, cg);
},
.union_init => if (use_old) try cg.airUnionInit(inst) else { .union_init => if (use_old) try cg.airUnionInit(inst) else {
const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; const ty_pl = air_datas[@intFromEnum(inst)].ty_pl;
const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data; const extra = cg.air.extraData(Air.UnionInit, ty_pl.payload).data;
@ -82199,14 +82266,14 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void {
.c_va_start => try cg.airVaStart(inst), .c_va_start => try cg.airVaStart(inst),
.work_item_id, .work_group_size, .work_group_id => unreachable, .work_item_id, .work_group_size, .work_group_id => unreachable,
} }
try cg.resetTemps(); try cg.resetTemps(@enumFromInt(0));
cg.checkInvariantsAfterAirInst(); cg.checkInvariantsAfterAirInst();
} }
verbose_tracking_log.debug("{}", .{cg.fmtTracking()}); verbose_tracking_log.debug("{}", .{cg.fmtTracking()});
} }
fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void { fn genLazy(cg: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
const pt = self.pt; const pt = cg.pt;
const zcu = pt.zcu; const zcu = pt.zcu;
const ip = &zcu.intern_pool; const ip = &zcu.intern_pool;
switch (ip.indexToKey(lazy_sym.ty)) { switch (ip.indexToKey(lazy_sym.ty)) {
@ -82215,97 +82282,98 @@ fn genLazy(self: *CodeGen, lazy_sym: link.File.LazySymbol) InnerError!void {
wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)}); wip_mir_log.debug("{}.@tagName:", .{enum_ty.fmt(pt)});
const param_regs = abi.getCAbiIntParamRegs(.auto); const param_regs = abi.getCAbiIntParamRegs(.auto);
const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*); const param_locks = cg.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
defer for (param_locks) |lock| self.register_manager.unlockReg(lock); defer for (param_locks) |lock| cg.register_manager.unlockReg(lock);
const ret_mcv: MCValue = .{ .register_pair = param_regs[0..2].* }; const ret_mcv: MCValue = .{ .register_pair = param_regs[0..2].* };
const enum_mcv: MCValue = .{ .register = param_regs[0] }; var enum_temp = try cg.tempInit(enum_ty, .{ .register = param_regs[0] });
const data_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); const data_reg = try cg.register_manager.allocReg(null, abi.RegisterClass.gp);
const data_lock = self.register_manager.lockRegAssumeUnused(data_reg); const data_lock = cg.register_manager.lockRegAssumeUnused(data_reg);
defer self.register_manager.unlockReg(data_lock); defer cg.register_manager.unlockReg(data_lock);
try self.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = lazy_sym.ty }); try cg.genLazySymbolRef(.lea, data_reg, .{ .kind = .const_data, .ty = lazy_sym.ty });
var data_off: i32 = 0; var data_off: i32 = 0;
const reset_index = cg.next_temp_index;
const tag_names = ip.loadEnumType(lazy_sym.ty).names; const tag_names = ip.loadEnumType(lazy_sym.ty).names;
for (0..tag_names.len) |tag_index| { for (0..tag_names.len) |tag_index| {
var enum_temp = try self.tempInit(enum_ty, enum_mcv);
const tag_name_len = tag_names.get(ip)[tag_index].length(ip); const tag_name_len = tag_names.get(ip)[tag_index].length(ip);
var tag_temp = try self.tempFromValue(try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index))); var tag_temp = try cg.tempFromValue(try pt.enumValueFieldIndex(enum_ty, @intCast(tag_index)));
const cc_temp = enum_temp.cmpInts(.neq, &tag_temp, self) catch |err| switch (err) { const cc_temp = enum_temp.cmpInts(.neq, &tag_temp, cg) catch |err| switch (err) {
error.SelectFailed => unreachable, error.SelectFailed => unreachable,
else => |e| return e, else => |e| return e,
}; };
try enum_temp.die(self); try tag_temp.die(cg);
try tag_temp.die(self); const skip_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined);
const skip_reloc = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined); try cc_temp.die(cg);
try cc_temp.die(self); try cg.resetTemps(reset_index);
try self.resetTemps();
try self.genSetReg( try cg.genSetReg(
ret_mcv.register_pair[0], ret_mcv.register_pair[0],
.usize, .usize,
.{ .register_offset = .{ .reg = data_reg, .off = data_off } }, .{ .register_offset = .{ .reg = data_reg, .off = data_off } },
.{}, .{},
); );
try self.genSetReg(ret_mcv.register_pair[1], .usize, .{ .immediate = tag_name_len }, .{}); try cg.genSetReg(ret_mcv.register_pair[1], .usize, .{ .immediate = tag_name_len }, .{});
try self.asmOpOnly(.{ ._, .ret }); try cg.asmOpOnly(.{ ._, .ret });
self.performReloc(skip_reloc); cg.performReloc(skip_reloc);
data_off += @intCast(tag_name_len + 1); data_off += @intCast(tag_name_len + 1);
} }
try enum_temp.die(cg);
try self.genSetReg(ret_mcv.register_pair[0], .usize, .{ .immediate = 0 }, .{}); try cg.genSetReg(ret_mcv.register_pair[0], .usize, .{ .immediate = 0 }, .{});
try self.asmOpOnly(.{ ._, .ret }); try cg.asmOpOnly(.{ ._, .ret });
}, },
.error_set_type => |error_set_type| { .error_set_type => |error_set_type| {
const err_ty: Type = .fromInterned(lazy_sym.ty); const err_ty: Type = .fromInterned(lazy_sym.ty);
wip_mir_log.debug("{}.@errorCast:", .{err_ty.fmt(pt)}); wip_mir_log.debug("{}.@errorCast:", .{err_ty.fmt(pt)});
const param_regs = abi.getCAbiIntParamRegs(.auto); const param_regs = abi.getCAbiIntParamRegs(.auto);
const param_locks = self.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*); const param_locks = cg.register_manager.lockRegsAssumeUnused(2, param_regs[0..2].*);
defer for (param_locks) |lock| self.register_manager.unlockReg(lock); defer for (param_locks) |lock| cg.register_manager.unlockReg(lock);
const ret_mcv: MCValue = .{ .register = param_regs[0] }; const ret_mcv: MCValue = .{ .register = param_regs[0] };
const err_mcv: MCValue = .{ .register = param_regs[0] }; const err_mcv: MCValue = .{ .register = param_regs[0] };
var err_temp = try cg.tempInit(err_ty, err_mcv);
const ExpectedContents = [32]Mir.Inst.Index; const ExpectedContents = [32]Mir.Inst.Index;
var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa);
const allocator = stack.get(); const allocator = stack.get();
const relocs = try allocator.alloc(Mir.Inst.Index, error_set_type.names.len); const relocs = try allocator.alloc(Mir.Inst.Index, error_set_type.names.len);
defer allocator.free(relocs); defer allocator.free(relocs);
const reset_index = cg.next_temp_index;
for (0.., relocs) |tag_index, *reloc| { for (0.., relocs) |tag_index, *reloc| {
var err_temp = try self.tempInit(err_ty, err_mcv); var tag_temp = try cg.tempInit(.anyerror, .{
var tag_temp = try self.tempInit(.anyerror, .{
.immediate = ip.getErrorValueIfExists(error_set_type.names.get(ip)[tag_index]).?, .immediate = ip.getErrorValueIfExists(error_set_type.names.get(ip)[tag_index]).?,
}); });
const cc_temp = err_temp.cmpInts(.eq, &tag_temp, self) catch |err| switch (err) { const cc_temp = err_temp.cmpInts(.eq, &tag_temp, cg) catch |err| switch (err) {
error.SelectFailed => unreachable, error.SelectFailed => unreachable,
else => |e| return e, else => |e| return e,
}; };
try err_temp.die(self); try tag_temp.die(cg);
try tag_temp.die(self); reloc.* = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined);
reloc.* = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined); try cc_temp.die(cg);
try cc_temp.die(self); try cg.resetTemps(reset_index);
try self.resetTemps();
} }
try err_temp.die(cg);
try self.genCopy(.usize, ret_mcv, .{ .immediate = 0 }, .{}); try cg.genCopy(.usize, ret_mcv, .{ .immediate = 0 }, .{});
for (relocs) |reloc| self.performReloc(reloc); for (relocs) |reloc| cg.performReloc(reloc);
assert(ret_mcv.register == err_mcv.register); assert(ret_mcv.register == err_mcv.register);
try self.asmOpOnly(.{ ._, .ret }); try cg.asmOpOnly(.{ ._, .ret });
}, },
else => return self.fail( else => return cg.fail(
"TODO implement {s} for {}", "TODO implement {s} for {}",
.{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) }, .{ @tagName(lazy_sym.kind), Type.fromInterned(lazy_sym.ty).fmt(pt) },
), ),
} }
try cg.resetTemps(@enumFromInt(0));
cg.checkInvariantsAfterAirInst();
} }
fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void { fn getValue(self: *CodeGen, value: MCValue, inst: ?Air.Inst.Index) !void {
@ -93621,17 +93689,17 @@ fn lowerBlock(self: *CodeGen, inst: Air.Inst.Index, body: []const Air.Inst.Index
} }
fn lowerSwitchBr( fn lowerSwitchBr(
self: *CodeGen, cg: *CodeGen,
inst: Air.Inst.Index, inst: Air.Inst.Index,
switch_br: Air.UnwrappedSwitch, switch_br: Air.UnwrappedSwitch,
condition: MCValue, condition: MCValue,
condition_dies: bool, condition_dies: bool,
is_loop: bool, is_loop: bool,
) !void { ) !void {
const zcu = self.pt.zcu; const zcu = cg.pt.zcu;
const condition_ty = self.typeOf(switch_br.operand); const condition_ty = cg.typeOf(switch_br.operand);
const condition_int_info = self.intInfo(condition_ty).?; const condition_int_info = cg.intInfo(condition_ty).?;
const condition_int_ty = try self.pt.intType(condition_int_info.signedness, condition_int_info.bits); const condition_int_ty = try cg.pt.intType(condition_int_info.signedness, condition_int_info.bits);
const ExpectedContents = extern struct { const ExpectedContents = extern struct {
liveness_deaths: [1 << 8 | 1]Air.Inst.Index, liveness_deaths: [1 << 8 | 1]Air.Inst.Index,
@ -93639,15 +93707,15 @@ fn lowerSwitchBr(
relocs: [1 << 6]Mir.Inst.Index, relocs: [1 << 6]Mir.Inst.Index,
}; };
var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =
std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); std.heap.stackFallback(@sizeOf(ExpectedContents), cg.gpa);
const allocator = stack.get(); const allocator = stack.get();
const state = try self.saveState(); const state = try cg.saveState();
const liveness = try self.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1); const liveness = try cg.liveness.getSwitchBr(allocator, inst, switch_br.cases_len + 1);
defer allocator.free(liveness.deaths); defer allocator.free(liveness.deaths);
if (!self.mod.pic and self.target.ofmt == .elf) table: { if (!cg.mod.pic and cg.target.ofmt == .elf) table: {
var prong_items: u32 = 0; var prong_items: u32 = 0;
var min: ?Value = null; var min: ?Value = null;
var max: ?Value = null; var max: ?Value = null;
@ -93690,41 +93758,41 @@ fn lowerSwitchBr(
if (prong_items < table_len >> 2) break :table; // no more than 75% waste if (prong_items < table_len >> 2) break :table; // no more than 75% waste
const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: { const condition_index = if (condition_dies and condition.isModifiable()) condition else condition_index: {
const condition_index = try self.allocTempRegOrMem(condition_ty, true); const condition_index = try cg.allocTempRegOrMem(condition_ty, true);
try self.genCopy(condition_ty, condition_index, condition, .{}); try cg.genCopy(condition_ty, condition_index, condition, .{});
break :condition_index condition_index; break :condition_index condition_index;
}; };
try self.spillEflagsIfOccupied(); try cg.spillEflagsIfOccupied();
if (min.?.orderAgainstZero(zcu).compare(.neq)) try self.genBinOpMir( if (min.?.orderAgainstZero(zcu).compare(.neq)) try cg.genBinOpMir(
.{ ._, .sub }, .{ ._, .sub },
condition_ty, condition_ty,
condition_index, condition_index,
.{ .air_ref = Air.internedToRef(min.?.toIntern()) }, .{ .air_ref = Air.internedToRef(min.?.toIntern()) },
); );
const else_reloc = if (switch_br.else_body_len > 0) else_reloc: { const else_reloc = if (switch_br.else_body_len > 0) else_reloc: {
var cond_temp = try self.tempInit(condition_ty, condition_index); var cond_temp = try cg.tempInit(condition_ty, condition_index);
var table_max_temp = try self.tempFromValue(try self.pt.intValue(condition_int_ty, table_len - 1)); var table_max_temp = try cg.tempFromValue(try cg.pt.intValue(condition_int_ty, table_len - 1));
const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, self) catch |err| switch (err) { const cc_temp = cond_temp.cmpInts(.gt, &table_max_temp, cg) catch |err| switch (err) {
error.SelectFailed => unreachable, error.SelectFailed => unreachable,
else => |e| return e, else => |e| return e,
}; };
try cond_temp.die(self); try cond_temp.die(cg);
try table_max_temp.die(self); try table_max_temp.die(cg);
const else_reloc = try self.asmJccReloc(cc_temp.tracking(self).short.eflags, undefined); const else_reloc = try cg.asmJccReloc(cc_temp.tracking(cg).short.eflags, undefined);
try cc_temp.die(self); try cc_temp.die(cg);
break :else_reloc else_reloc; break :else_reloc else_reloc;
} else undefined; } else undefined;
const table_start: u31 = @intCast(self.mir_table.items.len); const table_start: u31 = @intCast(cg.mir_table.items.len);
{ {
const condition_index_reg = if (condition_index.isRegister()) const condition_index_reg = if (condition_index.isRegister())
condition_index.getReg().? condition_index.getReg().?
else else
try self.copyToTmpRegister(.usize, condition_index); try cg.copyToTmpRegister(.usize, condition_index);
const condition_index_lock = self.register_manager.lockReg(condition_index_reg); const condition_index_lock = cg.register_manager.lockReg(condition_index_reg);
defer if (condition_index_lock) |lock| self.register_manager.unlockReg(lock); defer if (condition_index_lock) |lock| cg.register_manager.unlockReg(lock);
try self.truncateRegister(condition_ty, condition_index_reg); try cg.truncateRegister(condition_ty, condition_index_reg);
const ptr_size = @divExact(self.target.ptrBitWidth(), 8); const ptr_size = @divExact(cg.target.ptrBitWidth(), 8);
try self.asmMemory(.{ ._mp, .j }, .{ try cg.asmMemory(.{ ._mp, .j }, .{
.base = .table, .base = .table,
.mod = .{ .rm = .{ .mod = .{ .rm = .{
.size = .ptr, .size = .ptr,
@ -93735,32 +93803,32 @@ fn lowerSwitchBr(
}); });
} }
const else_reloc_marker: u32 = 0; const else_reloc_marker: u32 = 0;
assert(self.mir_instructions.len > else_reloc_marker); assert(cg.mir_instructions.len > else_reloc_marker);
try self.mir_table.appendNTimes(self.gpa, else_reloc_marker, table_len); try cg.mir_table.appendNTimes(cg.gpa, else_reloc_marker, table_len);
if (is_loop) try self.loop_switches.putNoClobber(self.gpa, inst, .{ if (is_loop) try cg.loop_switches.putNoClobber(cg.gpa, inst, .{
.start = table_start, .start = table_start,
.len = table_len, .len = table_len,
.min = min.?, .min = min.?,
.else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable", .else_relocs = if (switch_br.else_body_len > 0) .{ .forward = .empty } else .@"unreachable",
}); });
defer if (is_loop) { defer if (is_loop) {
var loop_switch_data = self.loop_switches.fetchRemove(inst).?.value; var loop_switch_data = cg.loop_switches.fetchRemove(inst).?.value;
switch (loop_switch_data.else_relocs) { switch (loop_switch_data.else_relocs) {
.@"unreachable", .backward => {}, .@"unreachable", .backward => {},
.forward => |*else_relocs| else_relocs.deinit(self.gpa), .forward => |*else_relocs| else_relocs.deinit(cg.gpa),
} }
}; };
var cases_it = switch_br.iterateCases(); var cases_it = switch_br.iterateCases();
while (cases_it.next()) |case| { while (cases_it.next()) |case| {
{ {
const table = self.mir_table.items[table_start..][0..table_len]; const table = cg.mir_table.items[table_start..][0..table_len];
for (case.items) |item| { for (case.items) |item| {
const val = Value.fromInterned(item.toInterned().?); const val = Value.fromInterned(item.toInterned().?);
var val_space: Value.BigIntSpace = undefined; var val_space: Value.BigIntSpace = undefined;
const val_bigint = val.toBigInt(&val_space, zcu); const val_bigint = val.toBigInt(&val_space, zcu);
var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined }; var index_bigint: std.math.big.int.Mutable = .{ .limbs = limbs, .positive = undefined, .len = undefined };
index_bigint.sub(val_bigint, min_bigint); index_bigint.sub(val_bigint, min_bigint);
table[index_bigint.toConst().to(u10) catch unreachable] = @intCast(self.mir_instructions.len); table[index_bigint.toConst().to(u10) catch unreachable] = @intCast(cg.mir_instructions.len);
} }
for (case.ranges) |range| { for (case.ranges) |range| {
var low_space: Value.BigIntSpace = undefined; var low_space: Value.BigIntSpace = undefined;
@ -93772,14 +93840,14 @@ fn lowerSwitchBr(
const start = index_bigint.toConst().to(u10) catch unreachable; const start = index_bigint.toConst().to(u10) catch unreachable;
index_bigint.sub(high_bigint, min_bigint); index_bigint.sub(high_bigint, min_bigint);
const end = @as(u11, index_bigint.toConst().to(u10) catch unreachable) + 1; const end = @as(u11, index_bigint.toConst().to(u10) catch unreachable) + 1;
@memset(table[start..end], @intCast(self.mir_instructions.len)); @memset(table[start..end], @intCast(cg.mir_instructions.len));
} }
} }
for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand); for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand);
try self.genBodyBlock(case.body); try cg.genBodyBlock(case.body);
try self.restoreState(state, &.{}, .{ try cg.restoreState(state, &.{}, .{
.emit_instructions = false, .emit_instructions = false,
.update_tracking = true, .update_tracking = true,
.resurrect = true, .resurrect = true,
@ -93790,21 +93858,21 @@ fn lowerSwitchBr(
const else_body = cases_it.elseBody(); const else_body = cases_it.elseBody();
const else_deaths = liveness.deaths.len - 1; const else_deaths = liveness.deaths.len - 1;
for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand); for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand);
self.performReloc(else_reloc); cg.performReloc(else_reloc);
if (is_loop) { if (is_loop) {
const loop_switch_data = self.loop_switches.getPtr(inst).?; const loop_switch_data = cg.loop_switches.getPtr(inst).?;
for (loop_switch_data.else_relocs.forward.items) |reloc| self.performReloc(reloc); for (loop_switch_data.else_relocs.forward.items) |reloc| cg.performReloc(reloc);
loop_switch_data.else_relocs.forward.deinit(self.gpa); loop_switch_data.else_relocs.forward.deinit(cg.gpa);
loop_switch_data.else_relocs = .{ .backward = @intCast(self.mir_instructions.len) }; loop_switch_data.else_relocs = .{ .backward = @intCast(cg.mir_instructions.len) };
} }
for (self.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) { for (cg.mir_table.items[table_start..][0..table_len]) |*entry| if (entry.* == else_reloc_marker) {
entry.* = @intCast(self.mir_instructions.len); entry.* = @intCast(cg.mir_instructions.len);
}; };
try self.genBodyBlock(else_body); try cg.genBodyBlock(else_body);
try self.restoreState(state, &.{}, .{ try cg.restoreState(state, &.{}, .{
.emit_instructions = false, .emit_instructions = false,
.update_tracking = true, .update_tracking = true,
.resurrect = true, .resurrect = true,
@ -93819,9 +93887,12 @@ fn lowerSwitchBr(
const relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len); const relocs = try allocator.alloc(Mir.Inst.Index, case.items.len + case.ranges.len);
defer allocator.free(relocs); defer allocator.free(relocs);
try self.spillEflagsIfOccupied(); var cond_temp = try cg.tempInit(condition_ty, condition);
const reset_index = cg.next_temp_index;
try cg.spillEflagsIfOccupied();
for (case.items, relocs[0..case.items.len]) |item, *reloc| { for (case.items, relocs[0..case.items.len]) |item, *reloc| {
const item_mcv = try self.resolveInst(item); const item_mcv = try cg.resolveInst(item);
const cc: Condition = switch (condition) { const cc: Condition = switch (condition) {
.eflags => |cc| switch (item_mcv.immediate) { .eflags => |cc| switch (item_mcv.immediate) {
0 => cc.negate(), 0 => cc.negate(),
@ -93829,27 +93900,24 @@ fn lowerSwitchBr(
else => unreachable, else => unreachable,
}, },
else => cc: { else => cc: {
var cond_temp = try self.tempInit(condition_ty, condition); var item_temp = try cg.tempInit(condition_ty, item_mcv);
var item_temp = try self.tempInit(condition_ty, item_mcv); const cc_temp = cond_temp.cmpInts(.eq, &item_temp, cg) catch |err| switch (err) {
const cc_temp = cond_temp.cmpInts(.eq, &item_temp, self) catch |err| switch (err) {
error.SelectFailed => unreachable, error.SelectFailed => unreachable,
else => |e| return e, else => |e| return e,
}; };
try cond_temp.die(self); try item_temp.die(cg);
try item_temp.die(self); const cc = cc_temp.tracking(cg).short.eflags;
const cc = cc_temp.tracking(self).short.eflags; try cc_temp.die(cg);
try cc_temp.die(self); try cg.resetTemps(reset_index);
try self.resetTemps();
break :cc cc; break :cc cc;
}, },
}; };
reloc.* = try self.asmJccReloc(cc, undefined); reloc.* = try cg.asmJccReloc(cc, undefined);
} }
for (case.ranges, relocs[case.items.len..]) |range, *reloc| { for (case.ranges, relocs[case.items.len..]) |range, *reloc| {
var cond_temp = try self.tempInit(condition_ty, condition); const min_mcv = try cg.resolveInst(range[0]);
const min_mcv = try self.resolveInst(range[0]); const max_mcv = try cg.resolveInst(range[1]);
const max_mcv = try self.resolveInst(range[1]);
// `null` means always false. // `null` means always false.
const lt_min = cc: switch (condition) { const lt_min = cc: switch (condition) {
.eflags => |cc| switch (min_mcv.immediate) { .eflags => |cc| switch (min_mcv.immediate) {
@ -93858,19 +93926,19 @@ fn lowerSwitchBr(
else => unreachable, else => unreachable,
}, },
else => { else => {
var min_temp = try self.tempInit(condition_ty, min_mcv); var min_temp = try cg.tempInit(condition_ty, min_mcv);
const cc_temp = cond_temp.cmpInts(.lt, &min_temp, self) catch |err| switch (err) { const cc_temp = cond_temp.cmpInts(.lt, &min_temp, cg) catch |err| switch (err) {
error.SelectFailed => unreachable, error.SelectFailed => unreachable,
else => |e| return e, else => |e| return e,
}; };
try min_temp.die(self); try min_temp.die(cg);
const cc = cc_temp.tracking(self).short.eflags; const cc = cc_temp.tracking(cg).short.eflags;
try cc_temp.die(self); try cc_temp.die(cg);
break :cc cc; break :cc cc;
}, },
}; };
const lt_min_reloc = if (lt_min) |cc| r: { const lt_min_reloc = if (lt_min) |cc| r: {
break :r try self.asmJccReloc(cc, undefined); break :r try cg.asmJccReloc(cc, undefined);
} else null; } else null;
// `null` means always true. // `null` means always true.
const lte_max = switch (condition) { const lte_max = switch (condition) {
@ -93880,38 +93948,41 @@ fn lowerSwitchBr(
else => unreachable, else => unreachable,
}, },
else => cc: { else => cc: {
var max_temp = try self.tempInit(condition_ty, max_mcv); var max_temp = try cg.tempInit(condition_ty, max_mcv);
const cc_temp = cond_temp.cmpInts(.lte, &max_temp, self) catch |err| switch (err) { const cc_temp = cond_temp.cmpInts(.lte, &max_temp, cg) catch |err| switch (err) {
error.SelectFailed => unreachable, error.SelectFailed => unreachable,
else => |e| return e, else => |e| return e,
}; };
try max_temp.die(self); try max_temp.die(cg);
const cc = cc_temp.tracking(self).short.eflags; const cc = cc_temp.tracking(cg).short.eflags;
try cc_temp.die(self); try cc_temp.die(cg);
break :cc cc; break :cc cc;
}, },
}; };
try cond_temp.die(self); try cg.resetTemps(reset_index);
try self.resetTemps();
// "Success" case is in `reloc`.... // "Success" case is in `reloc`....
if (lte_max) |cc| { if (lte_max) |cc| {
reloc.* = try self.asmJccReloc(cc, undefined); reloc.* = try cg.asmJccReloc(cc, undefined);
} else { } else {
reloc.* = try self.asmJmpReloc(undefined); reloc.* = try cg.asmJmpReloc(undefined);
} }
// ...and "fail" case falls through to next checks. // ...and "fail" case falls through to next checks.
if (lt_min_reloc) |r| self.performReloc(r); if (lt_min_reloc) |r| cg.performReloc(r);
} }
// The jump to skip this case if the conditions all failed. try cond_temp.die(cg);
const skip_case_reloc = try self.asmJmpReloc(undefined); try cg.resetTemps(@enumFromInt(0));
cg.checkInvariantsAfterAirInst();
for (liveness.deaths[case.idx]) |operand| try self.processDeath(operand); // The jump to skip this case if the conditions all failed.
const skip_case_reloc = try cg.asmJmpReloc(undefined);
for (liveness.deaths[case.idx]) |operand| try cg.processDeath(operand);
// Relocate all success cases to the body we're about to generate. // Relocate all success cases to the body we're about to generate.
for (relocs) |reloc| self.performReloc(reloc); for (relocs) |reloc| cg.performReloc(reloc);
try self.genBodyBlock(case.body); try cg.genBodyBlock(case.body);
try self.restoreState(state, &.{}, .{ try cg.restoreState(state, &.{}, .{
.emit_instructions = false, .emit_instructions = false,
.update_tracking = true, .update_tracking = true,
.resurrect = true, .resurrect = true,
@ -93919,16 +93990,16 @@ fn lowerSwitchBr(
}); });
// Relocate the "skip" branch to fall through to the next case. // Relocate the "skip" branch to fall through to the next case.
self.performReloc(skip_case_reloc); cg.performReloc(skip_case_reloc);
} }
if (switch_br.else_body_len > 0) { if (switch_br.else_body_len > 0) {
const else_body = cases_it.elseBody(); const else_body = cases_it.elseBody();
const else_deaths = liveness.deaths.len - 1; const else_deaths = liveness.deaths.len - 1;
for (liveness.deaths[else_deaths]) |operand| try self.processDeath(operand); for (liveness.deaths[else_deaths]) |operand| try cg.processDeath(operand);
try self.genBodyBlock(else_body); try cg.genBodyBlock(else_body);
try self.restoreState(state, &.{}, .{ try cg.restoreState(state, &.{}, .{
.emit_instructions = false, .emit_instructions = false,
.update_tracking = true, .update_tracking = true,
.resurrect = true, .resurrect = true,
@ -95003,7 +95074,7 @@ fn moveStrategy(cg: *CodeGen, ty: Type, class: Register.Class, aligned: bool) !M
.mmx => {}, .mmx => {},
.sse => switch (ty.zigTypeTag(zcu)) { .sse => switch (ty.zigTypeTag(zcu)) {
else => { else => {
const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target.*, .other), .none); const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, cg.target, .other), .none);
assert(std.mem.indexOfNone(abi.Class, classes, &.{ assert(std.mem.indexOfNone(abi.Class, classes, &.{
.integer, .sse, .sseup, .memory, .float, .float_combine, .integer, .sse, .sseup, .memory, .float, .float_combine,
}) == null); }) == null);
@ -99635,7 +99706,7 @@ fn airVaArg(self: *CodeGen, inst: Air.Inst.Index) !void {
const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } }; const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } };
const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } }; const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } };
const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none); const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target, .arg), .none);
switch (classes[0]) { switch (classes[0]) {
.integer => { .integer => {
assert(classes.len == 1); assert(classes.len == 1);
@ -99980,7 +100051,7 @@ fn resolveCallingConventionValues(
var ret_tracking_i: usize = 0; var ret_tracking_i: usize = 0;
const classes = switch (cc) { const classes = switch (cc) {
.x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none), .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target, .ret), .none),
.x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)}, .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)},
else => unreachable, else => unreachable,
}; };
@ -100069,7 +100140,7 @@ fn resolveCallingConventionValues(
var arg_mcv_i: usize = 0; var arg_mcv_i: usize = 0;
const classes = switch (cc) { const classes = switch (cc) {
.x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none), .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target, .arg), .none),
.x86_64_win => &.{abi.classifyWindows(ty, zcu)}, .x86_64_win => &.{abi.classifyWindows(ty, zcu)},
else => unreachable, else => unreachable,
}; };
@ -100373,7 +100444,7 @@ fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Ty
error.DivisionByZero => unreachable, error.DivisionByZero => unreachable,
error.UnexpectedRemainder => {}, error.UnexpectedRemainder => {},
}; };
const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none); const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target, .other), .none);
if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| { if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| {
part.* = switch (class) { part.* = switch (class) {
.integer => if (part_i < parts_len - 1) .integer => if (part_i < parts_len - 1)
@ -101339,6 +101410,7 @@ const Temp = struct {
const val_mcv = val.tracking(cg).short; const val_mcv = val.tracking(cg).short;
switch (val_mcv) { switch (val_mcv) {
else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }), else => |mcv| std.debug.panic("{s}: {}\n", .{ @src().fn_name, mcv }),
.none => {},
.undef => if (opts.safe) { .undef => if (opts.safe) {
var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp)); var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address().offset(opts.disp));
var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa }); var pat = try cg.tempInit(.u8, .{ .immediate = 0xaa });
@ -101371,19 +101443,19 @@ const Temp = struct {
.disp = opts.disp, .disp = opts.disp,
}), }),
), ),
.register => |val_reg| try dst.writeRegs(opts.disp, val_ty, &.{registerAlias( .register => |val_reg| try dst.writeReg(opts.disp, val_ty, registerAlias(
val_reg, val_reg,
@intCast(val_ty.abiSize(cg.pt.zcu)), @intCast(val_ty.abiSize(cg.pt.zcu)),
)}, cg), ), cg),
inline .register_pair, inline .register_pair,
.register_triple, .register_triple,
.register_quadruple, .register_quadruple,
=> |val_regs| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg), => |val_regs| try dst.writeRegs(opts.disp, val_ty, &val_regs, cg),
.register_offset => |val_reg_off| switch (val_reg_off.off) { .register_offset => |val_reg_off| switch (val_reg_off.off) {
0 => try dst.writeRegs(opts.disp, val_ty, &.{registerAlias( 0 => try dst.writeReg(opts.disp, val_ty, registerAlias(
val_reg_off.reg, val_reg_off.reg,
@intCast(val_ty.abiSize(cg.pt.zcu)), @intCast(val_ty.abiSize(cg.pt.zcu)),
)}, cg), ), cg),
else => continue :val_to_gpr, else => continue :val_to_gpr,
}, },
.register_overflow => |val_reg_ov| { .register_overflow => |val_reg_ov| {
@ -101401,7 +101473,7 @@ const Temp = struct {
else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }), else => std.debug.panic("{s}: {}\n", .{ @src().fn_name, val_ty.fmt(cg.pt) }),
}); });
const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu)); const first_size: u31 = @intCast(first_ty.abiSize(cg.pt.zcu));
try dst.writeRegs(opts.disp, first_ty, &.{registerAlias(val_reg_ov.reg, first_size)}, cg); try dst.writeReg(opts.disp, first_ty, registerAlias(val_reg_ov.reg, first_size), cg);
try cg.asmSetccMemory( try cg.asmSetccMemory(
val_reg_ov.eflags, val_reg_ov.eflags,
try dst.tracking(cg).short.mem(cg, .{ try dst.tracking(cg).short.mem(cg, .{
@ -101492,44 +101564,78 @@ const Temp = struct {
})); }));
} }
fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void { fn writeReg(dst: Temp, disp: i32, src_ty: Type, src_reg: Register, cg: *CodeGen) InnerError!void {
var part_disp = disp; const src_abi_size: u31 = @intCast(src_ty.abiSize(cg.pt.zcu));
var src_abi_size: u32 = @intCast(src_ty.abiSize(cg.pt.zcu)); const src_rc = src_reg.class();
for (src_regs) |src_reg| { if (src_rc == .x87 or std.math.isPowerOfTwo(src_abi_size)) {
const src_rc = src_reg.class(); const strat = try cg.moveStrategy(src_ty, src_rc, false);
const part_bit_size = @min(8 * src_abi_size, src_reg.bitSize()); try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{
const part_size = @divExact(part_bit_size, 8); .size = .fromBitSize(@min(8 * src_abi_size, src_reg.bitSize())),
if (src_rc == .x87 or std.math.isPowerOfTwo(part_size)) { .disp = disp,
const strat = try cg.moveStrategy(src_ty, src_rc, false); }), registerAlias(src_reg, src_abi_size));
try strat.write(cg, try dst.tracking(cg).short.mem(cg, .{ } else {
.size = .fromBitSize(part_bit_size), const frame_size = std.math.ceilPowerOfTwoAssert(u32, src_abi_size);
.disp = part_disp, const frame_index = try cg.allocFrameIndex(.init(.{
}), registerAlias(src_reg, part_size)); .size = frame_size,
} else { .alignment = .fromNonzeroByteUnits(frame_size),
const frame_size = std.math.ceilPowerOfTwoAssert(u32, part_size); }));
const frame_index = try cg.allocFrameIndex(.init(.{ const strat = try cg.moveStrategy(src_ty, src_rc, true);
.size = frame_size, try strat.write(cg, .{
.alignment = .fromNonzeroByteUnits(frame_size), .base = .{ .frame = frame_index },
})); .mod = .{ .rm = .{ .size = .fromSize(frame_size) } },
const strat = try cg.moveStrategy(src_ty, src_rc, true); }, registerAlias(src_reg, frame_size));
try strat.write(cg, .{ var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address());
.base = .{ .frame = frame_index }, try dst_ptr.toOffset(disp, cg);
.mod = .{ .rm = .{ .size = .fromSize(frame_size) } }, var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } });
}, registerAlias(src_reg, frame_size)); var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size });
var dst_ptr = try cg.tempInit(.usize, dst.tracking(cg).short.address()); try dst_ptr.memcpy(&src_ptr, &len, cg);
try dst_ptr.toOffset(part_disp, cg); try dst_ptr.die(cg);
var src_ptr = try cg.tempInit(.usize, .{ .lea_frame = .{ .index = frame_index } }); try src_ptr.die(cg);
var len = try cg.tempInit(.usize, .{ .immediate = src_abi_size }); try len.die(cg);
try dst_ptr.memcpy(&src_ptr, &len, cg);
try dst_ptr.die(cg);
try src_ptr.die(cg);
try len.die(cg);
}
part_disp += part_size;
src_abi_size -= part_size;
} }
} }
fn writeRegs(dst: Temp, disp: i32, src_ty: Type, src_regs: []const Register, cg: *CodeGen) InnerError!void {
const zcu = cg.pt.zcu;
const classes = std.mem.sliceTo(&abi.classifySystemV(src_ty, zcu, cg.target, .other), .none);
var next_class_index: u4 = 0;
var part_disp = disp;
var remaining_abi_size = src_ty.abiSize(zcu);
for (src_regs) |src_reg| {
const class_index = next_class_index;
const class = classes[class_index];
next_class_index = @intCast(switch (class) {
.integer, .memory, .float, .float_combine => class_index + 1,
.sse => std.mem.indexOfNonePos(abi.Class, classes, class_index + 1, &.{.sseup}) orelse classes.len,
.x87 => std.mem.indexOfNonePos(abi.Class, classes, class_index + 1, &.{.x87up}) orelse classes.len,
.sseup, .x87up, .complex_x87, .none, .win_i128, .integer_per_element => unreachable,
});
const part_size = switch (class) {
.integer, .sse, .memory => @min(8 * @as(u7, next_class_index - class_index), remaining_abi_size),
.x87 => 16,
.float => 4,
.float_combine => 8,
.sseup, .x87up, .complex_x87, .none, .win_i128, .integer_per_element => unreachable,
};
try dst.writeReg(part_disp, switch (class) {
.integer => .u64,
.sse => switch (part_size) {
else => unreachable,
8 => .f64,
16 => .vector_2_f64,
32 => .vector_4_f64,
},
.x87 => .f80,
.float => .f32,
.float_combine => .vector_2_f32,
.sseup, .x87up, .complex_x87, .memory, .none, .win_i128, .integer_per_element => unreachable,
}, src_reg, cg);
part_disp += part_size;
remaining_abi_size -= part_size;
}
assert(next_class_index == classes.len);
}
fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) InnerError!void { fn memcpy(dst: *Temp, src: *Temp, len: *Temp, cg: *CodeGen) InnerError!void {
while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| { while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| {
if (try temp.toReg(reg, cg)) break; if (try temp.toReg(reg, cg)) break;
@ -105786,9 +105892,9 @@ const Temp = struct {
}; };
}; };
fn resetTemps(cg: *CodeGen) InnerError!void { fn resetTemps(cg: *CodeGen, from_index: Temp.Index) InnerError!void {
var any_valid = false; var any_valid = false;
for (0..@intFromEnum(cg.next_temp_index)) |temp_index| { for (@intFromEnum(from_index)..@intFromEnum(cg.next_temp_index)) |temp_index| {
const temp: Temp.Index = @enumFromInt(temp_index); const temp: Temp.Index = @enumFromInt(temp_index);
if (temp.isValid(cg)) { if (temp.isValid(cg)) {
any_valid = true; any_valid = true;
@ -105800,7 +105906,7 @@ fn resetTemps(cg: *CodeGen) InnerError!void {
cg.temp_type[temp_index] = undefined; cg.temp_type[temp_index] = undefined;
} }
if (any_valid) return cg.fail("failed to kill all temps", .{}); if (any_valid) return cg.fail("failed to kill all temps", .{});
cg.next_temp_index = @enumFromInt(0); cg.next_temp_index = from_index;
} }
fn reuseTemp( fn reuseTemp(
@ -105889,70 +105995,75 @@ fn tempMemFromValue(cg: *CodeGen, value: Value) InnerError!Temp {
return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerUav(value)); return cg.tempInit(value.typeOf(cg.pt.zcu), try cg.lowerUav(value));
} }
fn tempFromOperand( fn tempFromOperand(cg: *CodeGen, op_ref: Air.Inst.Ref, op_dies: bool) InnerError!Temp {
cg: *CodeGen,
inst: Air.Inst.Index,
op_index: Liveness.OperandInt,
op_ref: Air.Inst.Ref,
ignore_death: bool,
) InnerError!Temp {
const zcu = cg.pt.zcu; const zcu = cg.pt.zcu;
const ip = &zcu.intern_pool; const ip = &zcu.intern_pool;
if (ignore_death or !cg.liveness.operandDies(inst, op_index)) { if (op_dies) {
if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; const temp_index = cg.next_temp_index;
const val = op_ref.toInterned().?; const temp: Temp = .{ .index = temp_index.toIndex() };
const gop = try cg.const_tracking.getOrPut(cg.gpa, val); const op_inst = op_ref.toIndex().?;
if (!gop.found_existing) gop.value_ptr.* = .init(init: { const tracking = cg.getResolvedInstValue(op_inst);
const const_mcv = try cg.genTypedValue(.fromInterned(val)); temp_index.tracking(cg).* = tracking.*;
switch (const_mcv) { if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
.lea_tlv => |tlv_sym| switch (cg.bin_file.tag) { cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
.elf, .macho => { cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
if (cg.mod.pic) { return temp;
try cg.spillRegisters(&.{ .rdi, .rax }); }
} else {
try cg.spillRegisters(&.{.rax}); if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst };
} const val = op_ref.toInterned().?;
const frame_index = try cg.allocFrameIndex(.init(.{ const gop = try cg.const_tracking.getOrPut(cg.gpa, val);
.size = 8, if (!gop.found_existing) gop.value_ptr.* = .init(init: {
.alignment = .@"8", const const_mcv = try cg.genTypedValue(.fromInterned(val));
})); switch (const_mcv) {
try cg.genSetMem( .lea_tlv => |tlv_sym| switch (cg.bin_file.tag) {
.{ .frame = frame_index }, .elf, .macho => {
0, if (cg.mod.pic) {
.usize, try cg.spillRegisters(&.{ .rdi, .rax });
.{ .lea_symbol = .{ .sym_index = tlv_sym } }, } else {
.{}, try cg.spillRegisters(&.{.rax});
); }
break :init .{ .load_frame = .{ .index = frame_index } }; const frame_index = try cg.allocFrameIndex(.init(.{
}, .size = 8,
else => break :init const_mcv, .alignment = .@"8",
}));
try cg.genSetMem(
.{ .frame = frame_index },
0,
.usize,
.{ .lea_symbol = .{ .sym_index = tlv_sym } },
.{},
);
break :init .{ .load_frame = .{ .index = frame_index } };
}, },
else => break :init const_mcv, else => break :init const_mcv,
} },
}); else => break :init const_mcv,
return cg.tempInit(.fromInterned(ip.typeOf(val)), gop.value_ptr.short); }
} });
return cg.tempInit(.fromInterned(ip.typeOf(val)), gop.value_ptr.short);
const temp_index = cg.next_temp_index;
const temp: Temp = .{ .index = temp_index.toIndex() };
const op_inst = op_ref.toIndex().?;
const tracking = cg.getResolvedInstValue(op_inst);
temp_index.tracking(cg).* = tracking.*;
if (!cg.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? };
cg.temp_type[@intFromEnum(temp_index)] = cg.typeOf(op_ref);
cg.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1);
return temp;
} }
inline fn tempsFromOperands(cg: *CodeGen, inst: Air.Inst.Index, op_refs: anytype) InnerError![op_refs.len]Temp { fn tempsFromOperandsInner(
var temps: [op_refs.len]Temp = undefined; cg: *CodeGen,
inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| { inst: Air.Inst.Index,
temp.* = try cg.tempFromOperand(inst, op_index, op_ref, inline for (0..op_index) |prev_op_index| { op_temps: []Temp,
if (op_ref == op_refs[prev_op_index]) break true; op_refs: []const Air.Inst.Ref,
} else false); ) InnerError!void {
} for (op_temps, 0.., op_refs) |*op_temp, op_index, op_ref| op_temp.* = try cg.tempFromOperand(op_ref, for (op_refs[0..op_index]) |prev_op_ref| {
return temps; if (op_ref == prev_op_ref) break false;
} else cg.liveness.operandDies(inst, @intCast(op_index)));
}
inline fn tempsFromOperands(
cg: *CodeGen,
inst: Air.Inst.Index,
op_refs: anytype,
) InnerError![op_refs.len]Temp {
var op_temps: [op_refs.len]Temp = undefined;
try cg.tempsFromOperandsInner(inst, &op_temps, &op_refs);
return op_temps;
} }
const Operand = union(enum) { const Operand = union(enum) {

8
src/arch/x86_64/abi.zig
View file

@ -100,7 +100,7 @@ pub const Context = enum { ret, arg, field, other };
/// There are a maximum of 8 possible return slots. Returned values are in /// There are a maximum of 8 possible return slots. Returned values are in
/// the beginning of the array; unused slots are filled with .none. /// the beginning of the array; unused slots are filled with .none.
pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8]Class { pub fn classifySystemV(ty: Type, zcu: *Zcu, target: *const std.Target, ctx: Context) [8]Class {
const memory_class = [_]Class{ const memory_class = [_]Class{
.memory, .none, .none, .none, .memory, .none, .none, .none,
.none, .none, .none, .none, .none, .none, .none, .none,
@ -148,7 +148,7 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8
result[0] = .integer; result[0] = .integer;
return result; return result;
}, },
.float => switch (ty.floatBits(target)) { .float => switch (ty.floatBits(target.*)) {
16 => { 16 => {
if (ctx == .field) { if (ctx == .field) {
result[0] = .memory; result[0] = .memory;
@ -330,7 +330,7 @@ fn classifySystemVStruct(
starting_byte_offset: u64, starting_byte_offset: u64,
loaded_struct: InternPool.LoadedStructType, loaded_struct: InternPool.LoadedStructType,
zcu: *Zcu, zcu: *Zcu,
target: std.Target, target: *const std.Target,
) u64 { ) u64 {
const ip = &zcu.intern_pool; const ip = &zcu.intern_pool;
var byte_offset = starting_byte_offset; var byte_offset = starting_byte_offset;
@ -379,7 +379,7 @@ fn classifySystemVUnion(
starting_byte_offset: u64, starting_byte_offset: u64,
loaded_union: InternPool.LoadedUnionType, loaded_union: InternPool.LoadedUnionType,
zcu: *Zcu, zcu: *Zcu,
target: std.Target, target: *const std.Target,
) u64 { ) u64 {
const ip = &zcu.intern_pool; const ip = &zcu.intern_pool;
for (0..loaded_union.field_types.len) |field_index| { for (0..loaded_union.field_types.len) |field_index| {

6
src/codegen/llvm.zig
View file

@ -11757,7 +11757,7 @@ fn firstParamSRet(fn_info: InternPool.Key.FuncType, zcu: *Zcu, target: std.Targe
} }
fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: std.Target) bool { fn firstParamSRetSystemV(ty: Type, zcu: *Zcu, target: std.Target) bool {
const class = x86_64_abi.classifySystemV(ty, zcu, target, .ret); const class = x86_64_abi.classifySystemV(ty, zcu, &target, .ret);
if (class[0] == .memory) return true; if (class[0] == .memory) return true;
if (class[0] == .x87 and class[2] != .none) return true; if (class[0] == .x87 and class[2] != .none) return true;
return false; return false;
@ -11867,7 +11867,7 @@ fn lowerSystemVFnRetTy(o: *Object, fn_info: InternPool.Key.FuncType) Allocator.E
return o.lowerType(return_type); return o.lowerType(return_type);
} }
const target = zcu.getTarget(); const target = zcu.getTarget();
const classes = x86_64_abi.classifySystemV(return_type, zcu, target, .ret); const classes = x86_64_abi.classifySystemV(return_type, zcu, &target, .ret);
if (classes[0] == .memory) return .void; if (classes[0] == .memory) return .void;
var types_index: u32 = 0; var types_index: u32 = 0;
var types_buffer: [8]Builder.Type = undefined; var types_buffer: [8]Builder.Type = undefined;
@ -12145,7 +12145,7 @@ const ParamTypeIterator = struct {
const zcu = it.object.pt.zcu; const zcu = it.object.pt.zcu;
const ip = &zcu.intern_pool; const ip = &zcu.intern_pool;
const target = zcu.getTarget(); const target = zcu.getTarget();
const classes = x86_64_abi.classifySystemV(ty, zcu, target, .arg); const classes = x86_64_abi.classifySystemV(ty, zcu, &target, .arg);
if (classes[0] == .memory) { if (classes[0] == .memory) {
it.zig_index += 1; it.zig_index += 1;
it.llvm_index += 1; it.llvm_index += 1;

2
src/main.zig
View file

@ -39,7 +39,7 @@ test {
_ = Package; _ = Package;
} }
const thread_stack_size = 32 << 20; const thread_stack_size = 50 << 20;
pub const std_options: std.Options = .{ pub const std_options: std.Options = .{
.wasiCwd = wasi_cwd, .wasiCwd = wasi_cwd,