mirrors/zig
1
0
Fork 0
mirror of https://codeberg.org/ziglang/zig.git synced 2025-12-06 13:54:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #22903 from alexrp/llvm-nobuiltin-memcpy-inline

Browse source 
`llvm`: Use inline variants of `memcpy`/`memset` intrinsics when using `-fno-builtin`
This commit is contained in:
Alex Rønne Petersen 2025-02-23 00:51:40 +01:00 • committed by GitHub
commit 5e203e157b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 83 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
src/codegen/llvm.zig
View file

@ -5771,6 +5771,7 @@ pub const FuncGen = struct {
try o.builder.intValue(.i8, 0xaa), try o.builder.intValue(.i8, 0xaa),
len, len,
if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
self.ng.ownerModule().no_builtin,
); );
const owner_mod = self.ng.ownerModule(); const owner_mod = self.ng.ownerModule();
if (owner_mod.valgrind) { if (owner_mod.valgrind) {
@ -5821,6 +5822,7 @@ pub const FuncGen = struct {
try o.builder.intValue(.i8, 0xaa), try o.builder.intValue(.i8, 0xaa),
len, len,
.normal, .normal,
self.ng.ownerModule().no_builtin,
); );
const owner_mod = self.ng.ownerModule(); const owner_mod = self.ng.ownerModule();
if (owner_mod.valgrind) { if (owner_mod.valgrind) {
@ -5902,7 +5904,7 @@ pub const FuncGen = struct {
const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment); const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);
_ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{}, &.{ dest_list, src_list }, ""); _ = try self.wip.callIntrinsic(.normal, .none, .va_copy, &.{dest_list.typeOfWip(&self.wip)}, &.{ dest_list, src_list }, "");
return if (isByRef(va_list_ty, zcu)) return if (isByRef(va_list_ty, zcu))
dest_list dest_list
else else
@ -5913,7 +5915,7 @@ pub const FuncGen = struct {
const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op; const un_op = self.air.instructions.items(.data)[@intFromEnum(inst)].un_op;
const src_list = try self.resolveInst(un_op); const src_list = try self.resolveInst(un_op);
_ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{}, &.{src_list}, ""); _ = try self.wip.callIntrinsic(.normal, .none, .va_end, &.{src_list.typeOfWip(&self.wip)}, &.{src_list}, "");
return .none; return .none;
} }
@ -5927,7 +5929,7 @@ pub const FuncGen = struct {
const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm(); const result_alignment = va_list_ty.abiAlignment(pt.zcu).toLlvm();
const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment); const dest_list = try self.buildAllocaWorkaround(va_list_ty, result_alignment);
_ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{}, &.{dest_list}, ""); _ = try self.wip.callIntrinsic(.normal, .none, .va_start, &.{dest_list.typeOfWip(&self.wip)}, &.{dest_list}, "");
return if (isByRef(va_list_ty, zcu)) return if (isByRef(va_list_ty, zcu))
dest_list dest_list
else else
@ -9734,6 +9736,7 @@ pub const FuncGen = struct {
if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8), if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
len, len,
if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal, if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
self.ng.ownerModule().no_builtin,
); );
if (safety and owner_mod.valgrind) { if (safety and owner_mod.valgrind) {
try self.valgrindMarkUndef(dest_ptr, len); try self.valgrindMarkUndef(dest_ptr, len);
@ -10041,9 +10044,22 @@ pub const FuncGen = struct {
try o.builder.undefValue(.i8); try o.builder.undefValue(.i8);
const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
if (intrinsic_len0_traps) { if (intrinsic_len0_traps) {
try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind); try self.safeWasmMemset(
dest_ptr,
fill_byte,
len,
dest_ptr_align,
access_kind,
);
} else { } else {
_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind); _ = try self.wip.callMemSet(
dest_ptr,
dest_ptr_align,
fill_byte,
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
} }
const owner_mod = self.ng.ownerModule(); const owner_mod = self.ng.ownerModule();
if (safety and owner_mod.valgrind) { if (safety and owner_mod.valgrind) {
@ -10060,9 +10076,22 @@ pub const FuncGen = struct {
const fill_byte = try o.builder.intValue(.i8, byte_val); const fill_byte = try o.builder.intValue(.i8, byte_val);
const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
if (intrinsic_len0_traps) { if (intrinsic_len0_traps) {
try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind); try self.safeWasmMemset(
dest_ptr,
fill_byte,
len,
dest_ptr_align,
access_kind,
);
} else { } else {
_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind); _ = try self.wip.callMemSet(
dest_ptr,
dest_ptr_align,
fill_byte,
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
} }
return .none; return .none;
} }
@ -10077,9 +10106,22 @@ pub const FuncGen = struct {
const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty); const len = try self.sliceOrArrayLenInBytes(dest_slice, ptr_ty);
if (intrinsic_len0_traps) { if (intrinsic_len0_traps) {
try self.safeWasmMemset(dest_ptr, fill_byte, len, dest_ptr_align, access_kind); try self.safeWasmMemset(
dest_ptr,
fill_byte,
len,
dest_ptr_align,
access_kind,
);
} else { } else {
_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind); _ = try self.wip.callMemSet(
dest_ptr,
dest_ptr_align,
fill_byte,
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
} }
return .none; return .none;
} }
@ -10131,6 +10173,7 @@ pub const FuncGen = struct {
elem_abi_align.toLlvm(), elem_abi_align.toLlvm(),
try o.builder.intValue(llvm_usize_ty, elem_abi_size), try o.builder.intValue(llvm_usize_ty, elem_abi_size),
access_kind, access_kind,
self.ng.ownerModule().no_builtin,
); );
} else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align); } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{ const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
@ -10158,7 +10201,14 @@ pub const FuncGen = struct {
const end_block = try self.wip.block(2, "MemsetTrapEnd"); const end_block = try self.wip.block(2, "MemsetTrapEnd");
_ = try self.wip.brCond(cond, memset_block, end_block, .none); _ = try self.wip.brCond(cond, memset_block, end_block, .none);
self.wip.cursor = .{ .block = memset_block }; self.wip.cursor = .{ .block = memset_block };
_ = try self.wip.callMemSet(dest_ptr, dest_ptr_align, fill_byte, len, access_kind); _ = try self.wip.callMemSet(
dest_ptr,
dest_ptr_align,
fill_byte,
len,
access_kind,
self.ng.ownerModule().no_builtin,
);
_ = try self.wip.br(end_block); _ = try self.wip.br(end_block);
self.wip.cursor = .{ .block = end_block }; self.wip.cursor = .{ .block = end_block };
} }
@ -10200,6 +10250,7 @@ pub const FuncGen = struct {
src_ptr_ty.ptrAlignment(zcu).toLlvm(), src_ptr_ty.ptrAlignment(zcu).toLlvm(),
len, len,
access_kind, access_kind,
self.ng.ownerModule().no_builtin,
); );
_ = try self.wip.br(end_block); _ = try self.wip.br(end_block);
self.wip.cursor = .{ .block = end_block }; self.wip.cursor = .{ .block = end_block };
@ -10213,6 +10264,7 @@ pub const FuncGen = struct {
src_ptr_ty.ptrAlignment(zcu).toLlvm(), src_ptr_ty.ptrAlignment(zcu).toLlvm(),
len, len,
access_kind, access_kind,
self.ng.ownerModule().no_builtin,
); );
return .none; return .none;
} }
@ -11346,6 +11398,7 @@ pub const FuncGen = struct {
ptr_alignment, ptr_alignment,
try o.builder.intValue(try o.lowerType(Type.usize), size_bytes), try o.builder.intValue(try o.lowerType(Type.usize), size_bytes),
access_kind, access_kind,
fg.ng.ownerModule().no_builtin,
); );
return result_ptr; return result_ptr;
} }
@ -11513,6 +11566,7 @@ pub const FuncGen = struct {
elem_ty.abiAlignment(zcu).toLlvm(), elem_ty.abiAlignment(zcu).toLlvm(),
try o.builder.intValue(try o.lowerType(Type.usize), elem_ty.abiSize(zcu)), try o.builder.intValue(try o.lowerType(Type.usize), elem_ty.abiSize(zcu)),
access_kind, access_kind,
self.ng.ownerModule().no_builtin,
); );
} }

27
src/codegen/llvm/Builder.zig
View file

@ -2634,6 +2634,7 @@ pub const Intrinsic = enum {
cos, cos,
pow, pow,
exp, exp,
exp10,
exp2, exp2,
ldexp, ldexp,
frexp, frexp,
@ -2801,22 +2802,22 @@ pub const Intrinsic = enum {
.va_start = .{ .va_start = .{
.ret_len = 0, .ret_len = 0,
.params = &.{ .params = &.{
.{ .kind = .{ .type = .ptr } }, .{ .kind = .overloaded },
}, },
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn }, .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
}, },
.va_end = .{ .va_end = .{
.ret_len = 0, .ret_len = 0,
.params = &.{ .params = &.{
.{ .kind = .{ .type = .ptr } }, .{ .kind = .overloaded },
}, },
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn }, .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
}, },
.va_copy = .{ .va_copy = .{
.ret_len = 0, .ret_len = 0,
.params = &.{ .params = &.{
.{ .kind = .{ .type = .ptr } }, .{ .kind = .overloaded },
.{ .kind = .{ .type = .ptr } }, .{ .kind = .{ .matches = 0 } },
}, },
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn }, .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .willreturn },
}, },
@ -2929,7 +2930,7 @@ pub const Intrinsic = enum {
.params = &.{ .params = &.{
.{ .kind = .overloaded, .attrs = &.{ .@"noalias", .nocapture, .writeonly } }, .{ .kind = .overloaded, .attrs = &.{ .@"noalias", .nocapture, .writeonly } },
.{ .kind = .overloaded, .attrs = &.{ .@"noalias", .nocapture, .readonly } }, .{ .kind = .overloaded, .attrs = &.{ .@"noalias", .nocapture, .readonly } },
.{ .kind = .overloaded, .attrs = &.{.immarg} }, .{ .kind = .overloaded },
.{ .kind = .{ .type = .i1 }, .attrs = &.{.immarg} }, .{ .kind = .{ .type = .i1 }, .attrs = &.{.immarg} },
}, },
.attrs = &.{ .nocallback, .nofree, .nounwind, .willreturn, .{ .memory = .{ .argmem = .readwrite } } }, .attrs = &.{ .nocallback, .nofree, .nounwind, .willreturn, .{ .memory = .{ .argmem = .readwrite } } },
@ -2959,7 +2960,7 @@ pub const Intrinsic = enum {
.params = &.{ .params = &.{
.{ .kind = .overloaded, .attrs = &.{ .nocapture, .writeonly } }, .{ .kind = .overloaded, .attrs = &.{ .nocapture, .writeonly } },
.{ .kind = .{ .type = .i8 } }, .{ .kind = .{ .type = .i8 } },
.{ .kind = .overloaded, .attrs = &.{.immarg} }, .{ .kind = .overloaded },
.{ .kind = .{ .type = .i1 }, .attrs = &.{.immarg} }, .{ .kind = .{ .type = .i1 }, .attrs = &.{.immarg} },
}, },
.attrs = &.{ .nocallback, .nofree, .nounwind, .willreturn, .{ .memory = .{ .argmem = .write } } }, .attrs = &.{ .nocallback, .nofree, .nounwind, .willreturn, .{ .memory = .{ .argmem = .write } } },
@ -3022,6 +3023,14 @@ pub const Intrinsic = enum {
}, },
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .speculatable, .willreturn, .{ .memory = Attribute.Memory.all(.none) } }, .attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .speculatable, .willreturn, .{ .memory = Attribute.Memory.all(.none) } },
}, },
.exp10 = .{
.ret_len = 1,
.params = &.{
.{ .kind = .overloaded },
.{ .kind = .{ .matches = 0 } },
},
.attrs = &.{ .nocallback, .nofree, .nosync, .nounwind, .speculatable, .willreturn, .{ .memory = Attribute.Memory.all(.none) } },
},
.ldexp = .{ .ldexp = .{
.ret_len = 1, .ret_len = 1,
.params = &.{ .params = &.{
@ -6093,6 +6102,7 @@ pub const WipFunction = struct {
src_align: Alignment, src_align: Alignment,
len: Value, len: Value,
kind: MemoryAccessKind, kind: MemoryAccessKind,
@"inline": bool,
) Allocator.Error!Instruction.Index { ) Allocator.Error!Instruction.Index {
var dst_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = dst_align })}; var dst_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = dst_align })};
var src_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = src_align })}; var src_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = src_align })};
@ -6104,7 +6114,7 @@ pub const WipFunction = struct {
try self.builder.attrs(&dst_attrs), try self.builder.attrs(&dst_attrs),
try self.builder.attrs(&src_attrs), try self.builder.attrs(&src_attrs),
}), }),
.memcpy, if (@"inline") .@"memcpy.inline" else .memcpy,
&.{ dst.typeOfWip(self), src.typeOfWip(self), len.typeOfWip(self) }, &.{ dst.typeOfWip(self), src.typeOfWip(self), len.typeOfWip(self) },
&.{ dst, src, len, switch (kind) { &.{ dst, src, len, switch (kind) {
.normal => Value.false, .normal => Value.false,
@ -6122,12 +6132,13 @@ pub const WipFunction = struct {
val: Value, val: Value,
len: Value, len: Value,
kind: MemoryAccessKind, kind: MemoryAccessKind,
@"inline": bool,
) Allocator.Error!Instruction.Index { ) Allocator.Error!Instruction.Index {
var dst_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = dst_align })}; var dst_attrs = [_]Attribute.Index{try self.builder.attr(.{ .@"align" = dst_align })};
const value = try self.callIntrinsic( const value = try self.callIntrinsic(
.normal, .normal,
try self.builder.fnAttrs(&.{ .none, .none, try self.builder.attrs(&dst_attrs) }), try self.builder.fnAttrs(&.{ .none, .none, try self.builder.attrs(&dst_attrs) }),
.memset, if (@"inline") .@"memset.inline" else .memset,
&.{ dst.typeOfWip(self), len.typeOfWip(self) }, &.{ dst.typeOfWip(self), len.typeOfWip(self) },
&.{ dst, val, len, switch (kind) { &.{ dst, val, len, switch (kind) {
.normal => Value.false, .normal => Value.false,